diff --git a/cmd/runnerd/main.go b/cmd/runnerd/main.go index ecb91e0..293ac48 100644 --- a/cmd/runnerd/main.go +++ b/cmd/runnerd/main.go @@ -3,6 +3,7 @@ package main import ( "context" + "errors" "flag" "fmt" "log" @@ -14,6 +15,7 @@ import ( "strings" "time" + "github.com/tokencanopy/rainier/checkpoint" "github.com/tokencanopy/rainier/internal/driver" "github.com/tokencanopy/rainier/internal/runnerd" ) @@ -68,6 +70,14 @@ func main() { "cgroup v2 mount point; empty means /sys/fs/cgroup. It is where host-side metering reads each microVM's cpu.stat and memory.current from") microvmSeccompOff := flag.Bool("microvm-seccomp-off", os.Getenv("RAINIER_MICROVM_SECCOMP_OFF") == "1", "turn OFF the jailer's seccomp filter on the Firecracker process. Seccomp is on by default and this exists for diagnosing a filter rejection on a new kernel, not for production") + checkpointStoreDir := flag.String("checkpoint-store-dir", envDefault("RAINIER_CHECKPOINT_STORE_DIR", ""), + "host directory holding the portable workspace checkpoints a cold suspend writes (required when --driver=microvm). A cold suspend does not report success until a session's workspace is a committed, verified checkpoint in here (ADR-0003 §4.4), and a deep-dormant resume rebuilds the workspace image from it. A hosted cell puts these in regional object storage instead") + checkpointKeyFile := flag.String("checkpoint-key-file", envDefault("RAINIER_CHECKPOINT_KEY_FILE", ""), + "file holding the 32-byte key every workspace checkpoint is wrapped under, as 64 hex characters or 32 raw bytes, mode 0600 (required when --driver=microvm). Without it the checkpoints in --checkpoint-store-dir would be a tenant's files in the clear on this host; with it, losing this file means losing every checkpoint it wrapped") + checkpointRestoreUID := flag.Int("checkpoint-restore-uid", envIntDefault("RAINIER_CHECKPOINT_RESTORE_UID", 1000), + "uid a restored workspace's files are given to before they become a filesystem: the user the session image runs its agent as. A checkpoint records modes and not owners, so without this a restored workspace comes back owned by root and the agent cannot write it. 0 means leave every file owned by this runner") + checkpointRestoreGID := flag.Int("checkpoint-restore-gid", envIntDefault("RAINIER_CHECKPOINT_RESTORE_GID", 1000), + "gid a restored workspace's files are given to; see --checkpoint-restore-uid") var microvmControlPlane capabilityFlag flag.Var(µvmControlPlane, "microvm-control-plane-cidr", "a regional control-plane range a microVM guest must not be able to reach; repeatable, or set RAINIER_MICROVM_CONTROL_PLANE_CIDRS to a comma-separated list") @@ -127,7 +137,19 @@ func main() { case *microvmImageURL != "": imageSource = driver.HTTPImageSource{Base: *microvmImageURL} } + // Where a cold suspend's workspace checkpoint goes, and what wraps it. + // Both are REQUIRED here, and this is a Fatal for the same reason the + // kernel and the rootfs are: a runner that started without them would + // accept placements and then fail every cold suspend — with a tenant's + // work still inside a VM the durability barrier will not let it + // terminate. + ckpt, err := checkpointConfig(*checkpointStoreDir, *checkpointKeyFile, + *checkpointRestoreUID, *checkpointRestoreGID) + if err != nil { + log.Fatalf("--driver=microvm: %v", err) + } mvm, err := driver.NewMicrovm(driver.MicrovmOpts{ + Checkpoint: ckpt, KernelPath: *kernelPath, BaseRootfs: *rootfsPath, StateDir: *microvmStateDir, @@ -214,6 +236,43 @@ func main() { } } +// checkpointConfig builds the microVM driver's checkpoint configuration from +// the two required flags, or says which one is missing and why it matters. +// +// It is a function of its arguments rather than of the flag variables so that +// the refusals are testable: every one of them is a condition a real deployment +// gets wrong at some point, and "the runner started anyway" is the outcome none +// of them may have. +func checkpointConfig(storeDir, keyFile string, uid, gid int) (*driver.CheckpointOpts, error) { + switch { + case storeDir == "": + return nil, errors.New("--checkpoint-store-dir is required: a cold suspend turns a session's workspace into a portable checkpoint before it terminates the VM (ADR-0003 §4.4), and there is deliberately no default — a temp-directory one would put a tenant's only durable copy somewhere the host reaps") + case keyFile == "": + return nil, errors.New("--checkpoint-key-file is required: a workspace checkpoint is encrypted under it, and an unencrypted one would be a tenant's files in the clear on this host") + case uid < 0 || gid < 0: + return nil, errors.New("--checkpoint-restore-uid and --checkpoint-restore-gid must not be negative") + } + store, err := driver.NewDirBlobStore(storeDir) + if err != nil { + return nil, err + } + key, err := driver.LoadCheckpointKey(keyFile) + if err != nil { + return nil, err + } + keys, err := checkpoint.NewStaticKeyWrapper(driver.SelfHostedCheckpointKeyRef, key) + if err != nil { + return nil, err + } + return &driver.CheckpointOpts{ + Store: store, + Keys: keys, + KeyRef: driver.SelfHostedCheckpointKeyRef, + OwnerUID: uid, + OwnerGID: gid, + }, nil +} + // egressProxyEndpoint resolves the one host-side destination a microVM // guest's firewall allows, from --microvm-egress-proxy or, failing that, from // --proxy-url. diff --git a/cmd/runnerd/main_test.go b/cmd/runnerd/main_test.go index ea8d4ff..fd67c93 100644 --- a/cmd/runnerd/main_test.go +++ b/cmd/runnerd/main_test.go @@ -2,84 +2,108 @@ package main import ( + "os" + "path/filepath" "strings" "testing" ) -// TestEgressProxyEndpoint covers the one flag that decides what a microVM -// guest's firewall lets out. -func TestEgressProxyEndpoint(t *testing.T) { +// The two flags a microVM runner cannot start without, and what each refusal +// has to say. They are Fatals in main for the reason the kernel and the rootfs +// are: a runner that started without a checkpoint store would accept placements +// and then fail every cold suspend, with a tenant's work still inside a VM the +// durability barrier will not let it terminate. + +func keyFile(t *testing.T) string { + t.Helper() + p := filepath.Join(t.TempDir(), "checkpoint.key") + key := make([]byte, 32) + for i := range key { + key[i] = byte(i + 1) + } + if err := os.WriteFile(p, key, 0o600); err != nil { + t.Fatal(err) + } + return p +} + +func TestCheckpointConfigBuildsTheDriversPorts(t *testing.T) { + dir := filepath.Join(t.TempDir(), "checkpoints") + opts, err := checkpointConfig(dir, keyFile(t), 1000, 1000) + if err != nil { + t.Fatalf("checkpointConfig: %v", err) + } + if opts.Store == nil || opts.Keys == nil { + t.Fatalf("the configuration is %+v, want a store and a key wrapper", opts) + } + if opts.KeyRef == "" { + t.Error("the configuration names no key reference; a manifest has to record one") + } + if opts.OwnerUID != 1000 || opts.OwnerGID != 1000 { + t.Errorf("the restored workspace would be given to %d:%d, want 1000:1000", opts.OwnerUID, opts.OwnerGID) + } + // The store directory is created, because a runner that started without one + // would discover it at the worst possible moment. + info, err := os.Stat(dir) + if err != nil { + t.Fatalf("the store directory was not created: %v", err) + } + if info.Mode().Perm() != 0o700 { + t.Errorf("the store directory is mode %v, want 0700", info.Mode().Perm()) + } +} + +func TestCheckpointConfigRefusesWhatARunnerCannotStartWith(t *testing.T) { + good := keyFile(t) + loose := filepath.Join(t.TempDir(), "loose.key") + if err := os.WriteFile(loose, make([]byte, 32), 0o644); err != nil { + t.Fatal(err) + } + cases := []struct { - name string - explicit string - proxyURL string - wantAddr string - wantPort int - wantErr string + name string + dir string + key string + uid int + gid int + wants string }{ - { - name: "no proxy at all is a runner with no exception", - wantAddr: "", - wantPort: 0, - }, - { - name: "an explicit address wins", - explicit: "10.44.0.9:3128", - proxyURL: "http://egressd:3129", - wantAddr: "10.44.0.9", - wantPort: 3128, - }, - { - name: "a proxy URL that is already an address", - proxyURL: "http://10.44.0.9:3129", - wantAddr: "10.44.0.9", - wantPort: 3129, - }, - { - name: "a proxy URL with no port takes the scheme's", - proxyURL: "https://10.44.0.9", - wantAddr: "10.44.0.9", - wantPort: 443, - }, - { - name: "a proxy URL naming a host must be given as an address", - proxyURL: "http://egressd:3129", - wantErr: "names a host and not an address", - }, - { - name: "an explicit endpoint that is not ip:port", - explicit: "10.44.0.9", - wantErr: "not ip:port", - }, - { - name: "an explicit endpoint with a nonsense port", - explicit: "10.44.0.9:not-a-port", - wantErr: "not a port", - }, - { - name: "an IPv6 proxy no guest could reach", - explicit: "[fd00::1]:3128", - wantErr: "IPv6", - }, + {"no store directory", "", good, 1000, 1000, "--checkpoint-store-dir is required"}, + {"no key file", t.TempDir(), "", 1000, 1000, "--checkpoint-key-file is required"}, + {"a key file that is not there", t.TempDir(), filepath.Join(t.TempDir(), "nope"), 1000, 1000, "checkpoint key file"}, + {"a key anyone can read", t.TempDir(), loose, 1000, 1000, "readable by group or other"}, + {"a negative uid", t.TempDir(), good, -1, 1000, "must not be negative"}, } for _, tc := range cases { t.Run(tc.name, func(t *testing.T) { - addr, port, err := egressProxyEndpoint(tc.explicit, tc.proxyURL) - if tc.wantErr != "" { - if err == nil { - t.Fatalf("egressProxyEndpoint(%q, %q) = %q/%d, want an error naming %q", tc.explicit, tc.proxyURL, addr, port, tc.wantErr) - } - if !strings.Contains(err.Error(), tc.wantErr) { - t.Fatalf("error = %q, want it to name %q", err, tc.wantErr) - } - return - } - if err != nil { - t.Fatalf("egressProxyEndpoint(%q, %q): %v", tc.explicit, tc.proxyURL, err) + _, err := checkpointConfig(tc.dir, tc.key, tc.uid, tc.gid) + if err == nil { + t.Fatal("the configuration was accepted") } - if addr != tc.wantAddr || port != tc.wantPort { - t.Fatalf("egressProxyEndpoint(%q, %q) = %q/%d, want %q/%d", tc.explicit, tc.proxyURL, addr, port, tc.wantAddr, tc.wantPort) + if !strings.Contains(err.Error(), tc.wants) { + t.Errorf("the refusal is %q, want it to name %q", err, tc.wants) } }) } } + +// TestCheckpointConfigSaysWhyRatherThanWhat. These lines are what an operator +// reads at three in the morning, so each one names the consequence and not just +// the missing argument. +func TestCheckpointConfigSaysWhyRatherThanWhat(t *testing.T) { + _, err := checkpointConfig("", keyFile(t), 1000, 1000) + if err == nil { + t.Fatal("an empty store directory was accepted") + } + for _, want := range []string{"portable checkpoint", "no default"} { + if !strings.Contains(err.Error(), want) { + t.Errorf("the refusal %q does not explain %q", err, want) + } + } + if _, err = checkpointConfig(t.TempDir(), "", 1000, 1000); err == nil { + t.Fatal("an empty key file was accepted") + } + if !strings.Contains(err.Error(), "in the clear") { + t.Errorf("the refusal %q does not explain what an unencrypted checkpoint would be", err) + } +} diff --git a/cmd/sessiond/coldsuspend_test.go b/cmd/sessiond/coldsuspend_test.go index 3a0a95b..e91aea3 100644 --- a/cmd/sessiond/coldsuspend_test.go +++ b/cmd/sessiond/coldsuspend_test.go @@ -13,13 +13,17 @@ import ( // what is exercised is the frame's whole journey — OnControl's kind // dispatch, the `cold` flag, the handler, and the two answers — rather than // quiesceCold called directly. -func coldWired(execs execKiller, boots *bootstrapper) (*rpcDispatcher, *recordingSender) { +// stream is the workspace streamer the handler runs before it answers ready; a +// nil one is a sandbox that does not stream, which is what these two tests +// want — the secrets and the execs are their subject, and the stream has a file +// of its own (workspacestream_test.go). +func coldWired(execs execKiller, boots *bootstrapper, stream workspaceStreamer) (*rpcDispatcher, *recordingSender) { d := newRPCDispatcher() sender := &recordingSender{} d.online(sender) d.RegisterEventHandler(relay.KindSuspending, func(ev relay.ControlEvent) { if ev.Cold { - quiesceCold(execs, d, nil, boots, ev.ID) + quiesceCold(execs, d, nil, boots, ev.ID, stream) return } quiesceExecs(execs, d, ev.ID) @@ -50,7 +54,7 @@ func TestAColdSuspendForgetsTheDeliveredSecrets(t *testing.T) { boots.remember([]string{"DEPLOY_KEY"}) execs := &recordingExecs{} - d, sender := coldWired(execs, boots) + d, sender := coldWired(execs, boots, nil) d.OnControl(suspendingFrame(t, true, 91)) if _, set := os.LookupEnv("DEPLOY_KEY"); set { @@ -85,7 +89,7 @@ func TestAWarmSuspendKeepsTheDeliveredSecrets(t *testing.T) { boots.remember([]string{"DEPLOY_KEY"}) execs := &recordingExecs{} - d, sender := coldWired(execs, boots) + d, sender := coldWired(execs, boots, nil) d.OnControl(suspendingFrame(t, false, 92)) if os.Getenv("DEPLOY_KEY") != "value_example" { diff --git a/cmd/sessiond/flush_test.go b/cmd/sessiond/flush_test.go index 6a492ff..e6661e5 100644 --- a/cmd/sessiond/flush_test.go +++ b/cmd/sessiond/flush_test.go @@ -28,7 +28,7 @@ func flushWired(execs execKiller, boots *bootstrapper, agents agentFlusher) (*rp }) d.RegisterEventHandler(relay.KindSuspending, func(ev relay.ControlEvent) { if ev.Cold { - quiesceCold(execs, d, nil, boots, ev.ID) + quiesceCold(execs, d, nil, boots, ev.ID, nil) return } quiesceExecs(execs, d, ev.ID) diff --git a/cmd/sessiond/main.go b/cmd/sessiond/main.go index 2aa95ac..b81617c 100644 --- a/cmd/sessiond/main.go +++ b/cmd/sessiond/main.go @@ -303,17 +303,30 @@ func main() { rpc.RegisterEventHandler(relay.KindFlush, func(ev relay.ControlEvent) { flushDisks(rpc, agentFlush, ev.ID) }) + // The workspace this session's cold suspend checkpoints. It is built + // here, beside the handler that uses it, because it needs the same + // dispatcher: the stream shares the handshake's writer, so that a chunk + // cannot interleave with the end marker that closes it. + streamer := streamWorkspaceFn(rpc, workspaceRoot) rpc.RegisterEventHandler(relay.KindSuspending, func(ev relay.ControlEvent) { if ev.Cold { // Not a freeze: this VM is ending, with no memory image // written anywhere. Everything that has to survive has to be - // on a disk by the time this answers, and everything that - // must not survive has to be gone. - quiesceCold(execs, rpc, agents, boots, ev.ID) + // on a disk — or, for the workspace, on the wire — by the time + // this answers, and everything that must not survive has to be + // gone. + quiesceCold(execs, rpc, agents, boots, ev.ID, streamer) return } quiesceExecs(execs, rpc, ev.ID) }) + // The host saying the checkpoint is committed and verified. Nothing is + // done with it and nothing is answered: the VM is about to be + // terminated, and the value of the event is that this session's own log + // says whether its work made it out. + rpc.RegisterEventHandler(relay.KindCheckpointCommitted, func(ev relay.ControlEvent) { + log.Printf("the runner reports this workspace's checkpoint committed and verified (suspend %d)", ev.ID) + }) } go func() { @@ -648,12 +661,23 @@ const agentHomeMount = "/rainier/agents" // is what makes the home's filesystem consistent before the block device // goes away under it; forgetting is belt and braces beside a VM that is // about to cease to exist, and it costs nothing. +// 4. STREAM the workspace to the host, which is the only copy of the user's +// work that survives this VM (ADR-0003 §2.3's deep-dormant tier). It runs +// LAST of the four, after the flush that put the agent's pending write on +// a disk and after the kill that stopped anything still writing, and before +// the answer, because the host may not terminate this VM until the tree is +// out of it. // -// Every step is best effort and the answer goes out regardless, for the same -// reason the warm path's does: the host terminates the VM when it hears -// nothing, so staying silent would only make the stop slower and put the -// reason nowhere. -func quiesceCold(execs execKiller, notifier eventNotifier, agents *agentSync, boots *bootstrapper, nonce uint64) { +// The first three steps are best effort and the answer goes out regardless, for +// the same reason the warm path's does: the host terminates the VM when it +// hears nothing, so staying silent would only make the stop slower and put the +// reason nowhere. The stream is best effort in the same sense and in no other: +// a failure is REPORTED, in the end marker, with counts and a stage — and it is +// the host that then refuses the suspend and keeps this VM alive. A guest +// cannot make that decision for itself, because a guest does not know whether +// its checkpoint committed. +func quiesceCold(execs execKiller, notifier eventNotifier, agents *agentSync, boots *bootstrapper, + nonce uint64, stream workspaceStreamer) { if err := notifier.Notify(relay.ControlEvent{Kind: relay.KindSuspendAck, ID: nonce}); err != nil { log.Printf("acknowledging the cold suspend notice: %v", err) } @@ -668,6 +692,13 @@ func quiesceCold(execs execKiller, notifier eventNotifier, agents *agentSync, bo if n := boots.forget(); n > 0 { log.Printf("forgot %d delivered environment secret(s) before this VM ends", n) } + if stream != nil { + rep, err := stream(nonce) + logWorkspaceEnd(rep, err) + if err := notifier.Notify(workspaceEnd(nonce, rep, err)); err != nil { + log.Printf("reporting the end of the workspace stream: %v", err) + } + } if err := notifier.Notify(relay.ControlEvent{Kind: relay.KindSuspendReady, ID: nonce}); err != nil { log.Printf("reporting the cold suspend ready: %v", err) } @@ -778,6 +809,12 @@ func (m *execCountMailbox) c() <-chan []byte { return m.ch } // can be tested without standing up a conn to have a sender for. type controlSender interface { Send(payload []byte) error + // SendStream is the second thing a connection carries: one chunk of the + // workspace a cold suspend is checkpointing. It is on this interface rather + // than on a second one because it goes through the same writer as Send — + // see relay.ControlSender.SendStream — and a type that could do one without + // the other would be a type that could interleave them. + SendStream(id uint64, chunk []byte) error } // pendingCap bounds the queue of control payloads waiting for a connection, diff --git a/cmd/sessiond/main_test.go b/cmd/sessiond/main_test.go index 043869b..bf9f595 100644 --- a/cmd/sessiond/main_test.go +++ b/cmd/sessiond/main_test.go @@ -314,6 +314,13 @@ func (s *stubSender) Send(p []byte) error { return nil } +// SendStream is the other half of controlSender. serveConn never calls it — +// only the cold suspend's workspace stream does — so this stub records the +// chunks and nothing reads them; the streaming tests use recordingSender. +func (s *stubSender) SendStream(_ uint64, chunk []byte) error { + return s.Send(chunk) +} + func (s *stubSender) count() int { s.mu.Lock() defer s.mu.Unlock() diff --git a/cmd/sessiond/rpc.go b/cmd/sessiond/rpc.go index 0fcd01b..7543d17 100644 --- a/cmd/sessiond/rpc.go +++ b/cmd/sessiond/rpc.go @@ -120,6 +120,18 @@ func (d *rpcDispatcher) Notify(ev relay.ControlEvent) error { return conn.sender.Send(b) } +// SendStream writes one chunk of stream id over the live connection, so that +// the workspace stream travels on the same writer as the handshake it is part +// of. Like Notify, no connection is an error rather than a silent drop: the +// host is waiting for these bytes and a dropped chunk is a truncated workspace. +func (d *rpcDispatcher) SendStream(id uint64, chunk []byte) error { + conn := d.conn.Load() + if conn == nil { + return errNoRPCConn + } + return conn.sender.SendStream(id, chunk) +} + var errNoRPCConn = errors.New("sessiond: no connection to runnerd") // online installs the sender for a freshly established connection. Any diff --git a/cmd/sessiond/rpc_test.go b/cmd/sessiond/rpc_test.go index 1b6242c..26a066f 100644 --- a/cmd/sessiond/rpc_test.go +++ b/cmd/sessiond/rpc_test.go @@ -419,11 +419,21 @@ func (r *recordingExecs) state() (int, time.Duration) { return r.calls, r.budget } -// recordingSender records the payloads a dispatcher sends upstream. +// recordingSender records the payloads a dispatcher sends upstream: control +// frames in `sent`, and the chunks of a workspace stream in `chunks`, keyed by +// the stream id so a test can prove the stream travelled under the suspend's +// own nonce. type recordingSender struct { - mu sync.Mutex - sent [][]byte - err error + mu sync.Mutex + sent [][]byte + chunks map[uint64][]byte + err error + // streamErr, when set, fails every chunk — a runner that has stopped + // taking the workspace stream. + streamErr error + // afterChunks, when positive, is how many chunks succeed before streamErr + // starts: a conn that dies MID-stream rather than before it. + afterChunks int } func (s *recordingSender) Send(p []byte) error { @@ -436,6 +446,29 @@ func (s *recordingSender) Send(p []byte) error { return nil } +func (s *recordingSender) SendStream(id uint64, chunk []byte) error { + s.mu.Lock() + defer s.mu.Unlock() + if s.streamErr != nil { + if s.afterChunks <= 0 { + return s.streamErr + } + s.afterChunks-- + } + if s.chunks == nil { + s.chunks = map[uint64][]byte{} + } + s.chunks[id] = append(s.chunks[id], chunk...) + return nil +} + +// stream returns everything sent under one stream id. +func (s *recordingSender) stream(id uint64) []byte { + s.mu.Lock() + defer s.mu.Unlock() + return append([]byte(nil), s.chunks[id]...) +} + func (s *recordingSender) events() []relay.ControlEvent { s.mu.Lock() defer s.mu.Unlock() diff --git a/cmd/sessiond/workspacestream.go b/cmd/sessiond/workspacestream.go new file mode 100644 index 0000000..0725ba4 --- /dev/null +++ b/cmd/sessiond/workspacestream.go @@ -0,0 +1,206 @@ +package main + +import ( + "context" + "errors" + "fmt" + "io" + "io/fs" + "log" + "os" + "time" + + "github.com/tokencanopy/rainier/checkpoint" + "github.com/tokencanopy/rainier/internal/relay" + "github.com/tokencanopy/rainier/internal/wstream" +) + +// The guest's half of the cold-suspend checkpoint: this VM is ending, and the +// only copy of the user's work that will survive it is the one this file puts +// on the wire. +// +// It runs INSIDE the cold suspend handshake, after the flush, the exec kill and +// the agent-home unmount, and before `suspend_ready` — because the host may not +// terminate the VM until the tree is out of it, and `suspend_ready` is what +// tells the host it may. See docs/design/2026-09-23-workspace-checkpoint-wiring.md. + +// streamChunkBytes is one FrameStream's payload. A frame is JSON with a +// base64'd payload, so 64 KiB of workspace becomes about 87 KiB on the wire — +// comfortably under the transport's 16 MiB frame limit and large enough that +// the per-frame cost is noise beside the copy. +const streamChunkBytes = 64 << 10 + +// workspaceStreamer is what the cold suspend handler needs from this file: put +// the workspace on the wire under this suspend's nonce, and say what went. It +// is a function rather than an interface because there is exactly one +// implementation and the tests need to substitute a failure, not a type. +// +// A nil one is a sandbox that does not stream: the local dev surface, and a +// Docker session, which never receives a cold notice at all (runnerd sends it +// only for a driver that withholds secrets). The handler then answers +// `suspend_ready` exactly as it did before this existed, and the host's own +// stream budget is what reports that no checkpoint was taken. +type workspaceStreamer func(nonce uint64) (wstream.Report, error) + +// streamWorkspaceFn builds the streamer main installs: the session's workspace +// root, streamed to the runner over the live control connection. +// +// The exclusions are the checkpoint library's own, asked for here rather than +// spelled: the bytes that must not leave this guest and the bytes the host +// would refuse are the same set, and a second spelling of it is a second thing +// to forget to update. +func streamWorkspaceFn(sender streamSender, root string) workspaceStreamer { + return func(nonce uint64) (wstream.Report, error) { + ctx, cancel := context.WithTimeout(context.Background(), workspaceStreamBudget) + defer cancel() + return streamWorkspace(ctx, sender, nonce, os.DirFS(root)) + } +} + +// workspaceStreamBudget bounds the guest's own side of the copy, and it is +// deliberately longer than the host's total stream budget (30 minutes, see the +// design note's §4). The host is the end that decides a stream has failed — +// it is the one holding the checkpoint writer — and a guest that gave up first +// would turn "the host is still reading" into a truncated stream. +const workspaceStreamBudget = 45 * time.Minute + +// streamSender is the one method this needs from the live connection: one +// chunk of one stream. It is the same writer every control frame goes through, +// so a chunk cannot interleave with the end marker that follows it. +type streamSender interface { + SendStream(id uint64, chunk []byte) error +} + +func streamWorkspace(ctx context.Context, sender streamSender, nonce uint64, fsys fs.FS) (wstream.Report, error) { + w := &chunkWriter{ + send: func(chunk []byte) error { return sender.SendStream(nonce, chunk) }, + buf: make([]byte, 0, streamChunkBytes), + } + rep, err := wstream.Write(ctx, fsys, w, wstream.Limits{Exclude: checkpoint.DefaultExclusions()}) + if err != nil { + return rep, err + } + if err := w.Flush(); err != nil { + return rep, err + } + // The counts the end marker carries are the counts of what actually + // reached the writer, so the host's "did I receive what you sent" check is + // against bytes this end put on the wire rather than bytes it walked. + rep.Bytes = w.sent + return rep, nil +} + +// chunkWriter turns the stream wstream.Write produces into FrameStream +// payloads of a bounded size. +// +// It buffers exactly one chunk and never more: the whole point of this design +// is that a workspace never exists in memory on either end, and a writer that +// grew its buffer to whatever it was handed would put the guest's side of that +// promise in the hands of the copy buffer above it. +type chunkWriter struct { + send func([]byte) error + buf []byte + sent int64 +} + +func (c *chunkWriter) Write(p []byte) (int, error) { + total := len(p) + for len(p) > 0 { + n := min(cap(c.buf)-len(c.buf), len(p)) + c.buf = append(c.buf, p[:n]...) + p = p[n:] + if len(c.buf) == cap(c.buf) { + if err := c.Flush(); err != nil { + return total - len(p), err + } + } + } + return total, nil +} + +// Flush sends whatever is pending. An empty buffer sends nothing: a +// zero-length frame would be indistinguishable from one that was lost. +func (c *chunkWriter) Flush() error { + if len(c.buf) == 0 { + return nil + } + if err := c.send(c.buf); err != nil { + return fmt.Errorf("wstream: the workspace chunk could not be sent to the runner: %w", errSendChunk) + } + c.sent += int64(len(c.buf)) + c.buf = c.buf[:0] + return nil +} + +// errSendChunk is what a failed send becomes, and the transport's own error is +// deliberately dropped rather than wrapped: a net error quotes the socket it +// failed on, which on a microVM host is a path inside the VM's jail, and this +// sentence travels to a session's error column. There is nothing in the +// original a person could act on that "the runner is not taking bytes" does not +// already say. +var errSendChunk = errors.New("the runner is not taking the workspace stream") + +// workspaceEnd is the end marker: the counts on success, the stage on failure, +// and never a path in either. +// +// A failure is reported in the `stage_failed` shape the boot chain already +// uses — a stage, an rc of -1, and a tail — because it is the same kind of +// fact: a named step of this session's lifecycle did not finish, and the +// sentence is the only thing that reaches whoever has to act on it. +func workspaceEnd(nonce uint64, rep wstream.Report, err error) relay.ControlEvent { + ev := relay.ControlEvent{ + Kind: relay.KindWorkspaceEnd, + ID: nonce, + Entries: rep.Entries, + Bytes: rep.Bytes, + } + if err == nil { + ev.OK = true + return ev + } + ev.Stage = workspaceStreamStage + ev.RC = -1 + ev.Tail = streamFailureTail(err) + return ev +} + +// workspaceStreamStage is the stage name a failed workspace stream reports. It +// is not one of the boot chain's stages — this one happens at the end of a +// session's life rather than the start — and it is spelled here, beside the +// event that carries it. +const workspaceStreamStage = "workspace_stream" + +// streamFailureTail is the sentence that travels with a failure. +// +// Only this package's and wstream's own errors are forwarded: both are written +// to carry no path, no file name and no symlink target (wstream's package +// comment). Anything else is reduced to a flat sentence, because an error from +// a file system, a transport or the standard library is not known to be +// path-free and this text reaches an operator's log. +func streamFailureTail(err error) string { + switch { + case errors.Is(err, wstream.ErrLimit), + errors.Is(err, wstream.ErrEntry), + errors.Is(err, wstream.ErrSource), + errors.Is(err, wstream.ErrFormat), + errors.Is(err, errSendChunk): + return err.Error() + case errors.Is(err, context.DeadlineExceeded): + return fmt.Sprintf("the workspace was still being streamed after %s", workspaceStreamBudget) + default: + return "the workspace could not be streamed to the runner" + } +} + +// logWorkspaceEnd puts the outcome in the session's own log, which is the one +// place it survives the VM. +func logWorkspaceEnd(rep wstream.Report, err error) { + if err != nil { + log.Printf("the workspace stream failed after %d entries and %d bytes: %v", rep.Entries, rep.Bytes, err) + return + } + log.Printf("streamed the workspace to the runner: %d entries, %d bytes, %d entries skipped as neither file, directory nor symlink", + rep.Entries, rep.Bytes, rep.Skipped) +} + +var _ io.Writer = (*chunkWriter)(nil) diff --git a/cmd/sessiond/workspacestream_test.go b/cmd/sessiond/workspacestream_test.go new file mode 100644 index 0000000..93ed6da --- /dev/null +++ b/cmd/sessiond/workspacestream_test.go @@ -0,0 +1,287 @@ +package main + +import ( + "archive/tar" + "bytes" + "encoding/binary" + "errors" + "io" + "os" + "path/filepath" + "strings" + "testing" + + "github.com/tokencanopy/rainier/internal/relay" + "github.com/tokencanopy/rainier/internal/wstream" +) + +// The guest's half of the cold-suspend checkpoint, wired the way main wires it: +// the frame arrives, the handler quiesces, the workspace goes out as a stream +// on the same connection, and the end marker and the ready answer follow it in +// that order. + +// workspaceFixture builds a session workspace: a file, a directory, a symbolic +// link, and the Rainier-owned path that must never leave the guest. +func workspaceFixture(t *testing.T) string { + t.Helper() + root := t.TempDir() + for rel, content := range map[string]string{ + "README.md": "hello", + "src/main.go": "package main\n", + ".rainier/session.json": "{\"token\":\"must-not-travel\"}", + } { + p := filepath.Join(root, rel) + if err := os.MkdirAll(filepath.Dir(p), 0o755); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(p, []byte(content), 0o644); err != nil { + t.Fatal(err) + } + } + if err := os.Symlink("src/main.go", filepath.Join(root, "entry")); err != nil { + t.Fatal(err) + } + return root +} + +// coldStreamWired is coldWired with the real streamer over root, so what these +// tests exercise is the whole journey: OnControl's dispatch, the cold branch, +// the stream, the end marker and the answers. +func coldStreamWired(t *testing.T, root string) (*rpcDispatcher, *recordingSender) { + t.Helper() + d := newRPCDispatcher() + sender := &recordingSender{} + d.online(sender) + d.RegisterEventHandler(relay.KindSuspending, func(ev relay.ControlEvent) { + if ev.Cold { + quiesceCold(&recordingExecs{}, d, nil, &bootstrapper{}, ev.ID, streamWorkspaceFn(d, root)) + return + } + quiesceExecs(&recordingExecs{}, d, ev.ID) + }) + return d, sender +} + +// decodeStream reads a workspace stream back the way the host does: the magic, +// the index, then the tar. +func decodeStream(t *testing.T, b []byte) (index []string, tarNames map[string]*tar.Header, bodies map[string]string) { + t.Helper() + if !bytes.HasPrefix(b, []byte(wstream.Magic)) { + t.Fatalf("the stream does not begin with the magic line (%d bytes)", len(b)) + } + r := bytes.NewReader(b[len(wstream.Magic):]) + for { + n, err := binary.ReadUvarint(r) + if err != nil { + t.Fatalf("reading an index length: %v", err) + } + if n == 0 { + break + } + rec := make([]byte, n) + if _, err := io.ReadFull(r, rec); err != nil { + t.Fatalf("reading an index record: %v", err) + } + index = append(index, string(rec[0:1])+" "+string(rec[1:])) + } + tarNames, bodies = map[string]*tar.Header{}, map[string]string{} + tr := tar.NewReader(r) + for { + hdr, err := tr.Next() + if errors.Is(err, io.EOF) { + return index, tarNames, bodies + } + if err != nil { + t.Fatalf("reading the tar: %v", err) + } + body, err := io.ReadAll(tr) + if err != nil { + t.Fatalf("reading a tar body: %v", err) + } + tarNames[hdr.Name] = hdr + bodies[hdr.Name] = string(body) + } +} + +// TestAColdSuspendStreamsTheWorkspaceBeforeItAnswersReady is §4.4 from inside +// the guest: the only copy of the user's work that survives this VM is the one +// this puts on the wire, and it goes out BEFORE the answer that lets the host +// terminate the VM. +func TestAColdSuspendStreamsTheWorkspaceBeforeItAnswersReady(t *testing.T) { + cleanEnv(t) + root := workspaceFixture(t) + d, sender := coldStreamWired(t, root) + + d.OnControl(suspendingFrame(t, true, 71)) + + got := sender.events() + if len(got) != 3 { + t.Fatalf("the sandbox answered %d events, want an ack, an end marker and a ready: %+v", len(got), got) + } + if got[0].Kind != relay.KindSuspendAck || got[1].Kind != relay.KindWorkspaceEnd || got[2].Kind != relay.KindSuspendReady { + t.Fatalf("the answers are %q, %q, %q; want ack, workspace_end, suspend_ready", + got[0].Kind, got[1].Kind, got[2].Kind) + } + for _, ev := range got { + if ev.ID != 71 { + t.Fatalf("an answer carries nonce %d, want the notice's 71", ev.ID) + } + } + end := got[1] + if !end.OK { + t.Fatalf("the end marker reports a failure: %+v", end) + } + + raw := sender.stream(71) + if int64(len(raw)) != end.Bytes { + t.Errorf("the end marker claims %d bytes, the stream carried %d", end.Bytes, len(raw)) + } + index, headers, bodies := decodeStream(t, raw) + if int64(len(index)) != end.Entries { + t.Errorf("the end marker claims %d entries, the index names %d", end.Entries, len(index)) + } + // The tree, in fs.WalkDir order, which is the order the checkpoint writer + // walks in and therefore the only order the host can read. + want := []string{"f README.md", "l entry", "d src", "f src/main.go"} + if len(index) != len(want) { + t.Fatalf("the index names %v, want %v", index, want) + } + for i := range want { + if index[i] != want[i] { + t.Fatalf("the index names %v, want %v", index, want) + } + } + if bodies["README.md"] != "hello" { + t.Errorf("README.md arrived as %q", bodies["README.md"]) + } + // A symbolic link travels as a link with its target, never followed. + if hdr := headers["entry"]; hdr.Typeflag != tar.TypeSymlink || hdr.Linkname != "src/main.go" { + t.Errorf("entry arrived as type %c pointing at %q, want a symlink to src/main.go", hdr.Typeflag, hdr.Linkname) + } +} + +// TestAColdSuspendStreamsNothingRainierOwns. The exclusion is a promise about +// the bytes that LEAVE the guest: /workspace/.rainier is Rainier's own control +// directory, and a bootstrap token in it must not travel into a checkpoint +// that outlives the session. +func TestAColdSuspendStreamsNothingRainierOwns(t *testing.T) { + cleanEnv(t) + root := workspaceFixture(t) + d, sender := coldStreamWired(t, root) + + d.OnControl(suspendingFrame(t, true, 72)) + + raw := sender.stream(72) + if bytes.Contains(raw, []byte("must-not-travel")) { + t.Fatal("the stream carries the contents of .rainier") + } + if bytes.Contains(raw, []byte(".rainier")) { + t.Fatal("the stream names .rainier") + } +} + +// TestAFailedWorkspaceStreamReportsTheStageAndTheCounts: the conn stops taking +// bytes half way through. The guest cannot decide that the suspend has failed — +// it does not know whether a checkpoint committed — so it REPORTS, in the +// stage_failed shape, with the counts it reached and no path at all, and the +// host is what refuses the suspend. +func TestAFailedWorkspaceStreamReportsTheStageAndTheCounts(t *testing.T) { + cleanEnv(t) + root := workspaceFixture(t) + // Big enough to need several chunks, so the failure lands MID-stream + // rather than before it. + big := bytes.Repeat([]byte("x"), 3*streamChunkBytes) + if err := os.WriteFile(filepath.Join(root, "big.bin"), big, 0o644); err != nil { + t.Fatal(err) + } + + d := newRPCDispatcher() + sender := &recordingSender{streamErr: errors.New("write unix /srv/rainier/state/instances/i-7/v1.sock_1024: broken pipe"), afterChunks: 1} + d.online(sender) + d.RegisterEventHandler(relay.KindSuspending, func(ev relay.ControlEvent) { + quiesceCold(&recordingExecs{}, d, nil, &bootstrapper{}, ev.ID, streamWorkspaceFn(d, root)) + }) + + d.OnControl(suspendingFrame(t, true, 73)) + + got := sender.events() + if len(got) != 3 { + t.Fatalf("the sandbox answered %d events, want an ack, an end marker and a ready: %+v", len(got), got) + } + end := got[1] + if end.Kind != relay.KindWorkspaceEnd || end.OK { + t.Fatalf("the end marker is %+v, want a workspace_end reporting failure", end) + } + if end.Stage != workspaceStreamStage || end.RC != -1 { + t.Errorf("the failure names stage %q rc %d, want %q and -1", end.Stage, end.RC, workspaceStreamStage) + } + if end.Entries == 0 || end.Bytes == 0 { + t.Errorf("the failure reports %d entries and %d bytes, want the counts it reached", end.Entries, end.Bytes) + } + // The transport's own error quotes the socket it failed on, which on a + // microVM host is a path inside the VM's jail. It must not reach a + // session's error column. + for _, leak := range []string{root, "big.bin", "README.md", "v1.sock", "/srv/rainier"} { + if strings.Contains(end.Tail, leak) { + t.Errorf("the failure's tail quotes %q: %q", leak, end.Tail) + } + } + if !strings.Contains(end.Tail, "not taking the workspace stream") { + t.Errorf("the failure's tail says %q, which names no cause", end.Tail) + } + // And the sandbox still answers ready: the host terminates the VM when it + // hears nothing, so silence would only make the stop slower. + if got[2].Kind != relay.KindSuspendReady { + t.Errorf("the last answer is %q, want suspend_ready", got[2].Kind) + } +} + +// TestAWorkspaceStreamWithNoConnectionFailsRatherThanDropping: a chunk with +// nowhere to go is a truncated workspace, and a truncated workspace that was +// reported as complete is the failure the whole barrier exists to prevent. +func TestAWorkspaceStreamWithNoConnectionFails(t *testing.T) { + cleanEnv(t) + root := workspaceFixture(t) + d := newRPCDispatcher() // never online + if _, err := streamWorkspaceFn(d, root)(74); err == nil { + t.Fatal("streaming with no connection reported success") + } +} + +// TestTheChunkWriterHoldsOneChunk. The whole design is that a workspace never +// exists in memory on either end; a writer that grew its buffer to whatever it +// was handed would put the guest's side of that promise in the hands of +// whatever called it. +func TestTheChunkWriterHoldsOneChunk(t *testing.T) { + var sent [][]byte + w := &chunkWriter{ + send: func(chunk []byte) error { + sent = append(sent, append([]byte(nil), chunk...)) + return nil + }, + buf: make([]byte, 0, 4), + } + if _, err := w.Write([]byte("abcdefghij")); err != nil { + t.Fatal(err) + } + if err := w.Flush(); err != nil { + t.Fatal(err) + } + want := []string{"abcd", "efgh", "ij"} + if len(sent) != len(want) { + t.Fatalf("the writer sent %d chunks (%q), want %v", len(sent), sent, want) + } + for i := range want { + if string(sent[i]) != want[i] { + t.Fatalf("chunk %d is %q, want %q", i, sent[i], want[i]) + } + } + if w.sent != 10 { + t.Errorf("the writer counted %d bytes, want 10", w.sent) + } + // A second Flush sends nothing: a zero-length frame would be + // indistinguishable from one that was lost. + if err := w.Flush(); err != nil || len(sent) != len(want) { + t.Errorf("flushing an empty buffer sent %d chunks (%v)", len(sent)-len(want), err) + } +} diff --git a/docs/design/2026-09-23-workspace-checkpoint-wiring.md b/docs/design/2026-09-23-workspace-checkpoint-wiring.md new file mode 100644 index 0000000..85dfb5f --- /dev/null +++ b/docs/design/2026-09-23-workspace-checkpoint-wiring.md @@ -0,0 +1,369 @@ +# Wiring the portable workspace checkpoint into cold suspend and resume + +Status: proposed, with the wiring in this change. Implements rainier-cloud +`adr-0003-serverless-microvm-architecture.md` §2.3 (the dormancy tiers), §4.1 +(the Suspend/Resume/RemoveWorkspace rows) and §4.4 (the durability barrier), and +`hosted-product-prd.md` §10 (a checkpoint created and restore-tested after every +clean cold suspension). + +Companions: `docs/design/2026-09-20-portable-workspace-checkpoint.md` (the format +and the library, [#99](https://github.com/tokencanopy/rainier/pull/99)) and +`docs/design/2026-09-20-microvm-bootstrap-token-and-vsock.md`, whose §4 is the +cold-suspend handshake this note extends. + +> `Suspend(warm=false)` must not report success, and the host must not release +> the workspace slot, until the portable checkpoint upload … completes, its +> manifest is committed atomically, and the restore test passes. (ADR-0003 §4.4) + +The ellipsis is "to regional GCS", which this change does not do: a self-hosted +runner's store is a local directory and a cell's is GCS, behind one +`checkpoint.BlobStore` (§8). Nothing else is elided. + +## 1. The problem + +The checkpoint library takes an `fs.FS`. A session's workspace is an **ext4 +image** the guest mounts as a block device, and the host holds nothing but that +file. So the wiring question is one question: **how does a file tree get from +inside the guest's filesystem to the checkpoint writer running on the host?** + +What decides it is that the host must never mount or parse a tenant's ext4 +image. A loop mount is the host kernel's ext4 driver reading attacker-controlled +metadata in ring 0 — the largest attack surface a multi-tenant host could +volunteer for, on a design (ADR-0003 §4.5) whose premise is that the tenant's +kernel is not the host's. + +## 2. Six options + +1. **Host loop-mount** (`mount -o loop`). Simplest to write, and it puts the host + kernel's ext4 driver in front of tenant-controlled bytes — and needs root or + `CAP_SYS_ADMIN` in the runner, which ADR-0003 §4.5 spends a jailer to avoid. + **Refused on the kernel attack surface alone.** +2. **Host userspace ext4 reader** (a Go ext4 parser, or `debugfs -R rdump`). The + same hostile bytes parsed out of ring 0 by something nobody here maintains — + still the host parsing tenant filesystem metadata, only slower. **Refused.** +3. **Guest-streamed tar.** The guest already has the filesystem mounted, is the + only party that must parse it, and is inside the isolation boundary that + exists for exactly this; the host receives a byte stream and applies its own + limits. The cost is a new stream and an ordering constraint (§4). **Chosen.** +4. **Opaque image blob** — checkpoint the ext4 file itself. Trivially correct and + completely non-portable: PRD §10 wants a checkpoint restorable on another + qualified provider, and this is a 10 GiB mostly-zero artifact only an ext4 + host can open. It also defeats the tree digest and the exclusions. **Refused.** +5. **Network filesystem** (9p/virtiofs for the workspace instead of a block + device). The host would hold the tree directly — and would then be serving a + filesystem to a hostile guest, with every session's I/O path changed for a + feature far smaller than the change. **Refused; worth revisiting only if the + workspace device changes for other reasons.** +6. **Block-level diff** (dirty-page tracking, incremental images). The cheapest + steady state, and it re-introduces (4)'s portability problem plus a chain of + diffs whose base may never be deleted. **Refused for v1**; §4.2 of the format + note keeps a differential format open *above* the tree, not below. + +## 3. The shape + +**Cold suspend.** During the `suspending{cold:true}` handshake, after sessiond +has flushed, killed its execs and unmounted the agent home, it opens a stream on +the existing relay conn and writes the workspace tree. runnerd feeds that stream +to a tar-backed `fs.FS` (`internal/wstream`), the checkpoint writer walks it, +encrypts frame by frame and commits to a blob store. Nothing plaintext reaches +host disk; the host holds one frame, one copy buffer and one entry index (§5). +When the manifest commits, the host runs `Verify`, tells the guest, and only +then terminates the VM, releases the slot and discards the rootfs. A cold +`Resume` that finds the image absent and a checkpoint recorded rebuilds the +image from it: §7. + +## 4. The handshake + +``` +runnerd/driver sessiond (guest) + | FrameControl suspending{cold,id=N} ------------>| + |<--------------- FrameControl suspend_ack{id=N} | (2 s budget) + | | flush, kill execs, + | | unmount agent home, + | | forget secrets + |<--------------- FrameStream[id=N] chunk | (30 s to the 1st chunk) + |<--------------- FrameStream[id=N] chunk | (60 s idle budget, + | ... checkpoint writer consumes as it goes ..| 30 min total) + |<--- FrameControl workspace_end{id=N,ok,entries,bytes} + |<--------------- FrameControl suspend_ready{id=N} | (30 s budget) + | ... Writer.Write commits the manifest; the | + | generation is recorded and persisted HERE | + | ... Reader.Verify reads the committed bytes ... | (10 min budget) + | FrameControl checkpoint_committed{id=N} -------->| (best effort) + | engine.Stop, release slot, discard rootfs | +``` + +`FrameStream` is a new frame type (5, spelled out because it is wire-visible) +carrying the suspend **nonce** in `AttachID` — the token that already keeps a +late answer from satisfying the next suspend, so a stream from an abandoned one +is routed to nothing rather than into the live one. Stream and control frames +share the sessiond-side `connWriter` and the hub's single read loop, so the end +marker can neither overtake the last chunk nor be reordered against it. + +**Budgets.** #98's cold suspend was 2 s + 30 s, which bought a flush, a +ten-second exec kill and an unmount; it cannot also buy a 10 GiB copy. The new +numbers, all fields a test can drive in milliseconds: + +| leg | budget | why | +| --- | --- | --- | +| notice → ack | 2 s | unchanged (#98): a round trip on an open socket | +| ack → first chunk | 30 s | exactly #98's old ready budget: flush, kill, unmount | +| chunk → chunk, and last chunk → end marker | 60 s | progress, not throughput | +| end marker → `suspend_ready` | 30 s | the guest has nothing left to do | +| whole stream | 30 min | 10 GiB (the workspace disk) at ~6 MiB/s | +| `Verify` | 10 min | one sequential read of the committed object | + +Worst case ≈ 41 minutes, and only for a session that is genuinely stuck. The +idle budget covers the HOST's slowness too: the stream is back-pressured by the +store through the pipe, so a store that stopped taking bytes stalls it and fails +the suspend — which keeps the VM. + +**A cold suspend is now one handshake, not two.** `Op` sent the cold notice +itself before calling the driver; a checkpointing driver owns the notice, +because it owns the stream that follows. runnerd asks +(`CheckpointsColdSuspend()`) and sends nothing when the driver does. + +## 5. The stream, and why it is not only a tar + +`fs.WalkDir` is not streamable: it calls `ReadDir(".")` and needs **every** root +child's name before it visits the first one, while a root child's subtree sits +between it and its next sibling. No emission order fixes that, and buffering +entry *content* to answer it is what this design refuses. So, two sections: + +``` +"rainier.wsstream.v1\n" +index: repeated uvarint(len) record, terminated by uvarint(0) + record := kind byte ('d' | 'f' | 'l') || name +tar: archive/tar, the same entries in the same order, bodies for regular files +``` + +The index is the tree's **shape only** — name and kind; every other attribute +(mode, size, mtime, link target) comes from the tar header when the walk reaches +that entry, so each fact has one source. The host buffers the index (capped by +`MaxEntries` and `MaxIndexBytes`, fail closed) and **never** content: the tar +reader is a one-way cursor the walk drags forward, and `Next` discards a skipped +body for free. + +The order is `fs.WalkDir` order — depth-first, lexical per directory — because +that is the order the checkpoint writer walks in and its reader's directory +stack depends on. `ReadDir` sorts, the cursor only moves forward and is checked +to be ON the entry it answers about, so an out-of-order stream fails the suspend +rather than producing a wrong checkpoint. Entries the host excludes are dropped +from the index and skipped by the same cursor; whatever follows the last entry +the walk wanted is drained under the same total-byte limit, before the guest's +end marker can arrive. + +**Limits the host enforces, all configurable** (`wstream.Limits`), each on bytes +the guest chose: + +| limit | default | refusal | +| --- | --- | --- | +| entries | 500,000 | index too long | +| index bytes | 64 MiB | index too large | +| entry name | 4096 bytes, 255 per element | not a valid tree entry | +| a name absolute, uncleaned, with `..`/`.`/NUL | — | not a valid tree entry | +| a symlink target absolute or leaving the tree | — | refused | +| one entry's bytes / the whole stream | 8 GiB / 32 GiB | too large | +| entry kind | dir, regular file, symlink only | refused | +| order | strict `fs.WalkDir` order, no duplicates | refused | + +They are a second fence, not the only one: `checkpoint` re-applies the name and +symlink rules on the way in and again on the way out. The host applies them +because it must not be the hop that trusts the guest. "Index bytes" is charged +what the structure *costs* — names plus ~96 bytes of record, node, map entry and +child pointer — so it bounds host memory rather than something that correlates +with it, and both ends charge it the same way. + +## 6. What happens when something fails + +- **The guest dies mid-stream** (or the conn dies): the reader sees the pipe + close before the end marker, the write fails, nothing is committed, and the VM + is left running — `settleFailedColdSuspend` rolls the entry back to "running" + when the VM is still up. The failure is named as the STREAM rather than as a + parse error, because the sandbox is what went away. +- **The guest reports its own failure**: `workspace_end{ok:false, stage}`, the + `stage_failed` shape, with counts and never a path. +- **The store is unreachable or refuses**: the manifest was never put, so that + generation stays free for the retry and the VM stays. +- **`Verify` fails**: the manifest IS committed, so the generation is spent — a + manifest key has no attempt suffix and put-if-absent never overwrites — and is + therefore **recorded and persisted the moment `Write` returns**, before the + verify. The suspend still fails, the VM still stays, and the retry takes the + NEXT generation. A record that advanced only on success would have every retry + lose its manifest put to `ErrExists`: a session that can never be cold-parked + again. The driver also probes for a free generation before it streams, so a + record *behind* the store heals itself. The unverified objects are left for + the sweeper; deleting them would race a reconciler that just committed them. +- **The sandbox is gone** (a crashed sessiond on a live VM): there is no + handshake to run, so the suspend fails and the VM stays — and keeps staying, + since nothing else will produce a checkpoint for it. The escape is `Destroy`. + A known cost: §10. + +**The order here is an inversion of §4.4's, not a stricter reading of it.** As +written, ADR-0003 ordered a cold suspend as flush, terminate, detach, *then* +produce the checkpoint from the detached disk — and said a host that died +mid-checkpoint left the disk intact for reconciliation to re-run the checkpoint +from it. This design checkpoints **from the running guest, before anything is +torn down**, and it had to: §4.5's rule that the host never mounts or parses a +tenant's ext4 image leaves the guest as the only party that can read the +workspace tree, so there is no "checkpoint it later from the disk" path to fall +back on. The barrier guarantee is unchanged — success is reported only after the +manifest commit and a passed restore test — but the failure consequence is +different and is stated plainly: **a host that dies mid-checkpoint has not +checkpointed that workspace**, and nothing can checkpoint it afterwards. The +disk stays where it is, the session resumes from it on that host, or it waits +for its next clean suspend; reconciliation cannot produce a checkpoint from a +detached disk. rainier-cloud is amending ADR-0003 §4.1, §4.3 and §4.4 to state +this order (`docs/architecture/adr-0003-serverless-microvm-architecture.md`, +revision 2.1). +- **A tampered checkpoint, or a restore's `mkfs` or chown failing**: the scratch + directory and the partial image are removed, nothing is attached, and the + session stays parked with its checkpoint intact. + +Every failure keeps the VM and names the **stage** — `stream`, `write`, +`verify`, `restore`, `owner`, `mkfs` — and never a path, a name or a value, +because these reach a session's error column (tenancy §15.1). That extends to +`mkfs.ext4 -d`'s own output, which names the files it was copying, and is +therefore counted rather than quoted. + +**A kept VM is a degraded VM**, inherently: by the time the stream runs, the +guest has done what #98's cold notice asks — execs dead, agent home unmounted, +delivered secrets forgotten — so a failed suspend leaves the VM running in that +state. The right outcome, and still not the session the user left. + +## 7. The restore path + +`Resume` on a cold record whose workspace image is gone: + +1. Refuse unless the record names a committed generation: "no image and no + checkpoint" is a session whose work is gone, and saying so beats booting an + empty workspace that looks resumed. +2. `mkdir` a scratch directory `0700` at `/restore/`, replacing + whatever a previous attempt left, removed by `defer` on every path. +3. `Reader.Restore` into it: authorization hook, manifest authentication, tree + digest, all of the library's checks. +4. **Give the tree to the user the guest runs as.** A checkpoint records modes + and not owners — a uid is not portable and the format is meant to be — so a + restored tree belongs to whoever restored it, which is root, while the agent + runs as the image's own user and the guest's `/init` chowns the mount point + only when it is EMPTY (so a resumed workspace's contents are left alone). + Without this the session comes back unable to write its own files. The uid + and gid are runner configuration (`--checkpoint-restore-uid/-gid`, default + 1000): only whoever built the image knows which user it runs as. +5. Build the image at `.partial` — `mkfs.ext4 -d ` through + `DiskFormatter`, which grows a `FormatFromDir` method for it — and rename it + into place only once it is a whole filesystem. The rename is load-bearing: + the trigger above is the image's *absence*, so a file created at the final + path and populated over minutes is one a crash leaves for the next resume to + find, skip the restore for, and hand the guest unformatted. No `-N`: mke2fs + sizes the inode table from the image (~655,000 inodes for 10 GiB, above the + entry ceiling), and sizing it for the restored tree would starve what the + session writes next. Then remove the scratch tree, attach, and boot. + +**The precondition, which is not small: the restore is reachable only from a +record this runnerd process still holds the guest configuration for.** A cold +resume of a record *recovered from disk* is refused before it gets here — the +boot configuration is held in memory only (ADR-0003 §2.7 item 1), so a relaunch +would boot a guest that is never told what it is — and a session dormant long +enough for the deep-dormant tier to have deleted its workspace image has almost +certainly outlived the process that created it. So in production this path is +behind that refusal, and the ORDER of the two is not what puts it there: running +the restore first would spend minutes of copying and a plaintext scratch tree on +a resume that is certain to fail a moment later, and widen the very window this +section bounds. What the driver does instead is tell the two apart and say so — +"the configuration did not survive a runnerd restart; its workspace image is +gone as well, so the resume that does re-resolve the configuration must also +restore committed checkpoint generation *N*" — which costs one `Stat` and no +tenant bytes, and which is the difference between one operator action and +another. Making the restore reachable after a restart is the **create-shaped +resume**, in which the control plane re-resolves the session's configuration +(§9); persisting that configuration on this host instead is the one answer +ADR-0003 §2.7 rules out. + +The scratch directory is no new exposure: the workspace image on the host is +plaintext today, in the same state directory, with provider encryption as +defense in depth. It is `0700`, never inside a jail, removed before the guest +boots on every path — and whatever a *crashed* host left under `/restore` +is removed when the driver starts, because a tree that outlives the process that +made it is a different claim than this one. `RemoveWorkspace` removes the +**image only**: the driver has no retention policy, and one that deleted a +checkpoint on teardown would make §4.4's deep-dormant tier unreachable. + +## 8. The self-hosted store and key + +Two required flags when `--driver=microvm`, both fail-closed: + +- `--checkpoint-store-dir`: a local directory blob store. `PutIfAbsent` writes a + temp file beside the key, fsyncs it, and `link(2)`s it into place — `EEXIST` + is `ErrExists` where a rename would silently replace a committed manifest — + and a failed write leaves nothing at the key and nothing beside it. +- `--checkpoint-key-file`: 32 bytes of key material (64 hex characters or 32 raw + bytes), mode `0600` or tighter, owned by the runner, becoming a + `checkpoint.StaticKeyWrapper` under `selfhosted/checkpoint/v1`. A missing + file, a wrong size, an all-zero key, a mode any other user can read, or an + owner that is not this runner are all refusals to START. + +Both are built in `cmd/runnerd` and handed to the driver as ports, so a cell +substitutes GCS and KMS without touching the driver. + +## 9. What rainier-cloud owes + +- A **GCS blob store** (`x-goog-if-generation-match: 0` for `PutIfAbsent`, + resumable upload for the content object) behind `checkpoint.BlobStore`. +- A **KMS wrapper** passing the authenticated context as AAD and returning the + concrete key version, plus the regional key-readiness rule `Preflight` is for. +- The **workspace identity**: this driver has no workspace id in its `Spec` and + uses the workspace volume name, so a cell carrying the control plane's + workspace id should pass that instead. +- **Checkpoint-age policy and the freshness view** (PRD §10), the **orphan + sweep** for objects whose manifest lost a race or failed its verify, and + deletion (tenancy §14.2) — above the driver, out of this change. +- A **dispatch timeout** for a cold stop covering §4's budgets: a cell that + cancels at 30 s gets a failed suspend and a live VM — safe, and useless. +- A **create-shaped resume**: a resume of a session whose runner has restarted, + in which the control plane re-resolves the guest configuration this host + deliberately does not persist. Without it the restore path in §7 is reachable + only inside the lifetime of the runnerd that created the session, which is not + the lifetime a deep-dormant session has. +- A **host-wide bound on concurrent checkpoints**. The index is `O(entries)` per + in-flight cold suspend, capped at 64 MiB (§5), and nothing bounds how many + suspends are in flight: a host drain (§4.5 rotation) cold-suspends every + session at once, so the ceiling is 64 MiB × slots — ~2 GB for 32. The driver + refuses a second checkpoint *per instance* and has no view of the host, so the + admission control belongs above it. (§10 has the alternative: an index carried + incrementally per directory, which removes the cost instead of bounding it.) +- **An owner for §4.4's disk-deletion guard.** "Deep-dormant transition is + permitted only after a verified checkpoint exists that is newer than the + disk's last write" is, today, owned by nobody: the driver's `RemoveWorkspace` + takes a session id, holds no retention policy, and cannot see a workspace's + last write; this change does not gate it, and nothing above it does either. + The `cell-worker` retention path that decides the transition and calls + `RemoveWorkspace` is where it has to live. +- **Something on the record that distinguishes committed from verified.** + `CheckpointAt` is written the moment `Write` returns — a COMMIT time, before + the restore test — because a generation is spent either way (§6). So a record + that says only "generation 4 at 12:01" cannot answer the freshness question + the guard above gates disk deletion on, and the fix is a record field and a + view, not a change to this barrier. +- **A deletion path for the blob store.** `DirBlobStore` has `Get` and + `PutIfAbsent` and nothing else, on purpose — the driver never deletes a + checkpoint — so the retention window, the orphan sweep and tenancy-directed + deletion (§14.2) have no implementation at either end yet. + +## 10. Open questions + +- **The index's memory.** The host's `O(entries)` index is the one place this + design spends what the library refuses to: up to 64 MiB per in-flight cold + suspend (§5). Bounding how many are in flight is §9's, and is admission + control rather than a format change. The other answer is a format change: an + index carried *incrementally per directory* would remove the cost instead of + bounding it, at the price of a stream that no longer round-trips through plain + `archive/tar`. +- **A session whose sandbox is gone cannot be cold-parked**, and pins a VM and a + slot until somebody destroys it (§6). The narrower rule — park without a + checkpoint, gate only the deep-dormant *disk deletion* on one — is what §4.4 + asks for, and is a change to what the runner records, not to this barrier. +- **Compression on the wire** (measurable, unmeasured), and **incremental + checkpoints**: every cold suspend writes the whole tree, and the generation + counter plus the attempt-suffixed content key leave room for a differential + format nothing here forecloses. diff --git a/docs/microvm-host-privileges.md b/docs/microvm-host-privileges.md index dc30025..e3a653c 100644 --- a/docs/microvm-host-privileges.md +++ b/docs/microvm-host-privileges.md @@ -25,7 +25,9 @@ everything not on this list. | A writable cgroup v2 parent | The jailer creates one cgroup per VM under it; ADR-0003 §4.6 meters each session from `cpu.stat` and `memory.current` there. | | `ip` (iproute2), `nft` (nftables) and `sysctl` on `PATH` | The network slot, its firewall, and turning forwarding on inside its namespace — a fresh namespace starts with `net.ipv4.ip_forward=0`, and a guest's packet leaves on a different interface from the one it arrived on. | | `firecracker` and `jailer` on `PATH` | There is no unjailed launch path. | -| `mkfs.ext4` on `PATH` | Session workspace and agent-home images are formatted before a guest is handed them. | +| `mkfs.ext4` on `PATH` | Session workspace and agent-home images are formatted before a guest is handed them, and a deep-dormant resume builds one from a restored tree with `mkfs.ext4 -d`. | +| `--checkpoint-store-dir`, a writable directory | Every cold suspend turns the session's workspace into a portable checkpoint before the VM is terminated (ADR-0003 §4.4); this is where one checkpoint's two objects go. There is deliberately no default — a temp-directory one would put a tenant's only durable copy somewhere the host reaps. | +| `--checkpoint-key-file`, mode `0600` or tighter and owned by the runner | The 32 bytes (64 hex characters, or 32 raw) every checkpoint is wrapped under. Without it the objects in the store directory would be a tenant's files in the clear on this host; with it, losing this one file loses every checkpoint it wrapped. A file that is missing, the wrong size, all zeros, readable by group or other, or owned by somebody else is a refusal to start. | One thing is **not** on that list and should be, on a production host: a state directory on a filesystem that can **reflink** — XFS formatted with @@ -42,6 +44,13 @@ The runner's startup error names the first item it is missing and what that item is for. `driver.MicrovmHostRequirements()` prints the whole list; it is generated from the same table the check reads, so the two cannot drift. +The last two rows are the exception to "generated from the same table": they are +flags rather than host state, so they are checked in `cmd/runnerd` before the +driver is built (and the key file's own checks in `driver.LoadCheckpointKey`). +They are on this list because they belong to the same promise — a runner that +starts without them is one whose cold suspends cannot satisfy §4.4, which is a +thing to discover at startup and not at the first suspend. + ## A systemd unit ```ini diff --git a/internal/driver/checkpointkey_other.go b/internal/driver/checkpointkey_other.go new file mode 100644 index 0000000..2e02c1e --- /dev/null +++ b/internal/driver/checkpointkey_other.go @@ -0,0 +1,11 @@ +//go:build !unix + +package driver + +import "io/fs" + +// checkKeyOwner has nothing to check off Unix: file ownership there is not a +// uid, and a microVM host is Linux by construction (ADR-0003 §4.5). The mode +// check in LoadCheckpointKey still applies, and is what the doc comment +// promises on this platform. +func checkKeyOwner(string, fs.FileInfo) error { return nil } diff --git a/internal/driver/checkpointkey_unix.go b/internal/driver/checkpointkey_unix.go new file mode 100644 index 0000000..b7931cc --- /dev/null +++ b/internal/driver/checkpointkey_unix.go @@ -0,0 +1,34 @@ +//go:build unix + +package driver + +import ( + "fmt" + "io/fs" + "os" + "syscall" +) + +// checkKeyOwner requires the checkpoint key file to be owned by the user this +// runner runs as. +// +// The mode check beside it (no group or other bits) is about who ELSE can read +// the key; this one is about who WROTE it. A file at mode 0600 owned by another +// user is a file whose owner knows the key and whose owner can replace it — so +// a runner pointed at one, by a path typo or by a shared directory somebody else +// can create in, would encrypt every workspace checkpoint on this host under a +// key that is not its own. That is not a theoretical arrangement: a runner +// commonly runs as root, and root can read a 0600 file belonging to anybody. +func checkKeyOwner(path string, info fs.FileInfo) error { + st, ok := info.Sys().(*syscall.Stat_t) + if !ok { + // A file system that does not report ownership. There is nothing to + // check and nothing to claim, so the mode check stands alone. + return nil + } + if uid := os.Getuid(); int(st.Uid) != uid { + return fmt.Errorf("microvm: the checkpoint key file %s is owned by uid %d, not by the user this runner runs as (%d)", + path, st.Uid, uid) + } + return nil +} diff --git a/internal/driver/checkpointkey_unix_test.go b/internal/driver/checkpointkey_unix_test.go new file mode 100644 index 0000000..342b4d1 --- /dev/null +++ b/internal/driver/checkpointkey_unix_test.go @@ -0,0 +1,58 @@ +//go:build unix + +package driver + +import ( + "io/fs" + "os" + "strings" + "syscall" + "testing" + "time" +) + +// checkKeyOwner is the one refusal in LoadCheckpointKey that cannot be +// exercised against a real file: a test has no privilege to create a file owned +// by somebody else, which is exactly why the guard was untested. The ownership +// is the only thing it reads, so the fs.FileInfo is the seam to fake — the +// alternative is a guard nothing exercises until a real host hits it. +type fakeKeyInfo struct{ sys any } + +func (fakeKeyInfo) Name() string { return "checkpoint.key" } +func (fakeKeyInfo) Size() int64 { return 32 } +func (fakeKeyInfo) Mode() fs.FileMode { return 0o600 } +func (fakeKeyInfo) ModTime() time.Time { return time.Time{} } +func (fakeKeyInfo) IsDir() bool { return false } +func (f fakeKeyInfo) Sys() any { return f.sys } + +// TestAKeyFileOwnedBySomebodyElseIsRefused. A 0600 file owned by another user +// is a key that user knows and can replace, and a runner commonly runs as root, +// which can read one. The mode check beside this one cannot see it. +func TestAKeyFileOwnedBySomebodyElseIsRefused(t *testing.T) { + foreign := uint32(os.Getuid()) + 1 + err := checkKeyOwner("/var/lib/rainier/checkpoint.key", fakeKeyInfo{sys: &syscall.Stat_t{Uid: foreign}}) + if err == nil { + t.Fatal("a checkpoint key file owned by another uid was accepted") + } + if !strings.Contains(err.Error(), "owned by uid") { + t.Errorf("the refusal does not say what is wrong: %v", err) + } +} + +// TestAKeyFileOwnedByThisRunnerIsAccepted, so the refusal above is about +// ownership and not about every file. +func TestAKeyFileOwnedByThisRunnerIsAccepted(t *testing.T) { + info := fakeKeyInfo{sys: &syscall.Stat_t{Uid: uint32(os.Getuid())}} + if err := checkKeyOwner("/var/lib/rainier/checkpoint.key", info); err != nil { + t.Errorf("a key file owned by this runner was refused: %v", err) + } +} + +// TestAFileSystemThatReportsNoOwnerIsNotRefused: there is nothing to check and +// nothing to claim, so the mode check stands alone rather than this one +// refusing every file on such a filesystem. +func TestAFileSystemThatReportsNoOwnerIsNotRefused(t *testing.T) { + if err := checkKeyOwner("/var/lib/rainier/checkpoint.key", fakeKeyInfo{sys: nil}); err != nil { + t.Errorf("a filesystem that does not report ownership was refused: %v", err) + } +} diff --git a/internal/driver/checkpointstore.go b/internal/driver/checkpointstore.go new file mode 100644 index 0000000..a6e766d --- /dev/null +++ b/internal/driver/checkpointstore.go @@ -0,0 +1,314 @@ +// internal/driver/checkpointstore.go +// +// The self-hosted half of the portable workspace checkpoint: where the objects +// go on a runner that has no cloud behind it, and where its key comes from. +// +// Both are deliberately small and deliberately here rather than in the +// checkpoint package. That package ships one storage backend (an in-memory map) +// and one key wrapper (a static key) on purpose — "restorable on another +// qualified provider" (PRD §10) is a property it keeps by never having met a +// provider — so a directory on a host is this repository's own concern, behind +// the same two ports a GCS bucket and a KMS go behind. +package driver + +import ( + "context" + "encoding/hex" + "errors" + "fmt" + "io" + "io/fs" + "log" + "os" + "path/filepath" + "strings" + + "github.com/tokencanopy/rainier/checkpoint" +) + +// SelfHostedCheckpointKeyRef is the key reference a self-hosted runner's +// checkpoints are wrapped under. It is a constant because there is exactly one +// key — rotation for a static wrapper is a new reference and a new file, and old +// checkpoints under the old one stop opening, which is the crypto-shred the +// deletion story rests on and the wrong behaviour for a fleet that wanted to +// keep reading them. A cell uses a real key service. +const SelfHostedCheckpointKeyRef checkpoint.KeyRef = "selfhosted/checkpoint/v1" + +// checkpointDirMode is the mode of every directory this store creates: the +// runner's own. A checkpoint is ciphertext, but the object LAYOUT names a +// workspace and a session (see the checkpoint package's storage layout), and a +// directory listing that tells a local user which sessions this host holds is a +// disclosure with nothing to buy it. +const checkpointDirMode fs.FileMode = 0o700 + +// DirBlobStore is a checkpoint.BlobStore in a directory on this host. +// +// It exists so a self-hosted runner can cold suspend at all. It is not a +// pretend cloud: there is no replication, no lifecycle policy and no +// cross-region anything, and a host that loses this directory has lost the +// checkpoints in it. What it does implement exactly is the one semantic the +// format's atomicity rests on — a conditional create that either produces the +// WHOLE object or leaves nothing at the key. +type DirBlobStore struct{ root string } + +var _ checkpoint.BlobStore = (*DirBlobStore)(nil) + +// NewDirBlobStore prepares dir as a checkpoint store, creating it 0700 if it is +// not there. +// +// It fails rather than falling back, for the reason every other microVM +// preflight does: a runner that started with nowhere to put a checkpoint would +// accept placements and then fail every cold suspend, with a tenant's work +// still inside a VM it cannot terminate. +func NewDirBlobStore(dir string) (*DirBlobStore, error) { + if dir == "" { + return nil, errors.New("microvm: a checkpoint store directory is required (--checkpoint-store-dir)") + } + abs, err := filepath.Abs(dir) + if err != nil { + return nil, fmt.Errorf("microvm: resolving the checkpoint store directory: %w", err) + } + if err := os.MkdirAll(abs, checkpointDirMode); err != nil { + return nil, fmt.Errorf("microvm: creating the checkpoint store directory: %w", err) + } + info, err := os.Stat(abs) + if err != nil { + return nil, fmt.Errorf("microvm: reading the checkpoint store directory: %w", err) + } + if !info.IsDir() { + return nil, fmt.Errorf("microvm: the checkpoint store path %s is not a directory", abs) + } + // A directory that was already there keeps whatever mode it was made with — + // MkdirAll is a no-op on one — and the default umask makes that 0755. It is + // tightened rather than refused: this directory is the runner's own by + // definition, the layout inside it names a workspace and a session for + // every checkpoint on this host (see checkpointDirMode), and an operator who + // mkdir'd it before starting the runner has made an ordinary mistake rather + // than a decision. + if info.Mode().Perm()&0o077 != 0 { + log.Printf("microvm: the checkpoint store directory %s was mode %04o; tightening it to 0700, because its layout names every session this host has checkpointed", + abs, info.Mode().Perm()) + if err := os.Chmod(abs, checkpointDirMode); err != nil { + return nil, fmt.Errorf("microvm: tightening the checkpoint store directory's mode: %w", err) + } + } + return &DirBlobStore{root: abs}, nil +} + +// PutIfAbsent writes the object at key, and either the whole of it appears or +// nothing does. +// +// The mechanism is a temporary file in the SAME directory, fsynced, then +// link(2)ed into place. link is what makes this conditional: it fails with +// EEXIST against an existing name, where rename would silently replace one — +// and replacing a committed manifest is the one operation this format has no +// answer for, because the data key inside it is the only way to read the +// content object it names. +// +// The temporary file is removed on every path, so a failed write leaves the +// directory as it found it. +func (s *DirBlobStore) PutIfAbsent(ctx context.Context, key string, write func(io.Writer) error) error { + if err := ctx.Err(); err != nil { + return err + } + path, err := s.path(key) + if err != nil { + return err + } + if _, err := os.Lstat(path); err == nil { + return fmt.Errorf("%w: %s", checkpoint.ErrExists, "the object is already present") + } else if !errors.Is(err, os.ErrNotExist) { + return fmt.Errorf("checkpoint store: reading the object: %w", err) + } + dir := filepath.Dir(path) + if err := os.MkdirAll(dir, checkpointDirMode); err != nil { + return fmt.Errorf("checkpoint store: creating the object's directory: %w", err) + } + + tmp, err := os.CreateTemp(dir, ".partial-*") + if err != nil { + return fmt.Errorf("checkpoint store: creating a temporary object: %w", err) + } + tmpName := tmp.Name() + committed := false + defer func() { + if !committed { + _ = os.Remove(tmpName) + } + }() + + if err := write(tmp); err != nil { + tmp.Close() + // The writer's own error, unwrapped: it is the checkpoint package's, + // which is written to carry no path and no content, and a caller + // matching on its sentinels must still be able to. + return err + } + // Both fsyncs are the durability half of the barrier. ADR-0003 §4.4 says a + // cold suspend may not report success until the manifest is committed, and + // an object still in this host's page cache is not committed to anything: a + // power loss between here and the VM's termination would lose the only copy + // of a workspace whose VM was told it was safe to end. + if err := tmp.Sync(); err != nil { + tmp.Close() + return fmt.Errorf("checkpoint store: syncing the object: %w", err) + } + if err := tmp.Close(); err != nil { + return fmt.Errorf("checkpoint store: closing the object: %w", err) + } + if err := os.Link(tmpName, path); err != nil { + if errors.Is(err, os.ErrExist) { + return fmt.Errorf("%w: %s", checkpoint.ErrExists, "the object is already present") + } + return fmt.Errorf("checkpoint store: committing the object: %w", err) + } + committed = true + _ = os.Remove(tmpName) + return syncDir(dir) +} + +// Open returns a reader over the object at key. +func (s *DirBlobStore) Open(ctx context.Context, key string) (io.ReadCloser, error) { + if err := ctx.Err(); err != nil { + return nil, err + } + path, err := s.path(key) + if err != nil { + return nil, err + } + f, err := os.Open(path) + if errors.Is(err, os.ErrNotExist) { + return nil, checkpoint.ErrNotFound + } + if err != nil { + return nil, fmt.Errorf("checkpoint store: opening the object: %w", err) + } + return f, nil +} + +// Delete removes the object at key. A missing object is success, because +// deletion is retried and a ledger that cannot report "already gone" as done +// never converges. +func (s *DirBlobStore) Delete(ctx context.Context, key string) error { + if err := ctx.Err(); err != nil { + return err + } + path, err := s.path(key) + if err != nil { + return err + } + if err := os.Remove(path); err != nil && !errors.Is(err, os.ErrNotExist) { + return fmt.Errorf("checkpoint store: deleting the object: %w", err) + } + return nil +} + +// path turns a storage key into a file path under this store's root. +// +// Every element is checked, even though the keys this store sees are built by +// the checkpoint package from identifiers it has already validated. That is the +// last-hop rule the rest of this repository applies at every syscall boundary: +// the check performed by the layer above is a check you are trusting, and what +// is on the other side of this one is os.Remove. +func (s *DirBlobStore) path(key string) (string, error) { + if key == "" || len(key) > maxCheckpointKeyLen { + return "", fmt.Errorf("checkpoint store: the object key is empty or over %d characters", maxCheckpointKeyLen) + } + elems := strings.Split(key, "/") + for _, e := range elems { + switch { + case e == "", e == ".", e == "..": + return "", errors.New("checkpoint store: the object key has an empty, \".\" or \"..\" element") + case strings.ContainsAny(e, "\x00\\"): + return "", errors.New("checkpoint store: the object key has an element with a NUL or a backslash in it") + } + } + return filepath.Join(append([]string{s.root}, elems...)...), nil +} + +// maxCheckpointKeyLen bounds a key, so that a path this store builds is a +// bounded string. It is over the format's own maximum (a 512-byte prefix plus +// the layout) and under any filesystem's. +const maxCheckpointKeyLen = 1024 + +// syncDir fsyncs a directory, which is what makes a newly linked name survive a +// power loss. It is best effort on a filesystem that refuses the call (some do +// for a directory opened read-only) and an error otherwise. +func syncDir(dir string) error { + d, err := os.Open(dir) + if err != nil { + return fmt.Errorf("checkpoint store: opening the object's directory to sync it: %w", err) + } + defer d.Close() + if err := d.Sync(); err != nil { + if errors.Is(err, os.ErrInvalid) || errors.Is(err, fs.ErrPermission) { + return nil + } + return fmt.Errorf("checkpoint store: syncing the object's directory: %w", err) + } + return nil +} + +// LoadCheckpointKey reads the 32-byte key a self-hosted runner wraps its +// checkpoints under, and refuses everything about the file that is not right. +// +// The refusals are the point. This one file is the difference between a +// checkpoint that is ciphertext and a checkpoint anybody who can read the store +// directory can open, and every condition below is one a real deployment gets +// wrong at some point: +// +// - a file that is not there, or is not a regular file; +// - a mode any other user on this host can read (a key is a key); +// - something that is not 32 bytes, in either spelling this accepts; +// - all zeros, which is a fine 32 bytes and is also how an unset key looks — +// refused for the reason ParseSecretsKey refuses one. +// +// Two spellings are accepted because both are what operators actually produce: +// 64 hex characters (`openssl rand -hex 32`, and what fits in a secret manager +// or a Kubernetes Secret) and 32 raw bytes (`openssl rand 32 > key`). Trailing +// whitespace is trimmed from the hex form, because a file written by `echo` has +// a newline in it and refusing that would be refusing the most likely correct +// key in the world. +func LoadCheckpointKey(path string) ([32]byte, error) { + var key [32]byte + if path == "" { + return key, errors.New("microvm: a checkpoint key file is required (--checkpoint-key-file)") + } + info, err := os.Lstat(path) + if err != nil { + return key, fmt.Errorf("microvm: reading the checkpoint key file: %w", err) + } + if !info.Mode().IsRegular() { + return key, fmt.Errorf("microvm: the checkpoint key file %s is not a regular file", path) + } + if info.Mode().Perm()&0o077 != 0 { + return key, fmt.Errorf("microvm: the checkpoint key file %s is mode %04o; it must not be readable by group or other (0600)", + path, info.Mode().Perm()) + } + if err := checkKeyOwner(path, info); err != nil { + return key, err + } + raw, err := os.ReadFile(path) + if err != nil { + return key, fmt.Errorf("microvm: reading the checkpoint key file: %w", err) + } + switch trimmed := strings.TrimSpace(string(raw)); { + case len(trimmed) == hex.EncodedLen(len(key)): + decoded, err := hex.DecodeString(trimmed) + if err != nil { + // The error is not wrapped: hex quotes the byte it choked on, and + // that byte is key material. + return key, fmt.Errorf("microvm: the checkpoint key file %s is %d characters but is not hex", path, len(trimmed)) + } + copy(key[:], decoded) + case len(raw) == len(key): + copy(key[:], raw) + default: + return key, fmt.Errorf("microvm: the checkpoint key file %s holds %d bytes; it must be 32 raw bytes or 64 hex characters", + path, len(raw)) + } + if key == ([32]byte{}) { + return key, fmt.Errorf("microvm: the checkpoint key file %s is all zeros, which is indistinguishable from an unset key", path) + } + return key, nil +} diff --git a/internal/driver/checkpointstore_test.go b/internal/driver/checkpointstore_test.go new file mode 100644 index 0000000..3dca883 --- /dev/null +++ b/internal/driver/checkpointstore_test.go @@ -0,0 +1,405 @@ +// internal/driver/checkpointstore_test.go +package driver + +import ( + "bytes" + "context" + "encoding/hex" + "errors" + "io" + "os" + "path/filepath" + "strings" + "sync" + "testing" + + "github.com/tokencanopy/rainier/checkpoint" +) + +// The self-hosted store and key. The store's one load-bearing property is +// put-if-absent: the format's atomic commit IS that conditional write, so a +// store that overwrote, or that left a half-written object visible, would make +// the durability barrier a barrier against nothing. + +func dirStore(t *testing.T) (*DirBlobStore, string) { + t.Helper() + dir := filepath.Join(t.TempDir(), "checkpoints") + s, err := NewDirBlobStore(dir) + if err != nil { + t.Fatalf("NewDirBlobStore: %v", err) + } + return s, dir +} + +func put(t *testing.T, s *DirBlobStore, key, body string) error { + t.Helper() + return s.PutIfAbsent(context.Background(), key, func(w io.Writer) error { + _, err := io.WriteString(w, body) + return err + }) +} + +func TestDirStoreRoundTrip(t *testing.T) { + s, _ := dirStore(t) + ctx := context.Background() + const key = "rainier.checkpoint.v1/ws/ws-1/sess/sess-1/gen/00000000000000000001/manifest.json" + + if err := put(t, s, key, "the manifest"); err != nil { + t.Fatalf("put: %v", err) + } + rc, err := s.Open(ctx, key) + if err != nil { + t.Fatalf("open: %v", err) + } + defer rc.Close() + got, err := io.ReadAll(rc) + if err != nil { + t.Fatal(err) + } + if string(got) != "the manifest" { + t.Errorf("the object read back as %q", got) + } +} + +// TestDirStoreNeverOverwrites is the atomic commit. Replacing a committed +// manifest is the one operation this format has no answer for: the data key +// inside it is the only way to read the content object it names. +func TestDirStoreNeverOverwrites(t *testing.T) { + s, _ := dirStore(t) + const key = "p/manifest.json" + + if err := put(t, s, key, "first"); err != nil { + t.Fatal(err) + } + err := put(t, s, key, "second") + if !errors.Is(err, checkpoint.ErrExists) { + t.Fatalf("a second put gave %v, want ErrExists", err) + } + rc, err := s.Open(context.Background(), key) + if err != nil { + t.Fatal(err) + } + defer rc.Close() + got, _ := io.ReadAll(rc) + if string(got) != "first" { + t.Errorf("the object is now %q; the first writer's bytes were replaced", got) + } +} + +// TestDirStoreRefusesAKeyThatAppearedWhileItWasWriting is the conditional +// create at the only place it is actually conditional. +// +// The existence check before the write catches a key that is already there, but +// it is a check and not a commit: the interesting case is the object appearing +// DURING a write, which is a retry racing a reconciler over a multi-megabyte +// upload. What refuses it is link(2), which fails with EEXIST where rename +// would silently replace a committed manifest — and replacing a manifest is the +// one operation this format has no answer for, because the data key inside it +// is the only way to read the content object it names. +func TestDirStoreRefusesAKeyThatAppearedWhileItWasWriting(t *testing.T) { + s, _ := dirStore(t) + ctx := context.Background() + const key = "p/manifest.json" + + err := s.PutIfAbsent(ctx, key, func(w io.Writer) error { + // The winner commits after this writer's existence check and before its + // own commit. + if err := put(t, s, key, "the winner"); err != nil { + return err + } + _, err := io.WriteString(w, "the loser") + return err + }) + if !errors.Is(err, checkpoint.ErrExists) { + t.Fatalf("the losing put gave %v, want ErrExists", err) + } + rc, err := s.Open(ctx, key) + if err != nil { + t.Fatal(err) + } + defer rc.Close() + got, _ := io.ReadAll(rc) + if string(got) != "the winner" { + t.Fatalf("the object is %q; the committed bytes were replaced", got) + } +} + +// TestDirStoreConcurrentPutsHaveOneWinner: two writers at one generation, which +// is a retry racing a reconciler. Exactly one may believe it committed. +func TestDirStoreConcurrentPutsHaveOneWinner(t *testing.T) { + s, _ := dirStore(t) + const key = "p/manifest.json" + const writers = 8 + + var wg sync.WaitGroup + results := make([]error, writers) + for i := range writers { + wg.Add(1) + go func() { + defer wg.Done() + results[i] = put(t, s, key, strings.Repeat("x", i+1)) + }() + } + wg.Wait() + + won := 0 + for i, err := range results { + switch { + case err == nil: + won++ + case !errors.Is(err, checkpoint.ErrExists): + t.Errorf("writer %d lost with %v, want ErrExists", i, err) + } + } + if won != 1 { + t.Fatalf("%d writers committed the same object, want exactly 1", won) + } +} + +// TestDirStoreLeavesNothingWhenTheWriteFails. PutIfAbsent's contract is that an +// errored write leaves NOTHING at the key — and, here, nothing beside it +// either: a partial file left in the directory is one a later listing counts +// and a later operator wonders about. +func TestDirStoreLeavesNothingWhenTheWriteFails(t *testing.T) { + s, dir := dirStore(t) + ctx := context.Background() + const key = "p/content.abc" + + want := errors.New("the writer failed half way") + err := s.PutIfAbsent(ctx, key, func(w io.Writer) error { + _, _ = io.WriteString(w, "half an object") + return want + }) + if !errors.Is(err, want) { + t.Fatalf("the put gave %v, want the writer's own error", err) + } + if _, err := s.Open(ctx, key); !errors.Is(err, checkpoint.ErrNotFound) { + t.Errorf("a failed write left an object at the key (%v)", err) + } + var left []string + _ = filepath.WalkDir(dir, func(p string, d os.DirEntry, err error) error { + if err == nil && !d.IsDir() { + left = append(left, p) + } + return nil + }) + if len(left) != 0 { + t.Errorf("a failed write left %v behind", left) + } +} + +func TestDirStoreOpenAndDeleteOfAMissingObject(t *testing.T) { + s, _ := dirStore(t) + ctx := context.Background() + if _, err := s.Open(ctx, "p/nothing"); !errors.Is(err, checkpoint.ErrNotFound) { + t.Errorf("opening a missing object gave %v, want ErrNotFound", err) + } + // A missing object is a successful delete: deletion is retried, and a + // ledger that cannot report "already gone" as done never converges. + if err := s.Delete(ctx, "p/nothing"); err != nil { + t.Errorf("deleting a missing object gave %v", err) + } +} + +// TestDirStoreRefusesAKeyThatIsNotOne. The keys this store sees are built by +// the checkpoint package from identifiers it has already validated; this is the +// last hop before os.Remove, and the last hop trusts nobody. +func TestDirStoreRefusesAKeyThatIsNotOne(t *testing.T) { + s, dir := dirStore(t) + ctx := context.Background() + for _, key := range []string{ + "", "..", "../escape", "a/../../escape", "a//b", "a/./b", "a/\x00b", + strings.Repeat("a", maxCheckpointKeyLen+1), + } { + t.Run(key, func(t *testing.T) { + if err := put(t, s, key, "x"); err == nil { + t.Errorf("the key %q was accepted", key) + } + if _, err := s.Open(ctx, key); err == nil { + t.Errorf("the key %q was opened", key) + } + if err := s.Delete(ctx, key); err == nil { + t.Errorf("the key %q was deleted", key) + } + }) + } + // And nothing escaped the root while we were trying. + if _, err := os.Stat(filepath.Join(filepath.Dir(dir), "escape")); !errors.Is(err, os.ErrNotExist) { + t.Error("a key with .. in it wrote outside the store directory") + } +} + +// TestDirStoreIsPrivate: the object layout names a workspace and a session, so +// a directory listing tells a local user which sessions this host holds. +func TestDirStoreIsPrivate(t *testing.T) { + s, dir := dirStore(t) + if err := put(t, s, "ws/ws-1/manifest.json", "x"); err != nil { + t.Fatal(err) + } + for _, p := range []string{dir, filepath.Join(dir, "ws"), filepath.Join(dir, "ws", "ws-1")} { + info, err := os.Stat(p) + if err != nil { + t.Fatal(err) + } + if info.Mode().Perm() != 0o700 { + t.Errorf("%s is mode %v, want 0700", p, info.Mode().Perm()) + } + } +} + +// TestDirStoreServesTheCheckpointLibrary is the whole point of the type: a +// checkpoint written through it is one the library reads back. +func TestDirStoreServesTheCheckpointLibrary(t *testing.T) { + s, _ := dirStore(t) + ctx := context.Background() + keys, err := checkpoint.NewStaticKeyWrapper(SelfHostedCheckpointKeyRef, [32]byte{4, 2}) + if err != nil { + t.Fatal(err) + } + w, err := checkpoint.NewWriter(s, keys, checkpoint.WriterOptions{KeyRef: SelfHostedCheckpointKeyRef}) + if err != nil { + t.Fatal(err) + } + root := t.TempDir() + if err := os.WriteFile(filepath.Join(root, "a.txt"), []byte("alpha"), 0o644); err != nil { + t.Fatal(err) + } + c := checkpoint.Context{Workspace: "ws-1", Session: "sess-1", Generation: 1} + if _, err := w.Write(ctx, c, checkpoint.DirSource(root)); err != nil { + t.Fatalf("writing a checkpoint into the directory store: %v", err) + } + r, err := checkpoint.NewReader(s, keys, checkpoint.ReaderOptions{ + Authorize: func(context.Context, checkpoint.Context, checkpoint.Manifest) error { return nil }, + }) + if err != nil { + t.Fatal(err) + } + if _, err := r.Verify(ctx, c); err != nil { + t.Fatalf("verifying it back: %v", err) + } + // A second write at the same generation loses, and leaves the winner's + // objects alone. + if _, err := w.Write(ctx, c, checkpoint.DirSource(root)); !errors.Is(err, checkpoint.ErrExists) { + t.Fatalf("a second checkpoint at the same generation gave %v, want ErrExists", err) + } + if _, err := r.Verify(ctx, c); err != nil { + t.Fatalf("the committed checkpoint no longer verifies after a lost race: %v", err) + } +} + +// TestDirStoreTightensADirectoryItDidNotCreate. MkdirAll is a no-op on a +// directory that is already there, so an operator who made it with the default +// umask would leave the layout — which names a workspace and a session for +// every checkpoint on this host — readable by everybody. +func TestDirStoreTightensADirectoryItDidNotCreate(t *testing.T) { + dir := filepath.Join(t.TempDir(), "premade") + if err := os.Mkdir(dir, 0o755); err != nil { + t.Fatal(err) + } + if err := os.Chmod(dir, 0o755); err != nil { + t.Fatal(err) + } + if _, err := NewDirBlobStore(dir); err != nil { + t.Fatalf("NewDirBlobStore: %v", err) + } + info, err := os.Stat(dir) + if err != nil { + t.Fatal(err) + } + if info.Mode().Perm() != 0o700 { + t.Errorf("a pre-existing store directory is still mode %v", info.Mode().Perm()) + } +} + +func TestNewDirBlobStoreRefusesWhatItCannotUse(t *testing.T) { + if _, err := NewDirBlobStore(""); err == nil { + t.Error("an empty store directory was accepted") + } + file := filepath.Join(t.TempDir(), "a-file") + if err := os.WriteFile(file, []byte("x"), 0o600); err != nil { + t.Fatal(err) + } + if _, err := NewDirBlobStore(file); err == nil { + t.Error("a store directory that is a file was accepted") + } +} + +// TestLoadCheckpointKeyAcceptsBothSpellings: 64 hex characters is what +// `openssl rand -hex 32` produces and what fits in a secret manager; 32 raw +// bytes is what `openssl rand 32 > key` produces. Both are what operators +// actually have. +func TestLoadCheckpointKeyAcceptsBothSpellings(t *testing.T) { + want := [32]byte{} + for i := range want { + want[i] = byte(i + 1) + } + dir := t.TempDir() + + rawPath := filepath.Join(dir, "raw.key") + if err := os.WriteFile(rawPath, want[:], 0o600); err != nil { + t.Fatal(err) + } + got, err := LoadCheckpointKey(rawPath) + if err != nil || got != want { + t.Fatalf("the raw key loaded as %x, %v", got, err) + } + + // With a trailing newline, because a file written by `echo` has one and + // refusing that would be refusing the most likely correct key in the world. + hexPath := filepath.Join(dir, "hex.key") + if err := os.WriteFile(hexPath, []byte(hex.EncodeToString(want[:])+"\n"), 0o600); err != nil { + t.Fatal(err) + } + got, err = LoadCheckpointKey(hexPath) + if err != nil || got != want { + t.Fatalf("the hex key loaded as %x, %v", got, err) + } +} + +// TestLoadCheckpointKeyFailsClosed walks the conditions a real deployment gets +// wrong. Every one of them is a runner that must not start. +func TestLoadCheckpointKeyFailsClosed(t *testing.T) { + dir := t.TempDir() + write := func(name string, body []byte, mode os.FileMode) string { + t.Helper() + p := filepath.Join(dir, name) + if err := os.WriteFile(p, body, mode); err != nil { + t.Fatal(err) + } + if err := os.Chmod(p, mode); err != nil { + t.Fatal(err) + } + return p + } + valid := bytes.Repeat([]byte{9}, 32) + + cases := []struct { + name string + path string + want string + }{ + {"no file at all", filepath.Join(dir, "absent.key"), "reading the checkpoint key file"}, + {"no path at all", "", "required"}, + {"group readable", write("group.key", valid, 0o640), "must not be readable by group or other"}, + {"world readable", write("world.key", valid, 0o644), "must not be readable by group or other"}, + {"too short", write("short.key", bytes.Repeat([]byte{1}, 16), 0o600), "must be 32 raw bytes or 64 hex"}, + {"64 characters of not-hex", write("nothex.key", []byte(strings.Repeat("z", 64)), 0o600), "is not hex"}, + {"all zeros", write("zero.key", make([]byte, 32), 0o600), "all zeros"}, + {"a directory", dir, "not a regular file"}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + _, err := LoadCheckpointKey(tc.path) + if err == nil { + t.Fatalf("the key was accepted") + } + if !strings.Contains(err.Error(), tc.want) { + t.Errorf("the refusal is %q, want it to name %q", err, tc.want) + } + // A refusal about a key file must not quote the key. + if strings.Contains(err.Error(), string(valid)) || strings.Contains(err.Error(), hex.EncodeToString(valid)) { + t.Errorf("the refusal quotes key material: %v", err) + } + }) + } +} diff --git a/internal/driver/driver.go b/internal/driver/driver.go index 7751adc..5ef6cd6 100644 --- a/internal/driver/driver.go +++ b/internal/driver/driver.go @@ -221,6 +221,22 @@ type HostedDriver interface { SetHost(MicrovmHost) } +// CheckpointingDriver is a driver whose COLD suspend runs the workspace +// checkpoint handshake itself: the `suspending{cold}` notice, the stream that +// follows it, the write, and the verify (ADR-0003 §4.4). +// +// The runner asks before it sends a cold notice of its own. The notice and the +// stream are one handshake with one nonce, so two notices would have the guest +// quiesce twice and stream into a suspend nobody is reading — which is a failed +// suspend on a session that had nothing wrong with it. +// +// It is an optional interface for the reason CapabilityDriver is: the Docker +// driver has nothing to say here, and this change promises to leave its +// behaviour exactly as it is. +type CheckpointingDriver interface { + CheckpointsColdSuspend() bool +} + type Driver interface { Create(ctx context.Context, spec Spec) (Handle, error) Suspend(ctx context.Context, id string, warm bool) error // warm=pause, cold=stop diff --git a/internal/driver/microvm.go b/internal/driver/microvm.go index 625599c..f7b22d6 100644 --- a/internal/driver/microvm.go +++ b/internal/driver/microvm.go @@ -45,6 +45,7 @@ import ( "syscall" "time" + "github.com/tokencanopy/rainier/checkpoint" "github.com/tokencanopy/rainier/internal/driver/netslot" "github.com/tokencanopy/rainier/protocol/runner" ) @@ -166,6 +167,14 @@ type MicrovmOpts struct { // OBSERVE the clones, not to substitute for a missing host capability. Clone Cloner + // Checkpoint is what a cold suspend turns this session's workspace into: a + // portable workspace checkpoint, committed and verified before the VM is + // terminated (ADR-0003 §4.4). nil is a driver that does not checkpoint — + // the local dev surface and the contract suite, which have no store and no + // key — and runnerd requires both for --driver=microvm, so no production + // path reaches that state. See microvm_checkpoint.go. + Checkpoint *CheckpointOpts + Engine MicrovmEngine // test seam; nil in production Net netslot.Host // test seam; nil in production Format DiskFormatter // test seam; nil in production @@ -294,6 +303,19 @@ type MicrovmEngine interface { // attach a 10 GiB file of zeroes to the guest as though it were a filesystem. type DiskFormatter interface { Format(path string) error + // FormatFromDir is Format with the filesystem POPULATED from dir, which is + // how a deep-dormant resume turns a restored checkpoint back into a block + // device (ADR-0003 §2.3). `mkfs.ext4 -d` needs no loop device, no + // CAP_SYS_ADMIN and no mount — it is the same call the environment-image + // build uses (rainier-cloud infra/scripts/build-env-image.sh), for the same + // reason: this host must never MOUNT a filesystem it is about to hand a + // tenant, and it must never mount one a tenant handed it. + // + // It is on this interface rather than beside it because the two are one + // capability — putting a filesystem on an image — and a driver that could + // do one without the other would be a driver that can suspend a session it + // cannot resume. + FormatFromDir(path, dir string) error } // instanceRecord is the persistent metadata stored on disk for each microVM @@ -307,6 +329,26 @@ type instanceRecord struct { PID int `json:"pid"` Cfg VMMConfig `json:"cfg"` + // The portable workspace checkpoint this session has, if any. All three are + // PERSISTED, and they have to be: a cold resume after a runnerd restart is + // exactly the case where the workspace image may be gone and the checkpoint + // is the only copy left, and a generation this process forgot would be one + // a retry overwrote or one a restore could not name. + // + // None of them is secret. A generation is a counter, a manifest key is + // derived from the workspace and session ids this record already carries, + // and the checkpoint itself is unreadable without a key that lives nowhere + // near this file. + // + // CheckpointGeneration is the count of COMMITTED checkpoints, which is the + // generation of the newest one and the number the next one is taken at plus + // one. It advances only on a commit, so a suspend that failed before the + // manifest's put-if-absent leaves the next attempt the same number — which + // is what makes a retry a retry rather than a hole in the sequence. + CheckpointGeneration uint64 `json:"checkpoint_generation,omitempty"` + CheckpointKey string `json:"checkpoint_key,omitempty"` + CheckpointAt string `json:"checkpoint_at,omitempty"` + // boot is the configuration the guest is handed over vsock, and channel // is the host end of that conn. Both are unexported and therefore never // serialized, which is the point: the configuration carries the session's @@ -346,6 +388,14 @@ type instanceRecord struct { // each other's socket. resuming bool + // checkpointing is the same kind of claim for the workspace checkpoint a + // cold suspend takes. Two of them on one instance would each send the guest + // a cold suspend notice with a nonce of its own, each arm a waiter that + // replaces the other's, and each read part of one stream — producing two + // checkpoints of two halves of a workspace, both committed and both wrong. + // The second is refused rather than queued. + checkpointing bool + // snapshotting is the same kind of claim for a Snapshot, and it does one // more thing: it says that the VM is paused BY THIS DRIVER for a copy, so // a hypervisor reporting it as paused is not a session anybody suspended. @@ -424,6 +474,11 @@ type Microvm struct { // connection goes, and who asks the control plane for a fresh bootstrap // token on a cold resume. nil is a real state — see MicrovmHost. host MicrovmHost + + // ckpt is the portable workspace checkpoint's configuration, or nil for a + // driver that does not checkpoint. It is read on every cold suspend and + // every cold resume; it is written once, by NewMicrovm. + ckpt *CheckpointOpts } // NewMicrovm creates a new microVM driver, or fails. @@ -482,7 +537,7 @@ func NewMicrovm(opts MicrovmOpts) (*Microvm, error) { // The environment images themselves live under "images", by digest, and // are never opened for writing after they land; see image.go. - for _, dir := range []string{"workspaces", "homes", "instances", "rootfs"} { + for _, dir := range []string{"workspaces", "homes", "instances", "rootfs", "restore"} { if err := os.MkdirAll(filepath.Join(opts.StateDir, dir), microvmDirMode); err != nil { return nil, fmt.Errorf("microvm: create state directory: %w", err) } @@ -496,6 +551,19 @@ func NewMicrovm(opts MicrovmOpts) (*Microvm, error) { cloner = NewFileCloner() } + // The checkpoint's configuration is validated HERE, once, rather than on + // the first cold suspend — which is the worst possible moment to discover + // that a store or a key was missing, because by then a guest has been told + // its VM is ending. + if opts.Checkpoint != nil { + if opts.Checkpoint.Store == nil { + return nil, errors.New("microvm: a checkpoint configuration needs a blob store (--checkpoint-store-dir): a cold suspend has nowhere to put a workspace without one") + } + if opts.Checkpoint.Keys == nil { + return nil, errors.New("microvm: a checkpoint configuration needs a key wrapper (--checkpoint-key-file): an unencrypted workspace checkpoint is not one") + } + } + m := &Microvm{ opts: opts, engine: engine, @@ -503,6 +571,7 @@ func NewMicrovm(opts MicrovmOpts) (*Microvm, error) { format: format, images: images, cloner: cloner, + ckpt: opts.Checkpoint, instances: make(map[string]*instanceRecord), } // Records first, leftovers second, and never the other way round: a @@ -511,9 +580,38 @@ func NewMicrovm(opts MicrovmOpts) (*Microvm, error) { m.recoverDiskInstances() m.reclaimNetworkSlots() m.reclaimOrphanRootfs() + m.reclaimRestoreScratch() return m, nil } +// reclaimRestoreScratch removes whatever a previous run's checkpoint restores +// left under "restore". +// +// Every ordinary path removes its own scratch directory with a defer, but a +// host that is powered off or a runnerd that is killed mid-restore removes +// nothing — and what is in there is a tenant's whole workspace in PLAINTEXT, +// which the design note's §7 is only allowed to call "no new exposure" because +// it is removed before the guest boots. A tree that outlives the process that +// made it is a different claim, and not one this design makes. +// +// The whole directory goes, not one instance's: a restore in flight is a +// restore this process is running, and this runs before anything is served. +func (m *Microvm) reclaimRestoreScratch() { + dir := filepath.Join(m.opts.StateDir, "restore") + entries, err := os.ReadDir(dir) + if err != nil || len(entries) == 0 { + return + } + log.Printf("microvm: removing %d restore scratch tree(s) a previous run left behind", len(entries)) + for _, e := range entries { + if err := os.RemoveAll(filepath.Join(dir, e.Name())); err != nil { + // The name is an instance id and the cause is reduced, because what + // is under it is a path from inside a workspace. + log.Printf("microvm: removing the restore scratch tree of %s: %s", e.Name(), pathFreeCause(err)) + } + } +} + // reclaimOrphanRootfs removes per-session root filesystems left by a previous // run with no session left to own them. // @@ -1389,6 +1487,24 @@ func (m *Microvm) launch(ctx context.Context, id string, spec Spec) (*instanceRe workspaceDisk := "" if spec.SessionID != "" { + // Whether this session can ever be CHECKPOINTED is decided here, at the + // create, rather than at its first cold suspend. The checkpoint format's + // identifier rule is stricter than this driver's path-segment one — it + // wants a leading alphanumeric and caps the length — so a session id + // that passes one and not the other would create happily and then fail + // every cold suspend at the configuration stage, which is precisely the + // "discover it at the worst possible moment" this driver's other + // preflights exist to prevent. + if m.ckpt != nil { + c := checkpoint.Context{ + Workspace: workspaceVolume(spec.SessionID), + Session: spec.SessionID, + Generation: 1, + } + if err := c.Validate(); err != nil { + return nil, fmt.Errorf("microvm: this session id cannot be carried in a workspace checkpoint, so the session could never be cold suspended: %w", err) + } + } path, err := m.workspaceDiskPath(spec.SessionID) if err != nil { return nil, err @@ -1531,6 +1647,22 @@ func (m *Microvm) Suspend(ctx context.Context, id string, warm bool) error { return fmt.Errorf("no such id %s", id) } + // THE DURABILITY BARRIER (ADR-0003 §4.4). A cold suspend ends this VM and + // keeps only the workspace, so the workspace becomes a committed, verified + // checkpoint FIRST — before the engine is stopped, before the rootfs is + // discarded, before the slot goes back to the pool. Every failure below + // returns with the VM untouched and running. + // + // It runs with the driver mutex released: it is a copy of a filesystem + // across a socket and can take minutes, and holding the mutex across it + // would freeze Inspect, List, Capacity, Create and Destroy for every other + // session on this host. + if !warm && m.ckpt != nil { + if err := m.checkpointOnSuspend(ctx, id); err != nil { + return err + } + } + if warm { if err := m.engine.Pause(ctx, id); err != nil { return err @@ -1717,8 +1849,21 @@ func (m *Microvm) Resume(ctx context.Context, id string) (bool, error) { // everything else, and persisting THAT is what this design rules out. // Rebuilding it from the control plane on a resume is the follow-up // (a create-shaped resume, ADR-0003 §2.3's portable checkpoint). + // + // This refusal sits ABOVE the deep-dormant restore below it, and the + // order is deliberate rather than incidental — see deepDormantNote, + // which is what makes a recovered session whose workspace is also gone + // say WHICH of the two things it is waiting for. if !bootLive { - return false, fmt.Errorf("cold resume of %s: this session's guest configuration was held in memory only (ADR-0003 §2.7 item 1) and did not survive a runnerd restart; a clean relaunch needs the control plane to re-resolve it, which is the portable-checkpoint work and not this change", id) + return false, m.coldResumeNeedsGuestConfig(id) + } + // The DEEP-DORMANT case (ADR-0003 §2.3): the workspace image is gone + // and the checkpoint is the only copy of this session's work. Rebuilt + // before anything else is allocated, because a resume that cannot + // produce a workspace should spend no token, no slot and no rootfs on + // finding that out. + if err := m.ensureWorkspaceForResume(ctx, id); err != nil { + return false, err } // A new VM gets a new token and a new socket. The token because the // old one is single-use and fenced by a placement generation the @@ -2405,6 +2550,41 @@ func (e *Ext4Formatter) Format(path string) error { return nil } +// FormatFromDir is Format with `-d`, which populates the new filesystem from a +// directory tree without mounting anything. +// +// The flags are Format's, plus the tree. Two things are deliberately NOT here: +// +// - no `-N`. mke2fs sizes the inode table from the image's size, which for a +// 10 GiB workspace is about 655,000 inodes — above wstream's default entry +// ceiling, so a restored tree fits. Deriving the count from the tree +// instead would size the table for what is being restored rather than for +// what the session will write next, and a resumed session that runs out of +// inodes is a worse failure than a restore that refuses. +// - no `-U`/`-E hash_seed`. An environment image is built reproducibly +// because it is published by digest; a workspace image is one session's and +// is never addressed by its bytes. +func (e *Ext4Formatter) FormatFromDir(path, dir string) error { + out, err := exec.Command(e.mkfs, "-F", "-q", "-m", "0", "-d", dir, path).CombinedOutput() + if err != nil { + // mke2fs's own output is deliberately NOT in the error, which is the + // one way this differs from Format above. `-d` copies a tenant's tree, + // and create_inode.c reports a per-file failure by naming the file it + // was copying ("while opening \"/src/secret-project/key.pem\" + // to copy") — a path from inside a workspace, on an error that travels + // to a session's error column, which the tenancy specification's §15.1 + // prohibits. `-q` suppresses progress, not that. + // + // What is left is the exit status and the count of output bytes, which + // says "it failed and it had something to say" without saying it. An + // operator who needs the text runs the same command by hand against a + // tree of their own. + return fmt.Errorf("mkfs.ext4 -d %s: %w (%d bytes of output withheld: it names files inside the tenant's tree)", + path, err, len(strings.TrimSpace(string(out)))) + } + return nil +} + // --------------------------------------------------------------------------- // Firecracker Engine (production Linux hosts with /dev/kvm) // --------------------------------------------------------------------------- diff --git a/internal/driver/microvm_checkpoint.go b/internal/driver/microvm_checkpoint.go new file mode 100644 index 0000000..b98b46a --- /dev/null +++ b/internal/driver/microvm_checkpoint.go @@ -0,0 +1,776 @@ +// internal/driver/microvm_checkpoint.go +// +// The durability barrier: a cold suspend does not succeed until this session's +// workspace is a committed, verified checkpoint. +// +// ADR-0003 §4.4, quoted, because everything here is an attempt to satisfy +// exactly it: "Suspend(warm=false) must not report success, and the host must +// not release the workspace slot, until the portable checkpoint upload … +// completes, its manifest is committed atomically, and the restore test +// passes." The ellipsis is "to regional GCS", which a self-hosted runner writes +// to a local directory instead, behind the one BlobStore port. +// +// The shape is in docs/design/2026-09-23-workspace-checkpoint-wiring.md. In one +// paragraph: the guest streams its workspace over the relay conn it already +// has, this file feeds that stream to the checkpoint writer as an fs.FS, the +// writer encrypts it frame by frame into a blob store, and only once the +// manifest has committed AND been verified does the VM get terminated. Nothing +// plaintext is written to host disk on this path: the bytes go from a socket, +// through a 32 KiB copy buffer, into a frame, into the store. +package driver + +import ( + "context" + "errors" + "fmt" + "io" + "io/fs" + "log" + "os" + "path/filepath" + "time" + + "github.com/tokencanopy/rainier/checkpoint" + "github.com/tokencanopy/rainier/internal/wstream" +) + +// CheckpointOpts is what a host must give this driver before a cold suspend can +// produce a checkpoint. It is all ports and policy — a blob store, a key +// wrapper, a key reference, limits — so that a self-hosted runner and a hosted +// cell differ in what they pass here and in nothing else. +// +// A nil *CheckpointOpts on MicrovmOpts is a driver that does not checkpoint: +// the local dev surface and the driver contract suite, which have no store and +// no key. Its cold suspend is what it was before this existed. That is a +// deliberate seam and not a fallback — runnerd REQUIRES both flags for +// --driver=microvm (see cmd/runnerd), so no production path reaches it. +type CheckpointOpts struct { + // Store is where the two objects of a checkpoint go. The local-directory + // store is in this package (DirBlobStore); GCS is rainier-cloud's. + Store checkpoint.BlobStore + // Keys wraps each checkpoint's data key. StaticKeyWrapper for self-hosted, + // a KMS for a cell. + Keys checkpoint.Wrapper + // KeyRef is the key-encryption key to wrap under. It may be an alias; what + // the wrapper reports is what the manifest records. + KeyRef checkpoint.KeyRef + // Prefix is the storage prefix every object goes under. It may be empty. + Prefix string + // Limits are what this host enforces on the guest's stream. The zero value + // is wstream's defaults, plus the checkpoint package's own exclusions, + // which are added by this driver and cannot be removed by configuration. + Limits wstream.Limits + // VerifyTimeout bounds the restore test that follows the commit. Zero means + // defaultCheckpointVerifyTimeout. + VerifyTimeout time.Duration + // OwnerUID and OwnerGID are the user a RESTORED workspace is given to + // before it becomes a filesystem: the uid and gid the guest's agent runs + // as, which is the session image's own user (1000:1000 for Rainier's). + // + // They are configuration and not a constant because a checkpoint records + // modes and not owners — a uid is not portable across hosts, and the format + // is meant to be — so the ownership has to come from somewhere, and the + // only party that knows which user this environment's image runs as is the + // operator who built it. + // + // 0, 0 means "leave every restored file owned by whoever restored it". That + // is right for a host whose guest runs as root and for a test with no + // privilege to give a file away, and wrong for every production microVM + // host — see chownTree for what it costs. + OwnerUID int + OwnerGID int +} + +// defaultCheckpointVerifyTimeout bounds Verify: one sequential read of the +// object that was just written, through the same store. Ten minutes is far +// above a local directory and above regional object storage for a workspace +// the guest was able to stream inside its own budget. +const defaultCheckpointVerifyTimeout = 10 * time.Minute + +// checkpointStage names which step of the barrier failed. It travels into a +// session's error column, so it is a WORD and never a path, a name or a value +// (tenancy §15.1) — the cost of that rule is that an operator gets a stage and +// a count instead of a file, which is the same trade the checkpoint library +// makes and for the same reason. +type checkpointStage string + +const ( + stageConfig checkpointStage = "config" + stageStream checkpointStage = "stream" + stageWrite checkpointStage = "write" + stageVerify checkpointStage = "verify" +) + +// checkpointResult is what a committed checkpoint leaves on the instance +// record. +type checkpointResult struct { + Generation uint64 + ManifestKey string + Entries int64 + Bytes int64 + Nonce uint64 +} + +// CheckpointsColdSuspend reports whether this driver's cold suspend runs the +// workspace checkpoint handshake itself. +// +// runnerd asks before it sends its own cold suspend notice: the notice and the +// stream that follows it are one handshake with one nonce, and two notices +// would have the guest quiesce twice and stream into a suspend nobody is +// reading. See runnerd.Op. +func (m *Microvm) CheckpointsColdSuspend() bool { return m.ckpt != nil } + +// checkpointOnSuspend is the barrier as Suspend reaches it: claim the instance, +// read what the checkpoint needs off the record, run it with the mutex +// released, and record the result. +// +// A session with no workspace disk — a create that carried no session id, which +// the driver contract's own fixtures do — has nothing to checkpoint and is not +// an error: there is no workspace, so there is no work to lose. +func (m *Microvm) checkpointOnSuspend(ctx context.Context, id string) error { + m.mu.Lock() + inst, ok := m.instances[id] + if !ok { + m.mu.Unlock() + return fmt.Errorf("no such id %s", id) + } + if inst.checkpointing { + m.mu.Unlock() + return fmt.Errorf("cold suspend of %s: a workspace checkpoint is already in flight for this instance", id) + } + sessionID, volume := inst.SessionID, inst.Volume + generation := inst.CheckpointGeneration + 1 + hasWorkspace := inst.Cfg.WorkspaceDiskPath != "" + inst.checkpointing = true + m.mu.Unlock() + defer func() { + m.mu.Lock() + if e, ok := m.instances[id]; ok { + e.checkpointing = false + } + m.mu.Unlock() + }() + + if !hasWorkspace || sessionID == "" { + log.Printf("microvm: %s has no workspace disk; there is nothing to checkpoint", id) + return nil + } + + _, err := m.checkpointWorkspace(ctx, id, sessionID, volume, generation, m.recordCheckpoint) + return err +} + +// recordCheckpoint puts a COMMITTED checkpoint on the instance record and on +// this host's disk, and it is called the moment the manifest's put-if-absent +// returns — before the verify, before the drain, before anything else that can +// still fail. +// +// That timing is the whole point, and getting it wrong is a trap with no +// bottom. A manifest key is derived from (workspace, session, generation) with +// no attempt suffix, and put-if-absent never overwrites: a generation whose +// manifest is committed is SPENT, permanently, whatever happens next. If the +// record only advanced on success, a failed verify would leave every later +// attempt recomputing the same spent generation, losing its manifest put to +// ErrExists, and this session could never be cold-parked again — while each +// attempt left another orphan content object behind. +// +// Persisted rather than merely bumped, for the same reason: `Suspend` has +// returns between here and its own saveRecord, and a runnerd restart that read +// back the older generation would walk into the same trap from the other side, +// and would restore an older checkpoint than the one that exists. +func (m *Microvm) recordCheckpoint(id string, res checkpointResult) error { + m.mu.Lock() + inst, ok := m.instances[id] + if !ok { + m.mu.Unlock() + // A Destroy raced the checkpoint. The objects are committed and are + // left exactly where they are: deleting a tenant's only copy of their + // work because the record went away is the one outcome this barrier + // exists to prevent, and the retention sweeper above this driver is + // what removes a checkpoint on purpose. + return fmt.Errorf("no such id %s", id) + } + inst.CheckpointGeneration = res.Generation + inst.CheckpointKey = res.ManifestKey + inst.CheckpointAt = time.Now().UTC().Format(time.RFC3339) + inst.bump() + rec := persistable(inst) + m.mu.Unlock() + return m.saveRecord(rec) +} + +// checkpointWorkspace is the barrier, in the order the barrier is: +// +// 1. ask the guest to cold suspend and read the workspace it streams back; +// 2. present that stream as the fs.FS the checkpoint writer walks; +// 3. write — which encrypts, uploads, and commits the manifest with one +// put-if-absent, the atomic commit §4.4 rests on; +// 4. VERIFY, by reading the committed object back through the store, because +// the claim being made is that what is stored is restorable and checking +// the bytes still in memory would test the encoder against itself; +// 5. tell the guest, and only then let the caller terminate the VM. +// +// Any failure before (4) returns and the VM is untouched: the caller has not +// stopped it, the workspace disk is where it was, and runnerd's own +// settleFailedColdSuspend rolls the session back to running. Failure AFTER the +// commit still fails the suspend — a checkpoint that cannot be read back is not +// one — and leaves the committed objects for the sweeper rather than deleting +// them, because a reconciler racing itself would then be able to delete the +// checkpoint it had just written. +func (m *Microvm) checkpointWorkspace(ctx context.Context, id, sessionID, volume string, generation uint64, + commit func(id string, res checkpointResult) error) (checkpointResult, error) { + opts := m.ckpt + host := m.currentHost() + if host == nil { + return checkpointResult{}, m.checkpointErr(id, stageConfig, + errors.New("this driver has no runner above it to stream the workspace over")) + } + c := checkpoint.Context{Workspace: volume, Session: sessionID, Generation: generation} + if err := c.Validate(); err != nil { + return checkpointResult{}, m.checkpointErr(id, stageConfig, err) + } + w, err := checkpoint.NewWriter(opts.Store, opts.Keys, checkpoint.WriterOptions{ + Prefix: opts.Prefix, + KeyRef: opts.KeyRef, + }) + if err != nil { + return checkpointResult{}, m.checkpointErr(id, stageConfig, err) + } + r, err := checkpoint.NewReader(opts.Store, opts.Keys, checkpoint.ReaderOptions{ + Prefix: opts.Prefix, + // The producer running its own restore test is the principal that just + // wrote the checkpoint, so this hook has nothing to decide. It is not a + // loophole — the library cannot judge a policy, only guarantee the step + // exists — and a DESTINATION restoring somebody else's checkpoint puts + // its real check here (see restoreWorkspaceDisk, which is the same + // principal on the same host). + Authorize: func(context.Context, checkpoint.Context, checkpoint.Manifest) error { return nil }, + }) + if err != nil { + return checkpointResult{}, m.checkpointErr(id, stageConfig, err) + } + // The generation this attempt actually takes, which is not always the one + // the record suggested. A manifest key has no attempt suffix and is never + // overwritten, so a generation whose manifest is already committed is spent + // — and the record can be behind the store: a host that died between the + // commit and its own saveRecord, a record recovered from an older copy. + // Asking costs one small GET before a copy measured in minutes, and not + // asking costs a session that can never be cold-parked again. + if c.Generation, err = m.freeGeneration(ctx, r, c); err != nil { + return checkpointResult{}, m.checkpointErr(id, stageConfig, err) + } + generation = c.Generation + + // The stream, on its own goroutine, writing into a pipe the checkpoint + // writer reads. There is no buffer between them and no temporary file: the + // pipe is what makes the guest's write rate the store's, which is the + // back-pressure that keeps a workspace from ever existing on this host. + pr, pw := io.Pipe() + type streamOutcome struct { + rep WorkspaceStream + err error + } + done := make(chan streamOutcome, 1) + go func() { + rep, err := host.StreamWorkspace(ctx, sessionID, pw) + // Closing the write end is what ends the reader below, with this + // error if there was one — so a guest that died mid-stream surfaces as + // a failed checkpoint rather than as a reader waiting for bytes that + // are not coming. + _ = pw.CloseWithError(err) + done <- streamOutcome{rep: rep, err: err} + }() + // Whatever happens below, the stream goroutine is unblocked and waited for + // before this returns: a suspend that left one running would have a guest + // still streaming into a pipe nobody reads, holding the hub's read loop. + fail := func(stage checkpointStage, err error) (checkpointResult, error) { + _ = pr.CloseWithError(err) + out := <-done + // Which of the two errors is the CAUSE. A guest that died mid-stream is + // seen here as a malformed or truncated stream — the reader's honest + // account of a pipe that closed under it — and reporting that would + // send somebody looking at a parser when the sandbox is what went away. + // A store that refused is the opposite: the stream then fails BECAUSE + // this end stopped taking bytes, and naming the stream would hide the + // outage. So the stream's own error wins exactly when the local one is + // about the bytes. + if out.err != nil && (stage == stageStream || isStreamError(err)) { + return checkpointResult{}, m.checkpointErr(id, stageStream, out.err) + } + return checkpointResult{}, m.checkpointErr(id, stage, err) + } + + lim := opts.Limits + // The exclusions are ADDED here, by this driver, in the one place every + // checkpoint of a workspace passes through. A host that configured none + // still gets the set the checkpoint library owns. + lim.Exclude = append(append([]string(nil), lim.Exclude...), checkpoint.DefaultExclusions()...) + fsys, err := wstream.NewFS(pr, lim) + if err != nil { + return fail(stageStream, err) + } + res, err := w.Write(ctx, c, checkpoint.Source{FS: fsys}) + if err != nil { + // Before the manifest's put-if-absent, so there is no checkpoint at + // this generation and it stays free for the next attempt. + return fail(stageWrite, err) + } + // COMMITTED. From here the generation is spent whatever happens, so it is + // recorded and persisted before anything below can fail — see + // recordCheckpoint for the trap this avoids. + committed := checkpointResult{Generation: generation, ManifestKey: res.ManifestKey} + if commit != nil { + if err := commit(id, committed); err != nil { + return fail(stageWrite, err) + } + } + + // The rest of the stream: the tar's trailing blocks, which the walk above + // never asked for, and anything the host excluded that sorts after the last + // entry it read. Drained rather than ignored, because the guest is blocked + // writing them and cannot send the end marker this barrier is waiting for + // until they are gone — and drained through the stream's OWN byte limit, so + // a guest that keeps writing is stopped by the bound that already governs + // everything else it sends rather than by a second, smaller number. + if err := fsys.Drain(); err != nil { + return fail(stageStream, err) + } + _ = pr.Close() + out := <-done + if out.err != nil { + return committed, m.checkpointErr(id, stageStream, out.err) + } + if out.rep.Entries != int64(fsys.Indexed()) { + return committed, m.checkpointErr(id, stageStream, + fmt.Errorf("the guest reported %d entries and described %d", out.rep.Entries, fsys.Indexed())) + } + + // The restore test, on a context of its own: the caller's may be nearly + // spent by a long stream, and a verify that is skipped for lack of time is + // a durability barrier that is not one. + verifyCtx, cancel := context.WithTimeout(context.WithoutCancel(ctx), opts.verifyTimeout()) + defer cancel() + rep, err := r.Verify(verifyCtx, c) + if err != nil { + return committed, m.checkpointErr(id, stageVerify, err) + } + log.Printf("microvm: %s checkpointed its workspace at generation %d: %d entries, %d bytes of tree, %d frames", + id, generation, rep.Entries, rep.FileBytes, rep.Summary.Frames) + + host.CheckpointCommitted(sessionID, out.rep.Nonce) + return checkpointResult{ + Generation: generation, + ManifestKey: res.ManifestKey, + Entries: rep.Entries, + Bytes: out.rep.Bytes, + Nonce: out.rep.Nonce, + }, nil +} + +// isStreamError reports whether err is about the BYTES the guest sent rather +// than about what this host did with them. It is the whole of how the barrier +// tells a sandbox that went away from a store that would not take an object, +// which are the two failures with opposite operator actions. +// maxGenerationProbe bounds the search for a free generation. Each step is one +// small GET, and needing more than a handful means the record and the store +// have drifted by more than any sequence of crashes explains — at which point +// refusing is better than walking a counter up someone else's checkpoints. +const maxGenerationProbe = 8 + +// freeGeneration returns the first generation at or after c.Generation that has +// no committed manifest. +// +// Anything but ErrNotFound counts as taken, deliberately: a manifest that is +// there but cannot be authenticated, or is wrapped under a key this host cannot +// reach, is still a manifest that put-if-absent will refuse to replace. +func (m *Microvm) freeGeneration(ctx context.Context, r *checkpoint.Reader, c checkpoint.Context) (uint64, error) { + start := c.Generation + for gen := start; gen < start+maxGenerationProbe; gen++ { + c.Generation = gen + _, err := r.Preflight(ctx, c) + switch { + case errors.Is(err, checkpoint.ErrNotFound): + return gen, nil + case errors.Is(err, context.Canceled), errors.Is(err, context.DeadlineExceeded): + return 0, err + case gen != start: + // Only worth a line when the record was actually behind; the first + // probe is the ordinary case and says nothing. + log.Printf("microvm: checkpoint generation %d is already committed for this session; trying the next", gen) + } + } + return 0, fmt.Errorf("the next %d checkpoint generations from %d are already committed; this session's record and its store have drifted", + maxGenerationProbe, start) +} + +// The checkpoint sentinels are in the list beside this package's own, and they +// have to be: the library reduces a file system's error to a CATEGORY rather +// than wrapping it (checkpoint/tree.go's fsCategory, which exists so that no +// path can travel out in an error), so a stream that failed under its walk +// arrives here as ErrSource with the sentence flattened into text. ErrSource +// and ErrEntry both mean "the bytes I was given are the problem", which for +// this caller means the stream. +func isStreamError(err error) bool { + for _, s := range []error{ + checkpoint.ErrSource, checkpoint.ErrEntry, + wstream.ErrFormat, wstream.ErrEntry, wstream.ErrTruncated, wstream.ErrLimit, wstream.ErrSource, + } { + if errors.Is(err, s) { + return true + } + } + return false +} + +// checkpointErr is the one shape a failed barrier reports: which stage, and the +// underlying sentence. +// +// The instance id is in it and nothing else is. A session id, a path, a file +// name, a symlink target and a file byte are all session content (tenancy +// §15.1) and this error reaches an operator's log and a session's error column; +// the checkpoint library and the stream package are both written to carry none +// of them, so what is wrapped here is safe to print. +func (m *Microvm) checkpointErr(id string, stage checkpointStage, err error) error { + return fmt.Errorf("microvm: the cold suspend of %s failed at the checkpoint %s stage: %w", id, stage, err) +} + +func (o *CheckpointOpts) verifyTimeout() time.Duration { + if o.VerifyTimeout > 0 { + return o.VerifyTimeout + } + return defaultCheckpointVerifyTimeout +} + +// --------------------------------------------------------------------------- +// the deep-dormant resume +// --------------------------------------------------------------------------- + +const ( + stageRestore checkpointStage = "restore" + stageOwner checkpointStage = "owner" + stageMkfs checkpointStage = "mkfs" +) + +// coldResumeNeedsGuestConfig is the refusal a cold resume gets when the record +// it is resuming was recovered from disk: the guest configuration this driver +// holds in memory only (ADR-0003 §2.7 item 1) did not survive the restart. +// +// It is stated here, beside the restore, because of what the review found: a +// session dormant long enough for the deep-dormant tier to have deleted its +// workspace image has almost certainly outlived the runnerd that created it, so +// this refusal is the gate the restore path below is actually behind, and the +// ORDER is not what puts it there. Running the restore first would not change +// the outcome by one call — it would spend minutes of copying and a plaintext +// scratch tree on a resume that is certain to fail three lines later, and widen +// the very exposure window §7 bounds by removing the tree before the guest +// boots. So the refusal stays first, and it says what it is waiting for. +// +// The precondition, written down rather than implied: THE DEEP-DORMANT RESTORE +// IS REACHABLE ONLY FROM A RECORD THIS PROCESS STILL HAS THE GUEST +// CONFIGURATION FOR. Making it reachable after a restart is not a change to +// this barrier; it is the create-shaped resume in which the control plane +// re-resolves the session's configuration (ADR-0003 §2.3, §2.7 item 1), which +// is rainier-cloud's and is listed as owed in the note's §9. Persisting the +// configuration here instead is the one answer this design rules out, and +// booting a guest that was never told what it is — no session id, no proxy, no +// boot chain — is what the refusal exists to prevent. +func (m *Microvm) coldResumeNeedsGuestConfig(id string) error { + msg := fmt.Sprintf("cold resume of %s: this session's guest configuration was held in memory only (ADR-0003 §2.7 item 1) and did not survive a runnerd restart; a clean relaunch needs the control plane to re-resolve it, which is the portable-checkpoint work and not this change", id) + if note := m.deepDormantNote(id); note != "" { + return errors.New(msg + "; " + note) + } + return errors.New(msg) +} + +// deepDormantNote says, when it applies, that this session is ALSO the +// deep-dormant case: its workspace image is gone, so whatever resumes it next +// has to rebuild the disk from a checkpoint before it boots anything. +// +// It exists so that the refusal above names which session is which. "Resume +// failed, configuration is gone" is one operator action (re-dispatch through +// the control plane); "and the only copy of this person's work is checkpoint +// generation 4" is a different one, and a session whose image is gone with NO +// checkpoint behind it is a third that nothing can recover at all. Telling them +// apart costs one Stat and no tenant bytes. +// +// Everything in the note is safe to print: a generation is a counter and an +// instance id is this host's own. A path, a name or a byte from inside the +// workspace is not, and none is here (tenancy §15.1). +func (m *Microvm) deepDormantNote(id string) string { + m.mu.Lock() + inst, ok := m.instances[id] + if !ok { + m.mu.Unlock() + return "" + } + diskPath, generation := inst.Cfg.WorkspaceDiskPath, inst.CheckpointGeneration + m.mu.Unlock() + + if diskPath == "" { + return "" + } + if _, err := os.Stat(diskPath); err == nil || !errors.Is(err, os.ErrNotExist) { + // The image is there, or this host cannot tell — either way the + // ordinary cold resume is what this session is waiting for, and the + // restore has nothing to say about it. + return "" + } + if generation == 0 { + return "its workspace image is gone as well and no checkpoint of it was ever committed, so there is nothing left to restore it from" + } + if m.ckpt == nil { + return fmt.Sprintf("its workspace image is gone as well, and the only copy of this session's work is committed checkpoint generation %d, which this runner has no checkpoint store configured to read", generation) + } + return fmt.Sprintf("its workspace image is gone as well, so the resume that does re-resolve the configuration must also restore committed checkpoint generation %d before it boots anything", generation) +} + +// ensureWorkspaceForResume is the deep-dormant branch of a cold resume: it does +// nothing at all unless this session's workspace image is GONE. +// +// The ordinary cold resume is unchanged and must stay that way. A parked +// session's workspace disk is right where it was (ADR-0003 §4.1's Suspend row +// persists it), the guest mounts the same block device, and restoring over it +// would replace a newer filesystem with an older checkpoint's copy of it. +// +// So the image's ABSENCE is the whole trigger, and it is the honest one: the +// only thing that removes it is RemoveWorkspace, called from above this driver +// when the session goes deep dormant. +func (m *Microvm) ensureWorkspaceForResume(ctx context.Context, id string) error { + m.mu.Lock() + inst, ok := m.instances[id] + if !ok { + m.mu.Unlock() + return fmt.Errorf("no such id %s", id) + } + sessionID, volume := inst.SessionID, inst.Volume + diskPath, generation := inst.Cfg.WorkspaceDiskPath, inst.CheckpointGeneration + m.mu.Unlock() + + if diskPath == "" { + return nil // a session with no workspace disk at all + } + if _, err := os.Stat(diskPath); err == nil { + return nil // the ordinary cold resume: the disk is where it was + } else if !errors.Is(err, os.ErrNotExist) { + return m.restoreErr(id, stageRestore, err) + } + + if m.ckpt == nil { + return fmt.Errorf("cold resume of %s: this session's workspace image is gone and this runner has no checkpoint store to restore it from", id) + } + if generation == 0 { + // Said plainly rather than booted around. An empty workspace that looks + // like a successful resume is the failure a person discovers by finding + // their work missing, which is the one outcome worth refusing for. + return fmt.Errorf("cold resume of %s: this session's workspace image is gone and no checkpoint of it was ever committed; there is nothing to restore", id) + } + return m.restoreWorkspaceDisk(ctx, id, sessionID, volume, diskPath, generation) +} + +// restoreWorkspaceDisk rebuilds a session's workspace image from its newest +// committed checkpoint, for the case ADR-0003 §2.3 calls deep dormant: the disk +// has been deleted and the checkpoint is the only copy of the work left. +// +// Restore into a scratch directory, chown it to the user the guest runs as, +// build an ext4 from it with `mkfs.ext4 -d`, remove the scratch directory. The +// host never mounts anything, at either end of the session's life. +// +// The scratch directory is no new exposure — the workspace image on this host +// is plaintext today, in this same state directory, with provider encryption as +// defense in depth — but it is 0700, it is never inside a jail, and it is +// removed before the guest boots, on every path including the failures. +func (m *Microvm) restoreWorkspaceDisk(ctx context.Context, id, sessionID, volume, diskPath string, generation uint64) error { + opts := m.ckpt + c := checkpoint.Context{Workspace: volume, Session: sessionID, Generation: generation} + if err := c.Validate(); err != nil { + return m.restoreErr(id, stageConfig, err) + } + r, err := checkpoint.NewReader(opts.Store, opts.Keys, checkpoint.ReaderOptions{ + Prefix: opts.Prefix, + // The principal restoring is the one that wrote it, on the host that + // holds the session. A DESTINATION in another cell puts its real check + // here; this hook exists so that no path through the library can be the + // place where authorizing a restore was forgotten. + Authorize: func(context.Context, checkpoint.Context, checkpoint.Manifest) error { return nil }, + }) + if err != nil { + return m.restoreErr(id, stageConfig, err) + } + + scratch, err := m.scratchDir(id) + if err != nil { + return m.restoreErr(id, stageRestore, err) + } + // Removed on EVERY path, including a restore that failed part way through a + // tree: a tenant's files left in a directory nobody names again would be + // exactly the exposure this design says it does not add. + defer func() { + if err := os.RemoveAll(scratch); err != nil { + // Reduced to its cause, like every other error on this path: a + // RemoveAll failure is an *fs.PathError naming the entry it could + // not remove, which here is a path from inside the workspace. + log.Printf("microvm: removing the restore scratch directory of %s: %s", id, pathFreeCause(err)) + } + }() + + rep, err := r.Restore(ctx, c, scratch) + if err != nil { + return m.restoreErr(id, stageRestore, err) + } + if err := chownTree(scratch, opts.OwnerUID, opts.OwnerGID); err != nil { + return m.restoreErr(id, stageOwner, err) + } + if err := m.formatRestored(diskPath, scratch); err != nil { + return m.restoreErr(id, stageMkfs, err) + } + log.Printf("microvm: %s restored its workspace from checkpoint generation %d: %d entries, %d bytes", + id, generation, rep.Entries, rep.FileBytes) + return nil +} + +// formatRestored builds the workspace image from dir and puts it at path, and +// the image appears at that path only when it is a whole filesystem. +// +// It is built at a PARTIAL name and renamed, which is the same atomic-commit +// discipline DirBlobStore uses one layer up, and here it is load-bearing for a +// reason a comment has to state: the trigger for restoring at all is the +// image's ABSENCE (ensureWorkspaceForResume). A 10 GiB file created at the +// final path and then populated by a mkfs that takes minutes is, for those +// minutes, a file a crash leaves behind — and the next resume would see it, +// skip the restore, attach an unformatted image, and hand the guest a +// filesystem its own /init would helpfully format and mount EMPTY. The +// checkpoint would still be in the store, and nothing would ever consult it +// again. A partial name is invisible to that Stat. +func (m *Microvm) formatRestored(path, dir string) error { + if err := os.MkdirAll(filepath.Dir(path), microvmDirMode); err != nil { + return err + } + partial := path + ".partial" + // Whatever a previous attempt left at that name goes first: it is a + // half-built image of this same session's workspace and nothing reads it. + if err := os.Remove(partial); err != nil && !errors.Is(err, os.ErrNotExist) { + return err + } + done := false + defer func() { + if !done { + _ = os.Remove(partial) + } + }() + + f, err := os.OpenFile(partial, os.O_RDWR|os.O_CREATE|os.O_EXCL, microvmFileMode) + if err != nil { + return err + } + if err := f.Truncate(workspaceDiskBytes); err != nil { + f.Close() + return err + } + f.Close() + if err := m.format.FormatFromDir(partial, dir); err != nil { + return err + } + if err := os.Rename(partial, path); err != nil { + return err + } + done = true + return nil +} + +// scratchDir makes this restore's private directory, 0700, under the state +// directory. The suffix is the boot counter rather than a random name so that a +// directory left behind by a host that died mid-restore is recognisably one +// instance's and is replaced by the next attempt rather than accumulating. +func (m *Microvm) scratchDir(id string) (string, error) { + if err := checkPathSegment("instance id", id); err != nil { + return "", err + } + dir := filepath.Join(m.opts.StateDir, "restore", id) + // Whatever a previous attempt left is gone before this one starts: a + // restore into a non-empty target is refused by the library, and a + // half-restored tree from a host that died is not a tree to build a + // filesystem from. + // + // Reduced to its cause, exactly like the deferred removal ten lines below + // and for the same reason: what this fails on is an entry INSIDE a previous + // restore of this workspace, so the *fs.PathError it returns names a + // tenant's file (tenancy §15.1), and this error reaches a session's error + // column through restoreErr. + if err := os.RemoveAll(dir); err != nil { + return "", fmt.Errorf("clearing the scratch directory a previous restore of %s left: %s", id, pathFreeCause(err)) + } + if err := os.MkdirAll(filepath.Dir(dir), microvmDirMode); err != nil { + return "", err + } + if err := os.Mkdir(dir, 0o700); err != nil { + return "", err + } + // MkdirAll and Mkdir both mask the mode with the process umask, so the + // permission a comment claims is set explicitly rather than requested. + if err := os.Chmod(dir, 0o700); err != nil { + return "", err + } + return dir, nil +} + +// chownTree gives every restored entry to the user the guest runs as. +// +// It is not optional on a real host, and it is the one thing about this path +// that a fake cannot show. A checkpoint records modes and not owners (the +// format is portable across hosts, and a uid is not), so a restored tree is +// owned by whoever restored it — root, on a microVM host. The guest's agent +// runs as the session image's own user, and the guest's /init chowns the +// workspace mount point only when it is EMPTY, precisely so that a resumed +// workspace's contents are left alone. So without this the session comes back +// with every one of its files owned by root and unwritable by the agent. +// +// A zero uid and gid mean "leave it", which is what a test on a machine with no +// privilege to give a file away wants, and what a host running the guest as +// root would want. Anything else is applied with Lchown, which changes the LINK +// rather than what it points at — a symlink out of the tree is impossible +// (checkLink refuses one at both ends), and following one would still be the +// wrong thing to do here. +func chownTree(root string, uid, gid int) error { + if uid == 0 && gid == 0 { + return nil + } + return filepath.WalkDir(root, func(p string, _ fs.DirEntry, err error) error { + // Both branches are reduced to their cause, and for one reason: the + // path here is this host's scratch directory plus a name from inside + // the workspace, and a name is session content (tenancy §15.1). The + // walk's own error (an Lstat or a ReadDir that failed) carries it just + // as much as Lchown's does. + if err != nil { + return fmt.Errorf("reading a restored entry: %s", pathFreeCause(err)) + } + if err := os.Lchown(p, uid, gid); err != nil { + return fmt.Errorf("giving a restored entry to %d:%d: %s", uid, gid, pathFreeCause(err)) + } + return nil + }) +} + +// pathFreeCause reduces a file-system error to the part that is safe to print. +// An *fs.PathError and an *os.LinkError both carry the path that failed — +// which, on the restore path, is a name from inside a tenant's workspace — +// wrapped around an errno that carries nothing. +func pathFreeCause(err error) string { + var pe *fs.PathError + if errors.As(err, &pe) && pe.Err != nil { + return pe.Err.Error() + } + var le *os.LinkError + if errors.As(err, &le) && le.Err != nil { + return le.Err.Error() + } + if err == nil { + return "unknown error" + } + // Anything else is an error this package did not produce, whose text is not + // known to be path-free. Reduced rather than forwarded. + return "input/output error" +} + +func (m *Microvm) restoreErr(id string, stage checkpointStage, err error) error { + return fmt.Errorf("microvm: the cold resume of %s failed at the checkpoint %s stage: %w", id, stage, err) +} diff --git a/internal/driver/microvm_checkpoint_test.go b/internal/driver/microvm_checkpoint_test.go new file mode 100644 index 0000000..ef32ed7 --- /dev/null +++ b/internal/driver/microvm_checkpoint_test.go @@ -0,0 +1,705 @@ +// internal/driver/microvm_checkpoint_test.go +package driver + +import ( + "context" + "errors" + "fmt" + "io" + "io/fs" + "os" + "path/filepath" + "strings" + "sync" + "testing" + + "github.com/tokencanopy/rainier/checkpoint" + "github.com/tokencanopy/rainier/internal/wstream" +) + +// The durability barrier, against the simulated engine, a guest that streams a +// real workspace, and an in-memory blob store. +// +// Every test here is about one rule: a cold suspend does not succeed until the +// manifest has committed AND verified, and every failure before that leaves the +// VM running. + +// workspaceMarker is a string that exists only inside the fixture workspace. If +// it appears anywhere under the state directory, this host wrote a tenant's +// plaintext to its own disk. +const workspaceMarker = "marker-that-must-never-reach-host-disk" + +// guestWorkspace builds the tree a fake guest streams. +func guestWorkspace(t *testing.T) string { + t.Helper() + root := t.TempDir() + for rel, content := range map[string]string{ + "README.md": "hello " + workspaceMarker, + "src/main.go": "package main // " + workspaceMarker, + ".rainier/session.json": `{"token":"must-not-travel"}`, + } { + p := filepath.Join(root, rel) + if err := os.MkdirAll(filepath.Dir(p), 0o755); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(p, []byte(content), 0o644); err != nil { + t.Fatal(err) + } + } + if err := os.Symlink("src/main.go", filepath.Join(root, "entry")); err != nil { + t.Fatal(err) + } + return root +} + +// streamingHost is the runner above the driver, for the checkpoint path: it +// answers StreamWorkspace by streaming a real directory in the real format, so +// what the driver's barrier consumes is what a guest produces. +type streamingHost struct { + stubMicrovmHost + root string + // cut, when positive, is how many bytes to stream before failing — a guest + // that died half way through. + cut int64 + // err, when set, is a handshake that failed before a byte moved. + err error + // entriesFudge is added to the entry count the guest reports, standing in + // for two ends that do not agree about what was carried. + entriesFudge int64 + + mu sync.Mutex + calls int + commit []uint64 +} + +func (h *streamingHost) StreamWorkspace(ctx context.Context, _ string, dst io.Writer) (WorkspaceStream, error) { + h.mu.Lock() + h.calls++ + h.mu.Unlock() + if h.err != nil { + return WorkspaceStream{}, h.err + } + if h.cut > 0 { + // Written straight into the sink and then abandoned, which is what the + // driver sees when a guest dies mid-stream: some bytes, and no end + // marker ever. + var buf strings.Builder + if _, err := wstream.Write(ctx, os.DirFS(h.root), &buf, wstream.Limits{}); err != nil { + return WorkspaceStream{}, err + } + s := buf.String() + if int64(len(s)) > h.cut { + s = s[:h.cut] + } + if _, err := io.WriteString(dst, s); err != nil { + return WorkspaceStream{}, err + } + return WorkspaceStream{}, errors.New("the sandbox connection ended part way through its workspace stream") + } + rep, err := wstream.Write(ctx, os.DirFS(h.root), dst, wstream.Limits{}) + if err != nil { + return WorkspaceStream{}, err + } + return WorkspaceStream{Entries: rep.Entries + h.entriesFudge, Bytes: rep.Bytes, Nonce: 7}, nil +} + +func (h *streamingHost) CheckpointCommitted(_ string, nonce uint64) { + h.mu.Lock() + defer h.mu.Unlock() + h.commit = append(h.commit, nonce) +} + +func (h *streamingHost) committedNonces() []uint64 { + h.mu.Lock() + defer h.mu.Unlock() + return append([]uint64(nil), h.commit...) +} + +// checkpointScene is a microVM driver with a checkpoint configuration, a guest +// that streams a real workspace, and one created session. +func checkpointScene(t *testing.T, store checkpoint.BlobStore) (*Microvm, *SimulatedEngine, *streamingHost, string, string) { + t.Helper() + stateDir := shortTempDir(t) + if store == nil { + store = checkpoint.NewMemoryStore() + } + keys, err := checkpoint.NewStaticKeyWrapper("selfhosted/checkpoint/v1", [32]byte{7, 8, 9}) + if err != nil { + t.Fatal(err) + } + m, sim := testMicrovm(t, MicrovmOpts{ + TotalSlots: 4, + StateDir: stateDir, + Checkpoint: &CheckpointOpts{Store: store, Keys: keys, KeyRef: keys.Ref()}, + }) + host := &streamingHost{root: guestWorkspace(t)} + m.SetHost(host) + + h, err := m.Create(context.Background(), Spec{SessionID: "sess-ckpt", BootstrapToken: "token_example"}) + if err != nil { + t.Fatal(err) + } + return m, sim, host, h.ID, stateDir +} + +// TestAColdSuspendCommitsAndVerifiesTheWorkspaceCheckpoint is ADR-0003 §4.4: +// the suspend does not report success until the manifest has committed and the +// committed bytes have been read back. +func TestAColdSuspendCommitsAndVerifiesTheWorkspaceCheckpoint(t *testing.T) { + store := checkpoint.NewMemoryStore() + m, sim, host, id, _ := checkpointScene(t, store) + ctx := context.Background() + + if err := m.Suspend(ctx, id, false); err != nil { + t.Fatalf("the cold suspend failed: %v", err) + } + + // Two objects: the content and the manifest whose put-if-absent IS the + // commit. + keys := store.Keys() + if len(keys) != 2 { + t.Fatalf("the store holds %v, want a content object and a manifest", keys) + } + var manifestKey string + for _, k := range keys { + if strings.HasSuffix(k, "/manifest.json") { + manifestKey = k + } + } + if manifestKey == "" { + t.Fatalf("the store holds %v, with no manifest", keys) + } + + // The record names the checkpoint, and it is PERSISTED: a cold resume after + // a runnerd restart is exactly the case where the image may be gone and + // this is the only copy left. + rec, ok := m.diskRecord(id) + if !ok { + t.Fatal("no instance record was written") + } + if rec.CheckpointGeneration != 1 { + t.Errorf("the record says generation %d, want 1", rec.CheckpointGeneration) + } + if rec.CheckpointKey != manifestKey { + t.Errorf("the record names checkpoint %q, the store holds %q", rec.CheckpointKey, manifestKey) + } + if rec.CheckpointAt == "" { + t.Error("the record does not say when the checkpoint was taken") + } + + // The VM was terminated only after all of that. + if st, _ := sim.State(ctx, id); st != VMMStateStopped { + t.Errorf("the VM is %v after a cold suspend, want stopped", st) + } + if got := host.committedNonces(); len(got) != 1 || got[0] != 7 { + t.Errorf("the guest was told %v about its checkpoint, want the one suspend's nonce", got) + } + + // And it is restorable, which is the whole claim: verify through the store, + // with the same context the driver used. + reader, err := checkpoint.NewReader(store, mustKeys(t), checkpoint.ReaderOptions{ + Authorize: func(context.Context, checkpoint.Context, checkpoint.Manifest) error { return nil }, + }) + if err != nil { + t.Fatal(err) + } + rep, err := reader.Verify(ctx, checkpoint.Context{ + Workspace: workspaceVolume("sess-ckpt"), Session: "sess-ckpt", Generation: 1}) + if err != nil { + t.Fatalf("the committed checkpoint does not verify: %v", err) + } + // The tree, minus the Rainier-owned directory the driver excludes whatever + // the guest sent. + if rep.Entries != 4 { + t.Errorf("the checkpoint carries %d entries, want 4 (.rainier excluded)", rep.Entries) + } + if rep.Symlinks != 1 { + t.Errorf("the checkpoint carries %d symlinks, want 1", rep.Symlinks) + } +} + +func mustKeys(t *testing.T) checkpoint.Wrapper { + t.Helper() + keys, err := checkpoint.NewStaticKeyWrapper("selfhosted/checkpoint/v1", [32]byte{7, 8, 9}) + if err != nil { + t.Fatal(err) + } + return keys +} + +// TestAColdSuspendKeepsTheVMWhenTheStoreRefuses. Nothing committed, so nothing +// may be lost: the VM stays, the generation does not advance, and the error +// names the stage. +func TestAColdSuspendKeepsTheVMWhenTheStoreRefuses(t *testing.T) { + store := &refusingStore{MemoryStore: checkpoint.NewMemoryStore(), refuse: errors.New("the bucket is not reachable")} + m, sim, _, id, _ := checkpointScene(t, store) + ctx := context.Background() + + err := m.Suspend(ctx, id, false) + if err == nil { + t.Fatal("a cold suspend with no store reported success") + } + if !strings.Contains(err.Error(), "checkpoint write stage") { + t.Errorf("the refusal does not name the stage: %v", err) + } + assertVMKept(t, m, sim, id) + if keys := store.Keys(); len(keys) != 0 { + t.Errorf("a failed checkpoint left %v in the store", keys) + } +} + +// TestAColdSuspendKeepsTheVMWhenTheGuestDiesMidStream, at both of the two +// places a guest can die: inside the index, where the tree is not even +// described yet, and inside the bodies, where the reader sees a truncated tar. +// +// Both must name the STREAM and not the parser. A guest that went away is a +// sandbox problem, and reporting it as a malformed stream would send somebody +// looking at a format when the VM is what disappeared. +func TestAColdSuspendKeepsTheVMWhenTheGuestDiesMidStream(t *testing.T) { + for _, cut := range []int64{64, 1024} { + t.Run(fmt.Sprintf("after %d bytes", cut), func(t *testing.T) { + store := checkpoint.NewMemoryStore() + m, sim, host, id, _ := checkpointScene(t, store) + host.cut = cut + ctx := context.Background() + + err := m.Suspend(ctx, id, false) + if err == nil { + t.Fatal("a cold suspend whose guest died mid-stream reported success") + } + if !strings.Contains(err.Error(), "checkpoint stream stage") { + t.Errorf("the refusal does not name the stage: %v", err) + } + if !strings.Contains(err.Error(), "ended part way through") { + t.Errorf("the refusal does not name the cause: %v", err) + } + assertVMKept(t, m, sim, id) + if keys := store.Keys(); len(keys) != 0 { + t.Errorf("a failed stream left %v in the store", keys) + } + }) + } +} + +// TestAColdSuspendKeepsTheVMWhenTheHandshakeFails: the sandbox never answered, +// or predates the vocabulary entirely. +func TestAColdSuspendKeepsTheVMWhenTheHandshakeFails(t *testing.T) { + store := checkpoint.NewMemoryStore() + m, sim, host, id, _ := checkpointScene(t, store) + host.err = errors.New("the sandbox did not acknowledge the cold suspend") + ctx := context.Background() + + if err := m.Suspend(ctx, id, false); err == nil { + t.Fatal("a cold suspend whose guest never answered reported success") + } + assertVMKept(t, m, sim, id) +} + +// TestAColdSuspendKeepsTheVMWhenVerifyFails is PRD §10's restore test as a +// GATE rather than a report: the manifest committed, the bytes came back wrong, +// and that is a failed suspend. +func TestAColdSuspendKeepsTheVMWhenVerifyFails(t *testing.T) { + store := &tamperingStore{MemoryStore: checkpoint.NewMemoryStore()} + m, sim, _, id, _ := checkpointScene(t, store) + ctx := context.Background() + + err := m.Suspend(ctx, id, false) + if err == nil { + t.Fatal("a cold suspend whose checkpoint does not verify reported success") + } + if !strings.Contains(err.Error(), "checkpoint verify stage") { + t.Errorf("the refusal does not name the stage: %v", err) + } + assertVMKept(t, m, sim, id) + // The generation IS recorded, and persisted, even though the suspend + // failed. A manifest key has no attempt suffix and put-if-absent never + // overwrites, so the generation whose manifest committed is spent for good + // — a record that pretended otherwise would have every later attempt lose + // its manifest put to ErrExists, and this session could never be cold + // parked again. + rec, ok := m.diskRecord(id) + if !ok { + t.Fatal("no instance record was written") + } + if rec.CheckpointGeneration != 1 { + t.Errorf("the record says generation %d after a committed-but-unverified checkpoint, want 1", rec.CheckpointGeneration) + } +} + +// TestAColdSuspendAfterAFailedVerifyTakesTheNextGeneration is the other half of +// that rule, and the reason it matters: the session is still parkable. +func TestAColdSuspendAfterAFailedVerifyTakesTheNextGeneration(t *testing.T) { + store := &tamperingStore{MemoryStore: checkpoint.NewMemoryStore(), once: true} + m, _, _, id, _ := checkpointScene(t, store) + ctx := context.Background() + + if err := m.Suspend(ctx, id, false); err == nil { + t.Fatal("the tampered checkpoint verified") + } + if err := m.Suspend(ctx, id, false); err != nil { + t.Fatalf("the retry after a failed verify also failed: %v", err) + } + rec, _ := m.diskRecord(id) + if rec.CheckpointGeneration != 2 { + t.Fatalf("the retry took generation %d, want 2", rec.CheckpointGeneration) + } + reader, err := checkpoint.NewReader(store, mustKeys(t), checkpoint.ReaderOptions{ + Authorize: func(context.Context, checkpoint.Context, checkpoint.Manifest) error { return nil }, + }) + if err != nil { + t.Fatal(err) + } + if _, err := reader.Verify(ctx, checkpoint.Context{ + Workspace: workspaceVolume("sess-ckpt"), Session: "sess-ckpt", Generation: 2}); err != nil { + t.Errorf("the retry's checkpoint does not verify: %v", err) + } +} + +// TestAColdSuspendSkipsAGenerationTheStoreAlreadyHas is the self-healing half: +// the record can be BEHIND the store — a host that died between the commit and +// its own record write, a record recovered from an older copy — and a generation +// whose manifest exists is spent whatever this record believes. +func TestAColdSuspendSkipsAGenerationTheStoreAlreadyHas(t *testing.T) { + store := checkpoint.NewMemoryStore() + m, _, _, id, _ := checkpointScene(t, store) + ctx := context.Background() + + if err := m.Suspend(ctx, id, false); err != nil { + t.Fatal(err) + } + // The record forgets, which is what a crash between the commit and the + // record's own write leaves behind. + m.mu.Lock() + m.instances[id].CheckpointGeneration = 0 + m.mu.Unlock() + + if err := m.Suspend(ctx, id, false); err != nil { + t.Fatalf("a suspend whose record was behind the store failed: %v", err) + } + rec, _ := m.diskRecord(id) + if rec.CheckpointGeneration != 2 { + t.Fatalf("the suspend took generation %d, want it to skip the one the store already had", rec.CheckpointGeneration) + } +} + +// TestAColdSuspendAppliesTheHostsOwnExclusions. The guest applies the +// checkpoint library's exclusions; the HOST applies those plus whatever this +// deployment added, and the guest never hears about the difference. An entry +// the guest streamed and this host excludes must not reach the checkpoint. +func TestAColdSuspendAppliesTheHostsOwnExclusions(t *testing.T) { + stateDir := shortTempDir(t) + store := checkpoint.NewMemoryStore() + keys, err := checkpoint.NewStaticKeyWrapper("selfhosted/checkpoint/v1", [32]byte{7, 8, 9}) + if err != nil { + t.Fatal(err) + } + m, _ := testMicrovm(t, MicrovmOpts{ + TotalSlots: 4, + StateDir: stateDir, + Checkpoint: &CheckpointOpts{ + Store: store, Keys: keys, KeyRef: keys.Ref(), + // This host's own policy, on top of the library's. The fake guest + // streams everything, including "src". + Limits: wstream.Limits{Exclude: []string{"src"}}, + }, + }) + m.SetHost(&streamingHost{root: guestWorkspace(t)}) + ctx := context.Background() + h, err := m.Create(ctx, Spec{SessionID: "sess-ckpt", BootstrapToken: "token_example"}) + if err != nil { + t.Fatal(err) + } + if err := m.Suspend(ctx, h.ID, false); err != nil { + t.Fatal(err) + } + + reader, err := checkpoint.NewReader(store, mustKeys(t), checkpoint.ReaderOptions{ + Authorize: func(context.Context, checkpoint.Context, checkpoint.Manifest) error { return nil }, + }) + if err != nil { + t.Fatal(err) + } + rep, err := reader.Verify(ctx, checkpoint.Context{ + Workspace: workspaceVolume("sess-ckpt"), Session: "sess-ckpt", Generation: 1}) + if err != nil { + t.Fatal(err) + } + // README.md and the symlink; "src", "src/main.go" and ".rainier" are all + // excluded — two by this host's configuration, one by the library's. + if rep.Entries != 2 { + t.Errorf("the checkpoint carries %d entries, want 2 (src and .rainier excluded)", rep.Entries) + } +} + +// TestAColdSuspendRefusesCountsTheGuestDisagreesWith: the guest says how many +// entries it streamed and the host counts what it indexed, and a difference +// means the two ends do not agree about what was carried. +func TestAColdSuspendRefusesCountsTheGuestDisagreesWith(t *testing.T) { + store := checkpoint.NewMemoryStore() + m, sim, host, id, _ := checkpointScene(t, store) + host.entriesFudge = 1 + + err := m.Suspend(context.Background(), id, false) + if err == nil { + t.Fatal("a stream whose counts disagree was accepted") + } + if !strings.Contains(err.Error(), "checkpoint stream stage") { + t.Errorf("the refusal does not name the stage: %v", err) + } + assertVMKept(t, m, sim, id) +} + +// TestACreateRefusesASessionItCouldNeverCheckpoint. The checkpoint format's +// identifier rule is stricter than this driver's path-segment one, and a +// session that passed one and not the other would create happily and then fail +// every cold suspend at the configuration stage. +func TestACreateRefusesASessionItCouldNeverCheckpoint(t *testing.T) { + store := checkpoint.NewMemoryStore() + m, _, _, _, _ := checkpointScene(t, store) + ctx := context.Background() + + for _, id := range []string{"_leading-underscore", "-leading-dash", strings.Repeat("s", 130)} { + if _, err := m.Create(ctx, Spec{SessionID: id, BootstrapToken: "token_example"}); err == nil { + t.Errorf("the session id %q was created and could never have been cold suspended", id) + } else if !strings.Contains(err.Error(), "could never be cold suspended") { + t.Errorf("the refusal for %q is %v", id, err) + } + } +} + +// TestAColdSuspendWithNoCheckpointConfigurationIsUnchanged. The contract suite +// and the local dev surface have no store and no key; their cold suspend is +// what it was before this existed. +func TestAColdSuspendWithNoCheckpointConfigurationIsUnchanged(t *testing.T) { + m, sim := testMicrovm(t, MicrovmOpts{TotalSlots: 4}) + m.SetHost(&stubMicrovmHost{}) + ctx := context.Background() + h, err := m.Create(ctx, Spec{SessionID: "sess-plain"}) + if err != nil { + t.Fatal(err) + } + if m.CheckpointsColdSuspend() { + t.Error("a driver with no checkpoint configuration claims it checkpoints cold suspends") + } + if err := m.Suspend(ctx, h.ID, false); err != nil { + t.Fatalf("the cold suspend failed: %v", err) + } + if st, _ := sim.State(ctx, h.ID); st != VMMStateStopped { + t.Errorf("the VM is %v, want stopped", st) + } +} + +// TestEachColdSuspendTakesTheNextGeneration. The generation is the count of +// COMMITTED checkpoints, so the second suspend of a session's life takes 2 — +// and both are readable, because a put-if-absent never overwrites. +func TestEachColdSuspendTakesTheNextGeneration(t *testing.T) { + store := checkpoint.NewMemoryStore() + m, _, _, id, _ := checkpointScene(t, store) + ctx := context.Background() + + if err := m.Suspend(ctx, id, false); err != nil { + t.Fatalf("the first cold suspend failed: %v", err) + } + if _, err := m.Resume(ctx, id); err != nil { + t.Fatalf("the cold resume failed: %v", err) + } + if err := m.Suspend(ctx, id, false); err != nil { + t.Fatalf("the second cold suspend failed: %v", err) + } + + rec, _ := m.diskRecord(id) + if rec.CheckpointGeneration != 2 { + t.Fatalf("the record says generation %d after two cold suspends, want 2", rec.CheckpointGeneration) + } + reader, err := checkpoint.NewReader(store, mustKeys(t), checkpoint.ReaderOptions{ + Authorize: func(context.Context, checkpoint.Context, checkpoint.Manifest) error { return nil }, + }) + if err != nil { + t.Fatal(err) + } + for gen := uint64(1); gen <= 2; gen++ { + if _, err := reader.Verify(ctx, checkpoint.Context{ + Workspace: workspaceVolume("sess-ckpt"), Session: "sess-ckpt", Generation: gen}); err != nil { + t.Errorf("generation %d does not verify: %v", gen, err) + } + } +} + +// TestAColdSuspendWritesNoPlaintextWorkspaceUnderTheStateDir is the claim the +// whole streaming design exists to make: the workspace goes from a socket, +// through one frame at a time, into a store, and never onto this host's disk. +// +// It is the same walk TestMicrovmWritesNoDecryptedEnvironment does, applied to +// the other thing this host must not be holding. +func TestAColdSuspendWritesNoPlaintextWorkspaceUnderTheStateDir(t *testing.T) { + store := checkpoint.NewMemoryStore() + m, _, _, id, stateDir := checkpointScene(t, store) + + if err := m.Suspend(context.Background(), id, false); err != nil { + t.Fatalf("the cold suspend failed: %v", err) + } + + err := filepath.WalkDir(stateDir, func(path string, d fs.DirEntry, err error) error { + if err != nil { + return err + } + if d.IsDir() { + return nil + } + info, err := d.Info() + if err != nil { + return err + } + // The workspace disk image is a 10 GiB sparse file; reading it whole + // would be the test's own problem rather than the driver's. + if info.Size() > 1<<20 { + return nil + } + b, err := os.ReadFile(path) + if err != nil { + return err + } + for _, leak := range []string{workspaceMarker, "must-not-travel", wstream.Magic} { + if strings.Contains(string(b), leak) { + return fmt.Errorf("%s holds workspace plaintext (%q)", strings.TrimPrefix(path, stateDir), leak) + } + } + return nil + }) + if err != nil { + t.Fatal(err) + } + // And the checkpoint itself carries none of it in the clear either: the + // manifest is plaintext JSON by design, so it is the one object worth + // checking by hand. + for _, k := range store.Keys() { + b, _ := store.Object(k) + for _, leak := range []string{workspaceMarker, "README", "main.go", "entry"} { + if strings.Contains(string(b), leak) { + t.Errorf("the object %s carries %q in the clear", k, leak) + } + } + } +} + +// TestTwoConcurrentColdSuspendsAreRefusedRatherThanRaced: two of them would +// each read part of one stream and commit two checkpoints of two halves of a +// workspace, both verified against their own halves and both wrong. +func TestTwoConcurrentColdSuspendsAreRefusedRatherThanRaced(t *testing.T) { + store := checkpoint.NewMemoryStore() + m, _, host, id, _ := checkpointScene(t, store) + + // A host whose stream blocks until released, so the second suspend + // certainly arrives while the first is in flight. + blocking := &blockingHost{ + streamingHost: host, + gate: make(chan struct{}), + entered: make(chan struct{}, 1), + } + m.SetHost(blocking) + + first := make(chan error, 1) + go func() { first <- m.Suspend(context.Background(), id, false) }() + <-blocking.entered + + second := m.Suspend(context.Background(), id, false) + if second == nil || !strings.Contains(second.Error(), "already in flight") { + t.Errorf("the second concurrent cold suspend gave %v, want a refusal", second) + } + close(blocking.gate) + if err := <-first; err != nil { + t.Fatalf("the first cold suspend failed: %v", err) + } +} + +type blockingHost struct { + *streamingHost + gate chan struct{} + entered chan struct{} +} + +func (b *blockingHost) StreamWorkspace(ctx context.Context, sessionID string, dst io.Writer) (WorkspaceStream, error) { + select { + case b.entered <- struct{}{}: + default: + } + <-b.gate + return b.streamingHost.StreamWorkspace(ctx, sessionID, dst) +} + +// assertVMKept is the rule every failure path above shares: a suspend that did +// not produce a checkpoint did not stop the VM either. +func assertVMKept(t *testing.T, m *Microvm, sim *SimulatedEngine, id string) { + t.Helper() + st, err := sim.State(context.Background(), id) + if err != nil { + t.Fatalf("asking the engine about %s: %v", id, err) + } + if st != VMMStateRunning { + t.Errorf("the VM is %v after a failed cold suspend, want it still running", st) + } + rec, ok := m.diskRecord(id) + if !ok { + t.Fatal("the instance record is gone after a failed cold suspend") + } + if rec.State != StateRunning { + t.Errorf("the record says %v after a failed cold suspend, want running", rec.State) + } + // The session's own rootfs is still there: a failed cold suspend has not + // begun tearing anything down. + path, err := m.sessionRootfsPath(id) + if err != nil { + t.Fatal(err) + } + if _, err := os.Stat(path); err != nil { + t.Errorf("a failed cold suspend removed the session's root filesystem: %v", err) + } +} + +// refusingStore is a blob store that will not take an object — a bucket that is +// unreachable, a credential that expired, a quota that is full. +type refusingStore struct { + *checkpoint.MemoryStore + refuse error +} + +func (r *refusingStore) PutIfAbsent(context.Context, string, func(io.Writer) error) error { + return r.refuse +} + +// tamperingStore commits what it is given and then changes it, which is a +// checkpoint that committed and cannot be read back: bit rot, a bug, or an +// attacker with write access to the bucket and no key. +type tamperingStore struct { + *checkpoint.MemoryStore + // once stops after the first content object, so a test can show that the + // NEXT attempt succeeds. + once bool + tampered bool +} + +func (s *tamperingStore) PutIfAbsent(ctx context.Context, key string, write func(io.Writer) error) error { + if err := s.MemoryStore.PutIfAbsent(ctx, key, write); err != nil { + return err + } + if s.once && s.tampered { + return nil + } + if strings.Contains(key, "/content.") { + s.tampered = true + b, ok := s.MemoryStore.Object(key) + if ok && len(b) > 0 { + b[len(b)/2] ^= 0xff + // Put it back under a second key-free path: MemoryStore's own + // overwrite is unexported, so the tamper is done by deleting and + // re-putting, which is exactly what an attacker with write access + // to the bucket has. + _ = s.MemoryStore.Delete(ctx, key) + _ = s.MemoryStore.PutIfAbsent(ctx, key, func(w io.Writer) error { + _, err := w.Write(b) + return err + }) + } + } + return nil +} diff --git a/internal/driver/microvm_kvm_test.go b/internal/driver/microvm_kvm_test.go index a5d0ec1..60ae1c5 100644 --- a/internal/driver/microvm_kvm_test.go +++ b/internal/driver/microvm_kvm_test.go @@ -81,6 +81,7 @@ import ( "encoding/json" "errors" "fmt" + "io" "os" "os/exec" "path/filepath" @@ -432,7 +433,27 @@ type kvmGuest struct { exchangedToken string exchanged chan struct{} flushes map[uint64]chan struct{} - dead chan struct{} + // suspends is the cold-suspend handshakes in flight on this guest, keyed by + // nonce — which is also the id of the workspace stream that answers each + // one. At most one is ever live; the map is what keeps a late answer from + // a suspend that gave up out of the next one's, exactly as the runner's own + // nonce matching does. + suspends map[uint64]*kvmSuspend + dead chan struct{} +} + +// kvmSuspend is one cold suspend's three answers on this harness: the +// acknowledgement, the workspace stream, and the end marker. The runner's own +// version of this is internal/runnerd/workspacestream.go; this one is the same +// shape with the budgets the harness already uses. +type kvmSuspend struct { + dst io.Writer + n int64 + werr error + ack chan struct{} + end chan relay.ControlEvent + ready chan struct{} + once sync.Once } func newKVMRunner(connectWait time.Duration) *kvmRunner { @@ -460,6 +481,7 @@ func (r *kvmRunner) GuestConnected(sessionID string, conn relay.Conn) { at: time.Now(), exchanged: make(chan struct{}), flushes: map[uint64]chan struct{}{}, + suspends: map[uint64]*kvmSuspend{}, dead: make(chan struct{}), } r.mu.Lock() @@ -607,7 +629,29 @@ func (r *kvmRunner) readGuest(sessionID string, g *kvmGuest) { return } f, err := relay.Decode(raw) - if err != nil || f.Type != relay.FrameControl { + if err != nil { + continue + } + if f.Type == relay.FrameStream { + // One chunk of a workspace. Written straight through, the way the + // runner writes it into the checkpoint writer's pipe: a harness + // that buffered the tree would be measuring something production + // does not do. + g.mu.Lock() + s := g.suspends[f.AttachID] + g.mu.Unlock() + if s != nil { + n, werr := s.dst.Write(f.Payload) + g.mu.Lock() + s.n += int64(n) + if werr != nil && s.werr == nil { + s.werr = werr + } + g.mu.Unlock() + } + continue + } + if f.Type != relay.FrameControl { continue } var ev relay.ControlEvent @@ -625,10 +669,117 @@ func (r *kvmRunner) readGuest(sessionID string, g *kvmGuest) { if ok { close(done) } + case ev.Kind == relay.KindSuspendAck, ev.Kind == relay.KindWorkspaceEnd, ev.Kind == relay.KindSuspendReady: + g.mu.Lock() + s := g.suspends[ev.ID] + g.mu.Unlock() + if s == nil { + continue + } + switch ev.Kind { + case relay.KindSuspendAck: + s.once.Do(func() { close(s.ack) }) + case relay.KindWorkspaceEnd: + select { + case s.end <- ev: + default: + } + case relay.KindSuspendReady: + close(s.ready) + } } } } +// StreamWorkspace is the harness's cold-suspend handshake: tell the guest, read +// the workspace it streams back into dst, and return when its end marker says +// the tree is complete. +// +// It is the same shape as (*runnerd.Server).StreamWorkspace, with this +// harness's single patience budget in place of the runner's three. It exists so +// that a run on a real KVM host exercises the whole barrier — a real guest, a +// real workspace, a real ext4 underneath it — which is the one thing no fake +// on a developer machine can evidence. +func (r *kvmRunner) StreamWorkspace(ctx context.Context, sessionID string, dst io.Writer) (WorkspaceStream, error) { + var none WorkspaceStream + g, err := r.awaitGuest(ctx, sessionID, r.connectWait) + if err != nil { + return none, fmt.Errorf("session %s has no sandbox connection to stream its workspace over: %w", sessionID, err) + } + + r.mu.Lock() + r.nonce++ + nonce := r.nonce + r.mu.Unlock() + + s := &kvmSuspend{ + dst: dst, + ack: make(chan struct{}), + end: make(chan relay.ControlEvent, 1), + ready: make(chan struct{}), + } + g.mu.Lock() + g.suspends[nonce] = s + g.mu.Unlock() + defer func() { + g.mu.Lock() + delete(g.suspends, nonce) + g.mu.Unlock() + }() + + if err := g.send(relay.ControlEvent{Kind: relay.KindSuspending, ID: nonce, Cold: true}); err != nil { + return none, fmt.Errorf("asking session %s to cold suspend: %w", sessionID, err) + } + + timer := time.NewTimer(r.connectWait) + defer timer.Stop() + var end relay.ControlEvent + select { + case end = <-s.end: + case <-g.dead: + return none, fmt.Errorf("session %s lost its sandbox connection part way through its workspace stream", sessionID) + case <-ctx.Done(): + return none, fmt.Errorf("streaming session %s's workspace: %w", sessionID, ctx.Err()) + case <-timer.C: + return none, fmt.Errorf("session %s did not finish streaming its workspace within %s", sessionID, r.connectWait) + } + + g.mu.Lock() + received, werr := s.n, s.werr + g.mu.Unlock() + switch { + case werr != nil: + return none, fmt.Errorf("session %s: the workspace stream could not be taken: %w", sessionID, werr) + case !end.OK: + return none, fmt.Errorf("session %s could not stream its workspace (stage %s, %d entries and %d bytes in): %s", + sessionID, end.Stage, end.Entries, end.Bytes, end.Tail) + case end.Bytes != received: + return none, fmt.Errorf("session %s streamed %d bytes of workspace and %d arrived", sessionID, end.Bytes, received) + } + + ready := time.NewTimer(r.connectWait) + defer ready.Stop() + select { + case <-s.ready: + case <-g.dead: + case <-ready.C: + } + return WorkspaceStream{Entries: end.Entries, Bytes: received, Nonce: nonce}, nil +} + +// CheckpointCommitted is the harness's half of the "your work is durable" +// event. The guest logs it and answers nothing, so there is nothing for this to +// wait on. +func (r *kvmRunner) CheckpointCommitted(sessionID string, nonce uint64) { + r.mu.Lock() + g := r.guests[sessionID] + r.mu.Unlock() + if g == nil { + return + } + _ = g.send(relay.ControlEvent{Kind: relay.KindCheckpointCommitted, ID: nonce}) +} + // answerSecrets plays the control plane's half of the bootstrap exchange. // // The token the guest sends is recorded BEFORE the answer, and it is recorded diff --git a/internal/driver/microvm_restore_test.go b/internal/driver/microvm_restore_test.go new file mode 100644 index 0000000..93371b7 --- /dev/null +++ b/internal/driver/microvm_restore_test.go @@ -0,0 +1,792 @@ +// internal/driver/microvm_restore_test.go +package driver + +import ( + "archive/tar" + "bytes" + "context" + "errors" + "io" + "os" + "path/filepath" + "strings" + "sync" + "testing" + + "github.com/tokencanopy/rainier/checkpoint" +) + +// The deep-dormant resume (ADR-0003 §2.3): the workspace image has been +// deleted and the checkpoint is the only copy of the session's work left. +// +// The rule every test here is about: the restore happens when the image is +// ABSENT and never otherwise, it leaves nothing behind, and a resume that +// cannot produce a workspace refuses rather than booting an empty one. + +// restoredTree reads back what the simulated formatter recorded, which is the +// tree the restore actually handed to mkfs. +func restoredTree(t *testing.T, imagePath string) map[string]string { + t.Helper() + b, err := os.ReadFile(imagePath) + if err != nil { + t.Fatalf("reading the rebuilt workspace image: %v", err) + } + out := map[string]string{} + tr := tar.NewReader(bytes.NewReader(b)) + for { + hdr, err := tr.Next() + if errors.Is(err, io.EOF) { + return out + } + if err != nil { + t.Fatalf("reading the rebuilt workspace image: %v", err) + } + body, err := io.ReadAll(tr) + if err != nil { + t.Fatal(err) + } + switch hdr.Typeflag { + case tar.TypeSymlink: + out[hdr.Name] = "-> " + hdr.Linkname + default: + out[hdr.Name] = string(body) + } + } +} + +// suspendedWithCheckpoint is a session that has been cold suspended, so its +// checkpoint is committed and verified, with its workspace image then REMOVED +// the way a deep-dormant transition removes it. +func suspendedWithCheckpoint(t *testing.T) (*Microvm, *streamingHost, string, string) { + t.Helper() + m, _, host, id, stateDir := checkpointScene(t, nil) + if err := m.Suspend(context.Background(), id, false); err != nil { + t.Fatalf("the cold suspend failed: %v", err) + } + return m, host, id, stateDir +} + +// TestAColdResumeRestoresTheWorkspaceWhenTheImageIsGone is the whole feature: +// the disk is gone, the checkpoint is not, and the session comes back with its +// files. +func TestAColdResumeRestoresTheWorkspaceWhenTheImageIsGone(t *testing.T) { + m, _, id, _ := suspendedWithCheckpoint(t) + ctx := context.Background() + + if err := m.RemoveWorkspace(ctx, "sess-ckpt"); err != nil { + t.Fatalf("removing the workspace image: %v", err) + } + disk, err := m.workspaceDiskPath("sess-ckpt") + if err != nil { + t.Fatal(err) + } + if _, err := os.Stat(disk); !errors.Is(err, os.ErrNotExist) { + t.Fatalf("the workspace image is still there: %v", err) + } + + restarted, err := m.Resume(ctx, id) + if err != nil { + t.Fatalf("the cold resume failed: %v", err) + } + if !restarted { + t.Error("the cold resume did not report a restart") + } + + tree := restoredTree(t, disk) + want := map[string]string{ + "README.md": "hello " + workspaceMarker, + "src": "", + "src/main.go": "package main // " + workspaceMarker, + "entry": "-> src/main.go", + } + if len(tree) != len(want) { + t.Fatalf("the rebuilt workspace holds %v, want %v", tree, want) + } + for name, body := range want { + if tree[name] != body { + t.Errorf("the rebuilt workspace holds %q at %s, want %q", tree[name], name, body) + } + } + // Nothing Rainier owns came back, because nothing Rainier owns went in. + for name := range tree { + if strings.HasPrefix(name, ".rainier") { + t.Errorf("the rebuilt workspace holds %s", name) + } + } + assertNoScratch(t, m, id) +} + +// TestAColdResumeLeavesAWorkspaceThatIsStillThereAlone. The ordinary cold +// resume is unchanged: a parked session's disk is right where it was, and +// restoring over it would replace a newer filesystem with an older copy of it. +func TestAColdResumeLeavesAWorkspaceThatIsStillThereAlone(t *testing.T) { + m, _, id, _ := suspendedWithCheckpoint(t) + ctx := context.Background() + + disk, err := m.workspaceDiskPath("sess-ckpt") + if err != nil { + t.Fatal(err) + } + before, err := os.Stat(disk) + if err != nil { + t.Fatalf("the parked session's workspace image is missing: %v", err) + } + + if _, err := m.Resume(ctx, id); err != nil { + t.Fatalf("the cold resume failed: %v", err) + } + + after, err := os.Stat(disk) + if err != nil { + t.Fatalf("the workspace image went away: %v", err) + } + if after.ModTime() != before.ModTime() || after.Size() != before.Size() { + t.Error("the cold resume rebuilt a workspace image that was still there") + } + assertNoScratch(t, m, id) +} + +// TestAColdResumeRefusesWhenThereIsNoCheckpoint. An empty workspace that looks +// like a successful resume is the failure a person discovers by finding their +// work missing. +func TestAColdResumeRefusesWhenThereIsNoCheckpoint(t *testing.T) { + store := checkpoint.NewMemoryStore() + m, _, _, id, _ := checkpointScene(t, store) + ctx := context.Background() + + if err := m.Suspend(ctx, id, false); err != nil { + t.Fatal(err) + } + // The checkpoint the suspend took is forgotten, standing in for a session + // whose image went away without one ever having been committed. + m.mu.Lock() + m.instances[id].CheckpointGeneration = 0 + m.mu.Unlock() + if err := m.RemoveWorkspace(ctx, "sess-ckpt"); err != nil { + t.Fatal(err) + } + + _, err := m.Resume(ctx, id) + if err == nil { + t.Fatal("a resume with no workspace and no checkpoint reported success") + } + if !strings.Contains(err.Error(), "nothing to restore") { + t.Errorf("the refusal is %v", err) + } +} + +// TestAColdResumeRefusesATamperedCheckpointAndLeavesNothingBehind: the +// checkpoint committed, somebody with write access to the store changed it, and +// what comes back is not the tenant's work. No image, no scratch directory, no +// VM. +func TestAColdResumeRefusesATamperedCheckpointAndLeavesNothingBehind(t *testing.T) { + store := checkpoint.NewMemoryStore() + m, sim, _, id, _ := checkpointScene(t, store) + ctx := context.Background() + + if err := m.Suspend(ctx, id, false); err != nil { + t.Fatal(err) + } + if err := m.RemoveWorkspace(ctx, "sess-ckpt"); err != nil { + t.Fatal(err) + } + // One flipped bit in the content object, which is exactly what the frame + // AEAD exists to catch. + for _, k := range store.Keys() { + if !strings.Contains(k, "/content.") { + continue + } + b, _ := store.Object(k) + b[len(b)/2] ^= 0xff + if err := store.Delete(ctx, k); err != nil { + t.Fatal(err) + } + if err := store.PutIfAbsent(ctx, k, func(w io.Writer) error { + _, err := w.Write(b) + return err + }); err != nil { + t.Fatal(err) + } + } + + _, err := m.Resume(ctx, id) + if err == nil { + t.Fatal("a tampered checkpoint was restored") + } + if !strings.Contains(err.Error(), "checkpoint restore stage") { + t.Errorf("the refusal does not name the stage: %v", err) + } + disk, err := m.workspaceDiskPath("sess-ckpt") + if err != nil { + t.Fatal(err) + } + if _, err := os.Stat(disk); !errors.Is(err, os.ErrNotExist) { + t.Error("a refused restore left a workspace image behind") + } + assertNoScratch(t, m, id) + if st, _ := sim.State(ctx, id); st == VMMStateRunning { + t.Error("a refused restore started the VM anyway") + } +} + +// TestAFailedMkfsLeavesNoHalfBuiltImage. A file that exists is a file the next +// create finds with Stat and hands to a guest as a filesystem, so a mkfs that +// failed takes its image with it. +func TestAFailedMkfsLeavesNoHalfBuiltImage(t *testing.T) { + m, _, id, _ := suspendedWithCheckpoint(t) + ctx := context.Background() + if err := m.RemoveWorkspace(ctx, "sess-ckpt"); err != nil { + t.Fatal(err) + } + m.format = failingFormatter{} + + _, err := m.Resume(ctx, id) + if err == nil { + t.Fatal("a resume whose mkfs failed reported success") + } + if !strings.Contains(err.Error(), "checkpoint mkfs stage") { + t.Errorf("the refusal does not name the stage: %v", err) + } + disk, err := m.workspaceDiskPath("sess-ckpt") + if err != nil { + t.Fatal(err) + } + if _, err := os.Stat(disk); !errors.Is(err, os.ErrNotExist) { + t.Error("a failed mkfs left a half-built workspace image behind") + } + assertNoScratch(t, m, id) +} + +// TestARestoreGivesTheTreeToTheGuestsUser. A checkpoint records modes and not +// owners, so a restored tree belongs to whoever restored it — root, on a real +// host — and the guest's agent runs as the session image's user. Without this +// the session comes back unable to write its own files. +// +// The uid under test is this process's own, because a test that could give a +// file away would be a test running as root. +func TestARestoreGivesTheTreeToTheGuestsUser(t *testing.T) { + root := t.TempDir() + if err := os.WriteFile(filepath.Join(root, "a.txt"), []byte("alpha"), 0o644); err != nil { + t.Fatal(err) + } + if err := chownTree(root, os.Getuid(), os.Getgid()); err != nil { + t.Fatalf("chowning a restored tree: %v", err) + } + // And 0,0 is "leave it", which is what a host whose guest runs as root + // wants and what keeps this path out of a test's way. + if err := chownTree(root, 0, 0); err != nil { + t.Fatalf("chowning to 0:0 should be a no-op: %v", err) + } + if err := chownTree(filepath.Join(root, "nope"), os.Getuid(), os.Getgid()); err == nil { + t.Error("chowning a tree that is not there reported success") + } +} + +// TestARestoreThatCannotGiveTheTreeAwayFailsClosed is the ownership step at its +// CALL SITE rather than as a function: a restore that could not hand the files +// to the guest's user would come back with a workspace the agent cannot write, +// which is a session that looks resumed and is not. It refuses instead, and +// leaves no image behind. +func TestARestoreThatCannotGiveTheTreeAwayFailsClosed(t *testing.T) { + if os.Getuid() == 0 { + t.Skip("running as root, which can give a file to anybody; the refusal under test is EPERM") + } + store := checkpoint.NewMemoryStore() + keys, err := checkpoint.NewStaticKeyWrapper("selfhosted/checkpoint/v1", [32]byte{7, 8, 9}) + if err != nil { + t.Fatal(err) + } + stateDir := shortTempDir(t) + m, _ := testMicrovm(t, MicrovmOpts{ + TotalSlots: 4, + StateDir: stateDir, + Checkpoint: &CheckpointOpts{ + Store: store, Keys: keys, KeyRef: keys.Ref(), + // A uid this unprivileged test process is not and cannot give a + // file to. In production it is the session image's own user; what + // is under test is what happens when the chown is refused. + OwnerUID: otherUID(), OwnerGID: otherUID(), + }, + }) + m.SetHost(&streamingHost{root: guestWorkspace(t)}) + ctx := context.Background() + h, err := m.Create(ctx, Spec{SessionID: "sess-ckpt", BootstrapToken: "token_example"}) + if err != nil { + t.Fatal(err) + } + if err := m.Suspend(ctx, h.ID, false); err != nil { + t.Fatal(err) + } + if err := m.RemoveWorkspace(ctx, "sess-ckpt"); err != nil { + t.Fatal(err) + } + + _, err = m.Resume(ctx, h.ID) + if err == nil { + t.Fatal("a restore that could not give the tree to the guest's user reported success") + } + if !strings.Contains(err.Error(), "checkpoint owner stage") { + t.Fatalf("the refusal does not name the stage: %v", err) + } + // And it says nothing about which file it was on: no name from inside the + // workspace, and no path at all — the scratch directory's own path would + // name the instance and then the tenant's tree. + for _, leak := range []string{"README", "main.go", "/"} { + if strings.Contains(err.Error(), leak) { + t.Errorf("the refusal carries %q, which is a path or a name from inside the workspace: %v", leak, err) + } + } + disk, err := m.workspaceDiskPath("sess-ckpt") + if err != nil { + t.Fatal(err) + } + if _, err := os.Stat(disk); !errors.Is(err, os.ErrNotExist) { + t.Error("a refused restore left a workspace image behind") + } + assertNoScratch(t, m, h.ID) +} + +// otherUID is a uid this process is not, so that an Lchown to it is refused. +func otherUID() int { + if os.Getuid() == 65534 { + return 65533 + } + return 65534 +} + +// TestAHalfBuiltImageIsNeverAtTheFinalPath. The trigger for restoring at all is +// the image's ABSENCE, so a 10 GiB file created at the final path and then +// populated by a mkfs measured in minutes is, for those minutes, a file a crash +// leaves behind — and the next resume would see it, skip the restore, and hand +// the guest an unformatted disk its own /init would helpfully format EMPTY. +func TestAHalfBuiltImageIsNeverAtTheFinalPath(t *testing.T) { + m, _, id, _ := suspendedWithCheckpoint(t) + ctx := context.Background() + if err := m.RemoveWorkspace(ctx, "sess-ckpt"); err != nil { + t.Fatal(err) + } + disk, err := m.workspaceDiskPath("sess-ckpt") + if err != nil { + t.Fatal(err) + } + // A formatter that looks at the world from inside mkfs: the final path must + // not exist while the filesystem is still being built. + watching := &watchingFormatter{final: disk} + m.format = watching + + if _, err := m.Resume(ctx, id); err != nil { + t.Fatalf("the cold resume failed: %v", err) + } + if watching.finalExisted { + t.Error("the workspace image was at its final path while mkfs was still populating it") + } + if _, err := os.Stat(disk); err != nil { + t.Errorf("the finished image is not at its final path: %v", err) + } + // And a mkfs that fails leaves neither the final name nor the partial one. + // The session is parked again first: a Resume of a RUNNING session restarts + // nothing and would prove nothing here. + if err := m.Suspend(ctx, id, false); err != nil { + t.Fatal(err) + } + if err := m.RemoveWorkspace(ctx, "sess-ckpt"); err != nil { + t.Fatal(err) + } + m.format = failingFormatter{} + if _, err := m.Resume(ctx, id); err == nil { + t.Fatal("a failed mkfs reported success") + } + for _, p := range []string{disk, disk + ".partial"} { + if _, err := os.Stat(p); !errors.Is(err, os.ErrNotExist) { + t.Errorf("a failed mkfs left %s behind", filepath.Base(p)) + } + } +} + +// watchingFormatter records whether the FINAL image path existed at the moment +// the filesystem was being written. +type watchingFormatter struct { + SimulatedDiskFormatter + final string + finalExisted bool +} + +func (w *watchingFormatter) FormatFromDir(path, dir string) error { + if _, err := os.Stat(w.final); err == nil { + w.finalExisted = true + } + return w.SimulatedDiskFormatter.FormatFromDir(path, dir) +} + +// TestRemoveWorkspaceKeepsTheCheckpoint. Deleting a checkpoint is a retention +// decision, and this driver does not have the policy: it does not know how old +// the checkpoint is, whether a deep-dormant tier is relying on it, or whether +// a restore is in flight somewhere else. RemoveWorkspace removes the IMAGE. +func TestRemoveWorkspaceKeepsTheCheckpoint(t *testing.T) { + store := checkpoint.NewMemoryStore() + m, _, _, id, _ := checkpointScene(t, store) + ctx := context.Background() + if err := m.Suspend(ctx, id, false); err != nil { + t.Fatal(err) + } + before := store.Keys() + + if err := m.RemoveWorkspace(ctx, "sess-ckpt"); err != nil { + t.Fatal(err) + } + + if after := store.Keys(); len(after) != len(before) { + t.Errorf("RemoveWorkspace left %v in the store, want %v", after, before) + } + rec, _ := m.diskRecord(id) + if rec.CheckpointGeneration != 1 { + t.Errorf("RemoveWorkspace forgot the checkpoint generation (%d)", rec.CheckpointGeneration) + } +} + +// TestAResumeWithNoStoreSaysSoRatherThanBootingEmpty: a runner with no +// checkpoint configuration cannot restore anything, and a session whose image +// is gone is not one it can bring back. +func TestAResumeWithNoStoreSaysSoRatherThanBootingEmpty(t *testing.T) { + m, _ := testMicrovm(t, MicrovmOpts{TotalSlots: 4}) + m.SetHost(&stubMicrovmHost{}) + ctx := context.Background() + h, err := m.Create(ctx, Spec{SessionID: "sess-nostore"}) + if err != nil { + t.Fatal(err) + } + if err := m.Suspend(ctx, h.ID, false); err != nil { + t.Fatal(err) + } + if err := m.RemoveWorkspace(ctx, "sess-nostore"); err != nil { + t.Fatal(err) + } + + if _, err := m.Resume(ctx, h.ID); err == nil { + t.Fatal("a resume with no workspace and no store reported success") + } else if !strings.Contains(err.Error(), "no checkpoint store") { + t.Errorf("the refusal is %v", err) + } +} + +// assertNoScratch is the rule every path above shares: the directory a restore +// unpacks a tenant's files into does not outlive the restore. +func assertNoScratch(t *testing.T, m *Microvm, id string) { + t.Helper() + dir := filepath.Join(m.opts.StateDir, "restore", id) + if _, err := os.Stat(dir); !errors.Is(err, os.ErrNotExist) { + t.Errorf("the restore scratch directory %s is still there (%v)", dir, err) + } +} + +// TestTheScratchDirectoryIsPrivateAndReplaced. +func TestTheScratchDirectoryIsPrivateAndReplaced(t *testing.T) { + m, _ := testMicrovm(t, MicrovmOpts{TotalSlots: 4}) + dir, err := m.scratchDir("i-scratch") + if err != nil { + t.Fatal(err) + } + info, err := os.Stat(dir) + if err != nil { + t.Fatal(err) + } + if info.Mode().Perm() != 0o700 { + t.Errorf("the scratch directory is mode %v, want 0700", info.Mode().Perm()) + } + // A tree a previous attempt left behind is replaced rather than restored + // into: the library refuses a non-empty target, and a half-restored tree is + // not one to build a filesystem from. + if err := os.WriteFile(filepath.Join(dir, "leftover"), []byte("x"), 0o600); err != nil { + t.Fatal(err) + } + again, err := m.scratchDir("i-scratch") + if err != nil { + t.Fatal(err) + } + entries, err := os.ReadDir(again) + if err != nil { + t.Fatal(err) + } + if len(entries) != 0 { + t.Errorf("the scratch directory still holds %d entries from a previous attempt", len(entries)) + } + if _, err := m.scratchDir("../escape"); err == nil { + t.Error("an instance id that is not a path segment was accepted") + } +} + +// recordingFormatter proves the restore path is taken only when it should be. +type recordingFormatter struct { + SimulatedDiskFormatter + mu sync.Mutex + dirs []string + paths []string +} + +func (r *recordingFormatter) FormatFromDir(path, dir string) error { + r.mu.Lock() + r.dirs = append(r.dirs, dir) + r.paths = append(r.paths, path) + r.mu.Unlock() + return r.SimulatedDiskFormatter.FormatFromDir(path, dir) +} + +func (r *recordingFormatter) calls() int { + r.mu.Lock() + defer r.mu.Unlock() + return len(r.dirs) +} + +// TestTheRestorePathIsTakenOnlyWhenTheImageIsAbsent, counted at the formatter +// rather than inferred from a file: the assertion is about which code ran. +func TestTheRestorePathIsTakenOnlyWhenTheImageIsAbsent(t *testing.T) { + format := &recordingFormatter{} + stateDir := shortTempDir(t) + keys, err := checkpoint.NewStaticKeyWrapper("selfhosted/checkpoint/v1", [32]byte{7, 8, 9}) + if err != nil { + t.Fatal(err) + } + m, _ := testMicrovm(t, MicrovmOpts{ + TotalSlots: 4, + StateDir: stateDir, + Format: format, + Checkpoint: &CheckpointOpts{Store: checkpoint.NewMemoryStore(), Keys: keys, KeyRef: keys.Ref()}, + }) + m.SetHost(&streamingHost{root: guestWorkspace(t)}) + ctx := context.Background() + h, err := m.Create(ctx, Spec{SessionID: "sess-ckpt", BootstrapToken: "token_example"}) + if err != nil { + t.Fatal(err) + } + + if err := m.Suspend(ctx, h.ID, false); err != nil { + t.Fatal(err) + } + if _, err := m.Resume(ctx, h.ID); err != nil { + t.Fatal(err) + } + if format.calls() != 0 { + t.Fatalf("a resume with its workspace image still there rebuilt it %d time(s)", format.calls()) + } + + if err := m.Suspend(ctx, h.ID, false); err != nil { + t.Fatal(err) + } + if err := m.RemoveWorkspace(ctx, "sess-ckpt"); err != nil { + t.Fatal(err) + } + if _, err := m.Resume(ctx, h.ID); err != nil { + t.Fatal(err) + } + if format.calls() != 1 { + t.Fatalf("a resume with no workspace image rebuilt it %d time(s), want 1", format.calls()) + } +} + +// restartedOver is the same host coming back: a second driver over the state +// directory a previous one left, which recovers the record off disk and has no +// guest configuration behind it (ADR-0003 §2.7 item 1). +func restartedOver(t *testing.T, stateDir string, ckpt *CheckpointOpts) (*Microvm, *recordingFormatter) { + t.Helper() + format := &recordingFormatter{} + m, _ := testMicrovm(t, MicrovmOpts{ + TotalSlots: 4, + StateDir: stateDir, + Format: format, + Checkpoint: ckpt, + }) + m.SetHost(&streamingHost{root: guestWorkspace(t)}) + return m, format +} + +// TestAColdResumeAfterARestartSaysWhichThingItIsWaitingFor is the precondition +// the note's §7 now states, held as a test rather than as prose. +// +// A session dormant long enough for the deep-dormant tier to have deleted its +// workspace image has almost certainly outlived the runnerd that created it — +// so the restore path is behind the refusal that a recovered record has no +// guest configuration, and no ordering of the two changes that: this driver +// cannot invent a session id, a proxy or a boot chain, and booting a guest that +// was never told what it is is the outcome the refusal exists for. +// +// What it CAN do is say which of the two things it is waiting for, and spend +// nothing finding out. So: the refusal names both, no restore runs, and no +// tenant tree is unpacked onto this host for a resume that is certain to fail. +func TestAColdResumeAfterARestartSaysWhichThingItIsWaitingFor(t *testing.T) { + keys, err := checkpoint.NewStaticKeyWrapper("selfhosted/checkpoint/v1", [32]byte{7, 8, 9}) + if err != nil { + t.Fatal(err) + } + ckpt := &CheckpointOpts{Store: checkpoint.NewMemoryStore(), Keys: keys, KeyRef: keys.Ref()} + stateDir := shortTempDir(t) + m, _ := testMicrovm(t, MicrovmOpts{TotalSlots: 4, StateDir: stateDir, Checkpoint: ckpt}) + m.SetHost(&streamingHost{root: guestWorkspace(t)}) + ctx := context.Background() + h, err := m.Create(ctx, Spec{SessionID: "sess-ckpt", BootstrapToken: "token_example"}) + if err != nil { + t.Fatal(err) + } + if err := m.Suspend(ctx, h.ID, false); err != nil { + t.Fatalf("the cold suspend failed: %v", err) + } + // The deep-dormant transition: the image goes, the checkpoint stays. + if err := m.RemoveWorkspace(ctx, "sess-ckpt"); err != nil { + t.Fatal(err) + } + + again, format := restartedOver(t, stateDir, ckpt) + _, err = again.Resume(ctx, h.ID) + if err == nil { + t.Fatal("a cold resume of a recovered record reported success") + } + for _, want := range []string{"did not survive a runnerd restart", "generation 1"} { + if !strings.Contains(err.Error(), want) { + t.Errorf("the refusal does not say %q: %v", want, err) + } + } + // Nothing was spent on it: no mkfs, no scratch tree, and the image is + // still absent rather than half built. + if format.calls() != 0 { + t.Errorf("the refused resume ran %d restore(s)", format.calls()) + } + assertNoScratch(t, again, h.ID) + disk, err := again.workspaceDiskPath("sess-ckpt") + if err != nil { + t.Fatal(err) + } + if _, err := os.Stat(disk); !errors.Is(err, os.ErrNotExist) { + t.Errorf("the refused resume left something at the workspace image path (%v)", err) + } +} + +// TestAColdResumeAfterARestartWithNoCheckpointSaysThatToo is the same refusal +// for the session nothing can recover: the image is gone and no checkpoint of +// it was ever committed. Said plainly, because the alternative is a person +// discovering it by finding their work missing. +func TestAColdResumeAfterARestartWithNoCheckpointSaysThatToo(t *testing.T) { + stateDir := shortTempDir(t) + m, _ := testMicrovm(t, MicrovmOpts{TotalSlots: 4, StateDir: stateDir}) + m.SetHost(&stubMicrovmHost{}) + ctx := context.Background() + h, err := m.Create(ctx, Spec{SessionID: "sess-nockpt"}) + if err != nil { + t.Fatal(err) + } + if err := m.Suspend(ctx, h.ID, false); err != nil { + t.Fatal(err) + } + if err := m.RemoveWorkspace(ctx, "sess-nockpt"); err != nil { + t.Fatal(err) + } + + keys, err := checkpoint.NewStaticKeyWrapper("selfhosted/checkpoint/v1", [32]byte{7, 8, 9}) + if err != nil { + t.Fatal(err) + } + again, format := restartedOver(t, stateDir, &CheckpointOpts{ + Store: checkpoint.NewMemoryStore(), Keys: keys, KeyRef: keys.Ref(), + }) + _, err = again.Resume(ctx, h.ID) + if err == nil { + t.Fatal("a cold resume of a recovered record with no workspace reported success") + } + if !strings.Contains(err.Error(), "no checkpoint of it was ever committed") { + t.Errorf("the refusal does not say the work is gone: %v", err) + } + if format.calls() != 0 { + t.Errorf("the refused resume ran %d restore(s)", format.calls()) + } +} + +// TestAPreviousRunsRestoreScratchIsRemovedAtStartup is reclaimRestoreScratch, +// which had no test: what is under /restore is a tenant's whole +// workspace in plaintext, and the note's §7 is only allowed to call the scratch +// directory "no new exposure" because nothing leaves one behind. A host that +// was powered off mid-restore leaves one behind, so the next start removes it. +func TestAPreviousRunsRestoreScratchIsRemovedAtStartup(t *testing.T) { + stateDir := shortTempDir(t) + planted := filepath.Join(stateDir, "restore", "i-old") + if err := os.MkdirAll(filepath.Join(planted, "src"), 0o700); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(filepath.Join(planted, "src", "f"), []byte(workspaceMarker), 0o600); err != nil { + t.Fatal(err) + } + + m, _ := testMicrovm(t, MicrovmOpts{TotalSlots: 4, StateDir: stateDir}) + + if _, err := os.Stat(planted); !errors.Is(err, os.ErrNotExist) { + t.Fatalf("a previous run's restore scratch tree survived the start (%v)", err) + } + // And nothing of it is anywhere else under the state directory either. + assertNoWorkspacePlaintext(t, m.opts.StateDir) +} + +// assertNoWorkspacePlaintext walks a state directory for the fixture +// workspace's marker: if it is there, this host has a tenant's files on its own +// disk in the clear. +func assertNoWorkspacePlaintext(t *testing.T, stateDir string) { + t.Helper() + err := filepath.WalkDir(stateDir, func(path string, d os.DirEntry, err error) error { + if err != nil || d.IsDir() { + return err + } + info, err := d.Info() + if err != nil { + return err + } + // The workspace disk image is a 10 GiB sparse file; reading it whole + // would be the test's own problem rather than the driver's. + if info.Size() > 1<<20 { + return nil + } + b, err := os.ReadFile(path) + if err != nil { + return err + } + if strings.Contains(string(b), workspaceMarker) { + return errors.New(strings.TrimPrefix(path, stateDir) + " holds workspace plaintext") + } + return nil + }) + if err != nil { + t.Error(err) + } +} + +// TestTheScratchDirectorysRemovalErrorNamesNoPath. What scratchDir's RemoveAll +// fails on is an entry INSIDE a previous restore of this workspace, so its +// *fs.PathError names a tenant's file — and this error reaches a session's +// error column (tenancy §15.1). +func TestTheScratchDirectorysRemovalErrorNamesNoPath(t *testing.T) { + if os.Getuid() == 0 { + t.Skip("root can remove a directory whatever its mode, so there is no failure to observe") + } + stateDir := shortTempDir(t) + m, _ := testMicrovm(t, MicrovmOpts{TotalSlots: 4, StateDir: stateDir}) + + const secret = "a-tenant-file-name-that-must-not-travel" + locked := filepath.Join(stateDir, "restore", "i-scratch", secret) + if err := os.MkdirAll(locked, 0o700); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(filepath.Join(locked, "child"), []byte("x"), 0o600); err != nil { + t.Fatal(err) + } + // Readable and searchable, not writable: the walk finds the child and + // cannot unlink it. + if err := os.Chmod(locked, 0o500); err != nil { + t.Fatal(err) + } + t.Cleanup(func() { _ = os.Chmod(locked, 0o700) }) + + _, err := m.scratchDir("i-scratch") + if err == nil { + t.Fatal("scratchDir succeeded over a directory it cannot clear") + } + for _, forbidden := range []string{secret, "child", stateDir, "/"} { + if strings.Contains(err.Error(), forbidden) { + t.Errorf("the error carries %q out of the workspace: %v", forbidden, err) + } + } +} diff --git a/internal/driver/microvm_sim_test.go b/internal/driver/microvm_sim_test.go index 0dd5d07..af26d31 100644 --- a/internal/driver/microvm_sim_test.go +++ b/internal/driver/microvm_sim_test.go @@ -18,8 +18,11 @@ package driver import ( + "archive/tar" "context" "fmt" + "io" + "io/fs" "os" "path/filepath" "strings" @@ -27,10 +30,67 @@ import ( ) // SimulatedDiskFormatter is the formatting half of the seam. +// +// FormatFromDir writes the tree it was given into the image file as a TAR, +// which is not an ext4 and does not pretend to be one — the point is that a +// test can read back what a restore put in, and that a fixture which never +// looked at the directory would be a fixture that passed while the restore was +// empty. What a real host does is mkfs.ext4 -d (Ext4Formatter), and only a real +// host can evidence that. type SimulatedDiskFormatter struct{} func (SimulatedDiskFormatter) Format(string) error { return nil } +func (SimulatedDiskFormatter) FormatFromDir(path, dir string) error { + f, err := os.Create(path) + if err != nil { + return err + } + defer f.Close() + tw := tar.NewWriter(f) + err = filepath.WalkDir(dir, func(p string, d fs.DirEntry, err error) error { + if err != nil || p == dir { + return err + } + info, err := d.Info() + if err != nil { + return err + } + link := "" + if info.Mode()&fs.ModeSymlink != 0 { + if link, err = os.Readlink(p); err != nil { + return err + } + } + hdr, err := tar.FileInfoHeader(info, link) + if err != nil { + return err + } + rel, err := filepath.Rel(dir, p) + if err != nil { + return err + } + hdr.Name = filepath.ToSlash(rel) + if err := tw.WriteHeader(hdr); err != nil { + return err + } + if !info.Mode().IsRegular() { + return nil + } + src, err := os.Open(p) + if err != nil { + return err + } + defer src.Close() + _, err = io.Copy(tw, src) + return err + }) + if err != nil { + return err + } + return tw.Close() +} + // --------------------------------------------------------------------------- // Simulated Engine (test seam) // --------------------------------------------------------------------------- diff --git a/internal/driver/microvm_test.go b/internal/driver/microvm_test.go index 7170208..59b59a0 100644 --- a/internal/driver/microvm_test.go +++ b/internal/driver/microvm_test.go @@ -122,6 +122,16 @@ type stubMicrovmHost struct { // flushErr, when set, is what a snapshot's flush reports — the state a // guest that cannot be asked, or will not answer, leaves the runner in. flushErr error + + // stream is the workspace a cold suspend's guest "streams", and streamErr + // is what that handshake reports instead. Both are nil here — this stub is + // for the tests whose subject is something else, and a driver with no + // checkpoint configuration never calls either — and the checkpoint tests + // use a host of their own (see microvm_checkpoint_test.go). + stream func(io.Writer) (WorkspaceStream, error) + streamErr error + // committed records the suspends the runner was told had committed. + committed []uint64 } func (h *stubMicrovmHost) GuestConnected(sessionID string, conn relay.Conn) { @@ -153,6 +163,27 @@ func (h *stubMicrovmHost) FlushGuest(_ context.Context, sessionID string) error return nil } +// StreamWorkspace is the runner running the cold-suspend handshake and copying +// the workspace the guest streams back. +func (h *stubMicrovmHost) StreamWorkspace(_ context.Context, _ string, dst io.Writer) (WorkspaceStream, error) { + h.mu.Lock() + stream, err := h.stream, h.streamErr + h.mu.Unlock() + if err != nil { + return WorkspaceStream{}, err + } + if stream == nil { + return WorkspaceStream{}, errors.New("this stub has no workspace to stream") + } + return stream(dst) +} + +func (h *stubMicrovmHost) CheckpointCommitted(_ string, nonce uint64) { + h.mu.Lock() + defer h.mu.Unlock() + h.committed = append(h.committed, nonce) +} + // flushCount is how many times a guest was asked to flush. A snapshot that // skipped it would publish an image missing whatever the guest had not // written yet. @@ -690,6 +721,10 @@ type failingFormatter struct{} func (failingFormatter) Format(string) error { return errors.New("mkfs.ext4 is not on PATH") } +func (failingFormatter) FormatFromDir(string, string) error { + return errors.New("mkfs.ext4 is not on PATH") +} + func TestMicrovmSnapshotRefAssociationAndStrip(t *testing.T) { m, _ := testMicrovm(t, MicrovmOpts{TotalSlots: 4}) ctx := context.Background() diff --git a/internal/driver/microvm_vsock.go b/internal/driver/microvm_vsock.go index 8230bce..3017c61 100644 --- a/internal/driver/microvm_vsock.go +++ b/internal/driver/microvm_vsock.go @@ -26,6 +26,7 @@ import ( "encoding/json" "errors" "fmt" + "io" "log" "net" "os" @@ -116,6 +117,41 @@ type MicrovmHost interface { // error here is a snapshot refused rather than an image published on // trust. See (*Microvm).Snapshot. FlushGuest(ctx context.Context, sessionID string) error + // StreamWorkspace runs the COLD suspend handshake with sessionID's guest + // and copies the workspace the guest streams back into dst, returning when + // the guest has said the tree is complete and that it is ready to be + // terminated. + // + // It is the fourth thing only the runner can do, and it is here for the + // same reason as FlushGuest: the handshake and the stream ride the + // session's relay hub, which lives in runnerd. + // + // The driver owns the BARRIER — what is done with the bytes, and the rule + // that the VM is not terminated until a manifest has committed and + // verified. This call owns the hop. Every failure it reports is a suspend + // that must not proceed: there is no such thing as a partially streamed + // workspace that is worth checkpointing. + StreamWorkspace(ctx context.Context, sessionID string, dst io.Writer) (WorkspaceStream, error) + // CheckpointCommitted tells the guest its workspace is durable. It is + // best effort and answers nothing: the VM is terminated moments later, and + // what the event buys is a line in the session's own log saying whether + // its work made it out. + CheckpointCommitted(sessionID string, nonce uint64) +} + +// WorkspaceStream is what one cold suspend's stream carried, as the guest +// counted it and as the runner received it. It holds counts and a nonce and +// nothing else — a name or a path from inside a workspace is session content +// (tenancy §15.1) and has no business on this seam. +type WorkspaceStream struct { + // Entries is how many tree entries the guest said it streamed. + Entries int64 + // Bytes is how many bytes actually ARRIVED, which the runner has already + // checked against the guest's own count. + Bytes int64 + // Nonce is the suspend this stream belonged to, so that the "your + // checkpoint committed" event can name it. + Nonce uint64 } var ( diff --git a/internal/relay/frame.go b/internal/relay/frame.go index 015c11d..085194f 100644 --- a/internal/relay/frame.go +++ b/internal/relay/frame.go @@ -25,6 +25,26 @@ const ( // wire-visible: the two ends of a live conn can be different builds, so // the numbers above must never renumber underneath it. FrameControl FrameType = 4 + // FrameStream carries one chunk of a BULK stream from the sandbox to the + // host — today there is exactly one, the workspace a cold suspend + // checkpoints — tagged in AttachID with the stream's id rather than an + // attachment's. + // + // It is its own type rather than a FrameServer on a reserved attachment id + // for two reasons. A reserved id is not reserved: Hub.next hands out + // 1, 2, 3… and a stream that collided with a live attachment would deliver + // a workspace's bytes into somebody's terminal. And the two have opposite + // routing — an attachment frame is forwarded to a client conn, a stream + // frame is consumed by the runner itself — so a type the demux can switch + // on is what keeps "forwarded to whoever is attached" from ever being the + // default for a stream. + // + // The stream id is the SUSPEND NONCE (see ControlEvent.ID), so a stream + // belonging to a suspend that already gave up is routed to nothing instead + // of into the one that replaced it. + // + // Its value is spelled out for the same reason FrameControl's is. + FrameStream FrameType = 5 ) type Frame struct { @@ -182,6 +202,26 @@ const ( // publishing an image nobody can vouch for. KindFlush = "flush" KindFlushed = "flushed" + // KindWorkspaceEnd ends the workspace stream a COLD suspend asks for: the + // sandbox saying it has written the whole tree (OK, with Entries and Bytes) + // or that it could not (OK false, with Stage and Tail). It travels upward + // only, carries the suspend's nonce in ID, and is the only thing that tells + // the host the stream it is reading is complete rather than truncated — + // a conn that simply stopped producing bytes is indistinguishable from a + // guest that finished, and a checkpoint of a truncated tree that VERIFIES + // is the one failure the durability barrier must not have. + // + // It precedes KindSuspendReady, because the tree has to be out of the guest + // before the guest declares itself ready to be terminated. + KindWorkspaceEnd = "workspace_end" + // KindCheckpointCommitted is the host telling the guest that its workspace + // is durable: the manifest is committed and verified. It travels downward + // only, carries the same nonce, and is best effort — the guest acts on + // nothing and answers nothing, and the VM is terminated immediately after. + // It exists so a session's own log says whether the work made it out, + // which is the one question a person asks about a suspend that ended + // badly. + KindCheckpointCommitted = "checkpoint_committed" ) type ControlEvent struct { @@ -250,6 +290,15 @@ type ControlEvent struct { // field reads a plain "suspending", quiesces its execs, and answers; the // host-side unmount still happens, the guest simply did not help. Cold bool `json:"cold,omitempty"` + // Entries and Bytes are KindWorkspaceEnd's counts: how many tree entries + // the sandbox streamed and how many bytes it wrote. The host compares Bytes + // against what it actually received, which is what turns "the conn went + // quiet" into a refused suspend rather than a checkpoint of half a tree. + // + // They are counts and never names: a path is session content (tenancy + // §15.1) and this channel's events reach an operator's log. + Entries int64 `json:"entries,omitempty"` + Bytes int64 `json:"bytes,omitempty"` } func Encode(f Frame) ([]byte, error) { return json.Marshal(f) } diff --git a/internal/relay/frame_test.go b/internal/relay/frame_test.go index 2de43ad..b71150b 100644 --- a/internal/relay/frame_test.go +++ b/internal/relay/frame_test.go @@ -46,7 +46,7 @@ func TestControlEventWireShape(t *testing.T) { if string(ev) != `{"kind":"setup_done"}` { t.Fatalf("plain event on the wire = %s, want {\"kind\":\"setup_done\"}", ev) } - for _, tag := range []string{"id", "ok", "payload", "stage", "rc", "tail", "live", "seq"} { + for _, tag := range []string{"id", "ok", "payload", "stage", "rc", "tail", "live", "seq", "entries", "bytes"} { if strings.Contains(string(ev), `"`+tag+`"`) { t.Fatalf("empty event leaked %q: %s", tag, ev) } diff --git a/internal/relay/runnerd_side.go b/internal/relay/runnerd_side.go index ea4e8fe..35f549b 100644 --- a/internal/relay/runnerd_side.go +++ b/internal/relay/runnerd_side.go @@ -3,6 +3,8 @@ package relay import ( "context" + "errors" + "fmt" "log" "sync" "time" @@ -30,6 +32,16 @@ type Hub struct { mu sync.Mutex next uint64 clients map[uint64]*hubClient // attachID → client + // streams is the sink table for FrameStream: stream id → the consumer of + // its chunks. It is separate from clients, and the ids come from a + // different source (a suspend nonce rather than h.next), because the two + // have opposite routing — an attachment's frame is FORWARDED to a client + // conn, a stream's frame is CONSUMED by this runner — and a table that + // mixed them could route a tenant's workspace into somebody's terminal. + // + // It holds at most one entry per in-flight stream, which today means at + // most one per cold suspend. + streams map[uint64]func([]byte) error // onControl is wired by the constructor and never written again — see // NewHubWithControl for why it isn't a settable field. onControl func(payload []byte) @@ -138,6 +150,7 @@ func newHub(ctx context.Context, sessionConn Conn, onControl func(payload []byte termWrite, execWrite time.Duration) *Hub { hctx, cancel := context.WithCancel(ctx) h := &Hub{conn: sessionConn, ctx: hctx, cancel: cancel, clients: map[uint64]*hubClient{}, + streams: map[uint64]func([]byte) error{}, onControl: onControl, termWrite: termWrite, execWrite: execWrite} go h.readLoop() return h @@ -178,6 +191,10 @@ func (h *Hub) readLoop() { } continue } + if f.Type == FrameStream { + h.deliverStream(f) + continue + } h.mu.Lock() client := h.clients[f.AttachID] h.mu.Unlock() @@ -229,6 +246,62 @@ func (h *Hub) writeClient(client *hubClient, payload []byte) error { return client.conn.Write(ctx, payload) } +// AddStream registers sink as the consumer of stream id's chunks, or refuses +// if something is already reading that id. +// +// The refusal is not a formality. A stream id is a suspend nonce, and two +// consumers on one id would each get some of a workspace's bytes — a +// checkpoint of half a tree, committed, verified against its own halves, and +// wrong. One reader or none. +// +// The sink runs ON readLoop's goroutine, which every attachment of this session +// shares. That is deliberate: it is what back-pressures the guest when the +// checkpoint writer above is not keeping up, so the workspace is never buffered +// on this host. It is affordable because the one caller is a COLD suspend — the +// agent is stopped and the execs are dead by the time a chunk arrives, so there +// is nothing else on this conn worth not blocking. A sink that returns an error +// is removed, and its caller learns from its own error rather than from here. +func (h *Hub) AddStream(id uint64, sink func([]byte) error) error { + if sink == nil { + return errors.New("relay: a stream needs a sink") + } + h.mu.Lock() + defer h.mu.Unlock() + if _, live := h.streams[id]; live { + return fmt.Errorf("relay: stream %d is already being read", id) + } + h.streams[id] = sink + return nil +} + +// RemoveStream unregisters a stream's sink. Chunks that arrive afterwards are +// dropped, which is the right answer for a stream whose reader has given up: +// there is nothing to do with the bytes, and nothing to tell a guest that is +// about to be terminated anyway. +func (h *Hub) RemoveStream(id uint64) { + h.mu.Lock() + delete(h.streams, id) + h.mu.Unlock() +} + +// deliverStream hands one chunk to its sink, or drops it. The lookup is under +// the lock and the call is not: a sink that blocks (which is the point — see +// AddStream) must not hold the table every attachment's open and close needs. +func (h *Hub) deliverStream(f Frame) { + h.mu.Lock() + sink := h.streams[f.AttachID] + h.mu.Unlock() + if sink == nil { + // A stream nobody is reading: a suspend that gave up, or a guest + // sending one that was never asked for. Dropped rather than buffered. + return + } + if err := sink(f.Payload); err != nil { + log.Printf("relay: dropping stream %d: %v", f.AttachID, err) + h.RemoveStream(f.AttachID) + } +} + // SendControl writes payload to the session as a FrameControl on AttachID 0: // the runnerd → sessiond direction of the control channel, the mirror of // ControlSender.Send on the other end. It is what carries a session-RPC diff --git a/internal/relay/session_side.go b/internal/relay/session_side.go index b2c76f1..e917e94 100644 --- a/internal/relay/session_side.go +++ b/internal/relay/session_side.go @@ -166,6 +166,23 @@ func (c *ControlSender) Send(payload []byte) error { return c.w.write(Frame{Type: FrameControl, AttachID: 0, Payload: payload}) } +// SendStream writes one chunk of stream id — today the workspace a cold +// suspend checkpoints (see FrameStream). +// +// It goes through the SAME writer as Send and as every attachment's forwarder, +// which is the whole reason it is a method here rather than a second writer on +// the conn: a workspace chunk interleaved with a control frame would corrupt +// both, and the end marker must not be able to overtake the last chunk it is +// marking the end of. +// +// It blocks while another frame is mid-write, and on a stalled conn it blocks +// until the context ServeSessionWithControl was given is cancelled. That is the +// right bound for this caller: the host is waiting for these bytes, and a +// chunk this end gave up on is a suspend that fails with no checkpoint. +func (c *ControlSender) SendStream(id uint64, chunk []byte) error { + return c.w.write(Frame{Type: FrameStream, AttachID: id, Payload: chunk}) +} + // ServeSession runs on the sessiond side: it reads Frames off conn (an // already-established outbound conn to runnerd) and demultiplexes them onto // s by AttachID — FrameOpen calls s.Attach and starts a per-attachment diff --git a/internal/relay/stream_test.go b/internal/relay/stream_test.go new file mode 100644 index 0000000..e2ba830 --- /dev/null +++ b/internal/relay/stream_test.go @@ -0,0 +1,109 @@ +package relay + +import ( + "context" + "encoding/json" + "testing" +) + +// The bulk stream a cold suspend carries: its frame's shape on the wire, and +// the one discipline that makes it safe — it goes through the same writer as +// every control frame, so the end marker cannot overtake the chunk it ends. + +// TestStreamFrameWireShape pins the frame a workspace chunk travels in. Both +// ends of this hop come from different build lineages — sessiond ships inside +// a session image, the hub runs on the host — so the type number and the field +// names are a contract, not an implementation detail. +func TestStreamFrameWireShape(t *testing.T) { + if FrameStream != 5 { + t.Fatalf("FrameStream is %d; renumbering it silently reroutes a tenant's workspace", FrameStream) + } + b, err := Encode(Frame{Type: FrameStream, AttachID: 91, Payload: []byte("abc")}) + if err != nil { + t.Fatal(err) + } + if string(b) != `{"t":5,"a":91,"p":"YWJj"}` { + t.Fatalf("a stream frame on the wire = %s", b) + } + f, err := Decode(b) + if err != nil { + t.Fatal(err) + } + if f.Type != FrameStream || f.AttachID != 91 || string(f.Payload) != "abc" { + t.Fatalf("a stream frame decoded as %+v", f) + } +} + +// TestWorkspaceEndCarriesCountsAndNoNames. The end marker is the only thing +// that tells the host a stream is COMPLETE rather than truncated, and the only +// thing it may carry about a workspace is how much of one there was: a path is +// session content (tenancy §15.1) and this event reaches an operator's log. +func TestWorkspaceEndCarriesCountsAndNoNames(t *testing.T) { + b, err := json.Marshal(ControlEvent{Kind: KindWorkspaceEnd, ID: 7, OK: true, Entries: 12, Bytes: 4096}) + if err != nil { + t.Fatal(err) + } + if string(b) != `{"kind":"workspace_end","id":7,"ok":true,"entries":12,"bytes":4096}` { + t.Fatalf("the end marker on the wire = %s", b) + } + var ev ControlEvent + if err := json.Unmarshal(b, &ev); err != nil { + t.Fatal(err) + } + if ev.Entries != 12 || ev.Bytes != 4096 { + t.Fatalf("the end marker decoded as %+v", ev) + } +} + +// TestSendStreamSharesTheControlWriter is the ordering guarantee the handshake +// rests on. A chunk and the end marker that follows it are written through one +// writer, so what the host reads is what the guest wrote, in that order. +func TestSendStreamSharesTheControlWriter(t *testing.T) { + a, b := newPipe() + defer a.Close() + defer b.Close() + + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + sender := &ControlSender{w: newConnWriter(ctx, a)} + + end, err := json.Marshal(ControlEvent{Kind: KindWorkspaceEnd, ID: 4, OK: true, Entries: 1, Bytes: 3}) + if err != nil { + t.Fatal(err) + } + if err := sender.SendStream(4, []byte("abc")); err != nil { + t.Fatal(err) + } + if err := sender.Send(end); err != nil { + t.Fatal(err) + } + + first := readFrame(t, b) + if first.Type != FrameStream || first.AttachID != 4 || string(first.Payload) != "abc" { + t.Fatalf("the first frame is %+v, want the chunk", first) + } + second := readFrame(t, b) + if second.Type != FrameControl || second.AttachID != 0 { + t.Fatalf("the second frame is %+v, want the end marker on the control channel", second) + } + var ev ControlEvent + if err := json.Unmarshal(second.Payload, &ev); err != nil { + t.Fatal(err) + } + if ev.Kind != KindWorkspaceEnd || ev.ID != 4 { + t.Fatalf("the second frame carries %+v, want the workspace end marker for suspend 4", ev) + } +} + +func readFrame(t *testing.T, c Conn) Frame { + t.Helper() + raw, err := c.Read(context.Background()) + if err != nil { + t.Fatalf("reading a frame: %v", err) + } + f, err := Decode(raw) + if err != nil { + t.Fatalf("decoding a frame: %v", err) + } + return f +} diff --git a/internal/runnerd/microvm.go b/internal/runnerd/microvm.go index 3cee822..72efbc7 100644 --- a/internal/runnerd/microvm.go +++ b/internal/runnerd/microvm.go @@ -352,3 +352,11 @@ func (s *Server) driverCapabilities() []string { func (s *Server) withholdsSecrets() bool { return slices.Contains(s.driverCapabilities(), runner.CapabilityMicrovmV1) } + +// driverCheckpointsColdSuspend reports whether this runner's driver runs the +// cold-suspend workspace handshake itself, in which case this runner does not +// send a notice of its own. See driver.CheckpointingDriver. +func (s *Server) driverCheckpointsColdSuspend() bool { + cd, ok := s.drv.(driver.CheckpointingDriver) + return ok && cd.CheckpointsColdSuspend() +} diff --git a/internal/runnerd/runnerd.go b/internal/runnerd/runnerd.go index 195cbb2..4220fd4 100644 --- a/internal/runnerd/runnerd.go +++ b/internal/runnerd/runnerd.go @@ -122,6 +122,18 @@ type Server struct { // flushWait is how long a guest gets to flush. A field so a test does not // spend the production budget; written only immediately after New. flushWait time.Duration + // workspaces is the one waiter per session for the end marker of a cold + // suspend's workspace stream, and the three budgets that stream gets. It is + // separate from the suspend waiter that shares its nonce because the two + // answer different questions — "the tree is out of the guest" and "this VM + // may go" — and the host does the whole checkpoint between them. See + // workspacestream.go; the budgets are fields for the reason every other one + // here is, and are written only immediately after New. + workspaceMu sync.Mutex + workspaces map[string]*workspaceWaiter + workspaceStartWait time.Duration + workspaceIdleWait time.Duration + workspaceTotalWait time.Duration // displacedHubGrace is how long a hub a re-register replaced keeps // delivering before it is closed. A field for the same reason, and // written only immediately after New. @@ -252,8 +264,12 @@ func New(drv driver.Driver, dialBase, egressAdmin, proxyURL string) *Server { coldSuspendReadyWait: defaultColdSuspendReadyWait, displacedHubGrace: defaultDisplacedHubGrace, flushWait: defaultFlushWait, + workspaceStartWait: defaultWorkspaceStartWait, + workspaceIdleWait: defaultWorkspaceIdleWait, + workspaceTotalWait: defaultWorkspaceTotalWait, suspends: map[string]*suspendWaiter{}, flushes: map[string]*flushWaiter{}, + workspaces: map[string]*workspaceWaiter{}, runnerRPC: newRunnerRPCTable()} // The microVM driver is composed BELOW this server and needs three things // from above it — where a guest's control conn goes, where a cold resume's @@ -722,7 +738,13 @@ func (s *Server) Op(ctx context.Context, id, op string, warm bool) error { // delivers the SIGTERM sessiond's own handler already answers, // and a notice sent there would be a behaviour change on the one // path this design promises to leave exactly as it is. - if s.withholdsSecrets() { + // + // Unless the DRIVER sends it. A driver that checkpoints the + // workspace owns the whole handshake — the notice, the stream the + // guest answers it with, and the barrier that stream feeds — and a + // second notice from here would have the sandbox quiesce twice and + // stream into a suspend nobody is reading. + if s.withholdsSecrets() && !s.driverCheckpointsColdSuspend() { s.quiesceExecs(ctx, id, true) } // Cold suspend (docker stop) kills the container's sessiond, @@ -1218,6 +1240,12 @@ func (s *Server) routeControl(id string, boot, reg uint64, payload []byte) { case relay.KindSuspendReady: // And its execs are gone, so the container may be frozen. s.deliverSuspend(id, ev.ID, true) + case relay.KindWorkspaceEnd: + // The sandbox has finished streaming its workspace, or could not. Like + // the suspend answers and the flush, it stays between this process and + // that sandbox: what is waiting on it is the cold suspend's own + // durability barrier, inside this runner. + s.deliverWorkspaceEnd(id, ev) case relay.KindFlushed: // The sandbox has put what it wrote on its block devices, so the host // may copy the file underneath it. Like the suspend answers, this diff --git a/internal/runnerd/workspacestream.go b/internal/runnerd/workspacestream.go new file mode 100644 index 0000000..83c061a --- /dev/null +++ b/internal/runnerd/workspacestream.go @@ -0,0 +1,311 @@ +// internal/runnerd/workspacestream.go +// +// The runner's half of the cold-suspend checkpoint: it asks a guest to cold +// suspend, reads the workspace the guest streams back, and hands the bytes to +// whoever is checkpointing them. +// +// It lives here rather than in the driver for the same reason FlushGuest does: +// the guest's control connection belongs to the session's relay hub, and the +// hub lives in this package. The driver owns the BARRIER — what is done with +// the bytes, and what may not happen until the manifest commits — and calls +// this for the hop it cannot reach. +package runnerd + +import ( + "context" + "encoding/json" + "fmt" + "io" + "log" + "sync" + "sync/atomic" + "time" + + "github.com/tokencanopy/rainier/internal/driver" + "github.com/tokencanopy/rainier/internal/relay" +) + +// The four budgets a cold suspend's stream gets. They replace one stopwatch +// with a shape, because the work behind them has a shape: a fixed amount of +// quiescing, then a copy whose length is the workspace's, then an answer. +// +// See docs/design/2026-09-23-workspace-checkpoint-wiring.md §4. Every one is a +// field on the Server (not a constant read here) so a test can drive it in +// milliseconds, exactly as the suspend and flush budgets are. +const ( + // defaultWorkspaceStartWait is from the sandbox's acknowledgement to the + // first chunk: the flush, the ten-second exec kill and the unmount that + // #98's cold budget already bought, unchanged. + defaultWorkspaceStartWait = 30 * time.Second + // defaultWorkspaceIdleWait is between chunks, and from the last chunk to + // the end marker. It bounds PROGRESS rather than throughput, and it + // deliberately covers this host's own slowness too: the stream is + // back-pressured by whatever is consuming it, so a blob store that has + // stopped taking bytes stalls the stream and fails the suspend — which + // keeps the VM, which is the right answer. + defaultWorkspaceIdleWait = 60 * time.Second + // defaultWorkspaceTotalWait is the ceiling on the whole copy. The workspace + // disk is 10 GiB (driver.workspaceDiskBytes); thirty minutes is that at + // about 6 MiB/s, which is well under what a vsock hop and a local store + // manage and well over what a stalled one does. + defaultWorkspaceTotalWait = 30 * time.Minute +) + +// workspaceWaiter is one in-flight stream's end marker, matched by nonce. It is +// separate from the suspend waiter that carries the same nonce because the two +// answer different questions — "the tree is out" and "this VM may go" — and the +// host acts between them. +type workspaceWaiter struct { + nonce uint64 + done chan struct{} + once sync.Once + // The end marker's own words, written before done is closed and read only + // after it, so the channel is the synchronization. + ok bool + entries int64 + bytes int64 + stage string + tail string +} + +func (s *Server) armWorkspaceWaiter(id string, nonce uint64) *workspaceWaiter { + w := &workspaceWaiter{nonce: nonce, done: make(chan struct{})} + s.workspaceMu.Lock() + s.workspaces[id] = w + s.workspaceMu.Unlock() + return w +} + +func (s *Server) disarmWorkspaceWaiter(id string, w *workspaceWaiter) { + s.workspaceMu.Lock() + if s.workspaces[id] == w { + delete(s.workspaces, id) + } + s.workspaceMu.Unlock() +} + +// deliverWorkspaceEnd wakes this session's stream waiter, and only when the +// NONCE matches the stream that is actually in flight: a late end marker from a +// suspend that already gave up must not tell the next one that a tree it never +// received is complete. +func (s *Server) deliverWorkspaceEnd(id string, ev relay.ControlEvent) { + s.workspaceMu.Lock() + w := s.workspaces[id] + s.workspaceMu.Unlock() + switch { + case w == nil: + return + case w.nonce != ev.ID: + log.Printf("session %s: a workspace end marker for %d arrived while %d is in flight; dropping it", id, ev.ID, w.nonce) + default: + w.once.Do(func() { + w.ok, w.entries, w.bytes = ev.OK, ev.Entries, ev.Bytes + w.stage, w.tail = ev.Stage, ev.Tail + close(w.done) + }) + } +} + +// StreamWorkspace runs the cold-suspend handshake with sessionID's sandbox and +// copies the workspace it streams back into dst. +// +// It returns when the guest has said the tree is complete AND said it is ready +// to be terminated, and not before: the counts in the end marker are what turn +// "the connection went quiet" into a refused suspend rather than a checkpoint +// of half a tree. +// +// Every failure is an error rather than a shrug, which is the difference +// between this and quiesceExecs. A warm suspend that goes ahead unquiesced +// freezes a container that will be thawed again; a COLD one that goes ahead +// unstreamed terminates a VM whose workspace nobody has a copy of. So a sandbox +// that never registered, a sessiond that predates this vocabulary, a conn that +// died and a guest that is simply too slow all read the same way here: this +// session cannot be checkpointed, so it is not suspended either. +func (s *Server) StreamWorkspace(ctx context.Context, sessionID string, dst io.Writer) (driver.WorkspaceStream, error) { + var none driver.WorkspaceStream + hub, ok := s.reg.hub(sessionID) + if !ok { + return none, fmt.Errorf("session %s has no sandbox connection to stream its workspace over", sessionID) + } + nonce := s.suspendNonce.Add(1) + sw := s.armSuspendWaiter(sessionID, nonce) + defer s.disarmSuspendWaiter(sessionID, sw) + ww := s.armWorkspaceWaiter(sessionID, nonce) + defer s.disarmWorkspaceWaiter(sessionID, ww) + + // received is written by the sink, which runs on the hub's read loop, and + // read here. Atomic rather than "ordered by the end marker": the ordering + // argument is true on the happy path and this value is also read on the + // paths where it is not, and a counter that is correct only when nothing + // went wrong is not a counter worth checking a checkpoint against. + var received atomic.Int64 + progress := make(chan struct{}, 1) + failed := make(chan error, 1) + if err := hub.AddStream(nonce, func(chunk []byte) error { + n, err := dst.Write(chunk) + received.Add(int64(n)) + if err != nil { + select { + case failed <- err: + default: + } + return err + } + select { + case progress <- struct{}{}: + default: + } + return nil + }); err != nil { + return none, fmt.Errorf("session %s: %w", sessionID, err) + } + defer hub.RemoveStream(nonce) + + b, err := json.Marshal(relay.ControlEvent{Kind: relay.KindSuspending, ID: nonce, Cold: true}) + if err != nil { + return none, fmt.Errorf("encoding the cold suspend notice for %s: %w", sessionID, err) + } + if err := hub.SendControl(b); err != nil { + return none, fmt.Errorf("asking session %s to cold suspend: %w", sessionID, err) + } + + // "Did you hear me." A sandbox that predates this vocabulary answers + // nothing at all, and there is no checkpoint to be had from one — so unlike + // the warm path, which shrugs and freezes anyway, this is a refusal. + ackTimer := time.NewTimer(s.suspendAckWait) + defer ackTimer.Stop() + select { + case <-sw.ack: + case err := <-failed: + return none, fmt.Errorf("session %s: the workspace stream could not be taken: %w", sessionID, err) + case <-hub.Done(): + return none, fmt.Errorf("session %s lost its sandbox connection before it acknowledged the cold suspend", sessionID) + case <-ctx.Done(): + return none, fmt.Errorf("waiting for session %s to acknowledge the cold suspend: %w", sessionID, ctx.Err()) + case <-ackTimer.C: + return none, fmt.Errorf("session %s did not acknowledge the cold suspend within %s "+ + "(a sandbox that predates the workspace stream never will, and cannot be checkpointed)", + sessionID, s.suspendAckWait) + } + + if err := s.readStream(ctx, sessionID, hub, ww, &received, progress, failed); err != nil { + return none, err + } + if !ww.ok { + // The guest's own refusal, in the stage_failed shape: a stage and a + // tail, with counts and no path. Both strings are the SANDBOX's, so + // both are bounded here before they travel any further — sessiond is + // careful about what it puts in them (see its streamFailureTail), and + // this hop is where that stops being something to rely on. + return none, fmt.Errorf("session %s could not stream its workspace (stage %s, %d entries and %d bytes in): %s", + sessionID, clampGuestText(ww.stage), ww.entries, ww.bytes, clampGuestText(ww.tail)) + } + if got := received.Load(); ww.bytes != got { + // The one check only this hop can make: the guest says how much it + // wrote, this end knows how much arrived, and a difference is a stream + // that lost bytes somewhere between them. + return none, fmt.Errorf("session %s streamed %d bytes of workspace and %d arrived; "+ + "the stream is not complete and nothing may be checkpointed from it", sessionID, ww.bytes, got) + } + + // And "you may go". It follows the end marker immediately on the same + // ordered conn, so this is almost always already satisfied. + readyTimer := time.NewTimer(s.coldSuspendReadyWait) + defer readyTimer.Stop() + select { + case <-sw.ready: + case <-hub.Done(): + // The conn died after the whole tree and the end marker arrived. The + // stream is complete and checked, and a guest that cannot say a last + // word is not a reason to throw away a workspace that is already out of + // it — the VM is being terminated either way. + log.Printf("session %s: the sandbox connection ended after its workspace stream but before it reported ready; continuing", sessionID) + case <-ctx.Done(): + return none, fmt.Errorf("waiting for session %s to report ready after its workspace stream: %w", sessionID, ctx.Err()) + case <-readyTimer.C: + log.Printf("session %s: the sandbox streamed its whole workspace but did not report ready within %s; continuing", + sessionID, s.coldSuspendReadyWait) + } + return driver.WorkspaceStream{Entries: ww.entries, Bytes: received.Load(), Nonce: nonce}, nil +} + +// maxGuestText bounds a string the sandbox chose before this runner puts it in +// an error. The frame limit above it is 16 MiB, which is not a bound for +// something that reaches an operator's log and a session's error column. +const maxGuestText = 512 + +// clampGuestText truncates a sandbox-supplied string and says that it did. +func clampGuestText(s string) string { + if len(s) <= maxGuestText { + return s + } + return s[:maxGuestText] + "… (truncated)" +} + +// readStream waits for the end marker under the three budgets: one for the +// stream to START, one for it to keep MOVING, and one for the whole of it. +// +// The idle budget is reset by progress rather than by arrival count, so a +// steady copy of a large workspace never touches it and a stalled one always +// does, whichever end stalled. +func (s *Server) readStream(ctx context.Context, sessionID string, hub *relay.Hub, ww *workspaceWaiter, + received *atomic.Int64, progress <-chan struct{}, failed <-chan error) error { + total := time.NewTimer(s.workspaceTotalWait) + defer total.Stop() + quiet := time.NewTimer(s.workspaceStartWait) + defer quiet.Stop() + started := false + + for { + select { + case <-ww.done: + return nil + case <-progress: + if !started { + started = true + } + quiet.Reset(s.workspaceIdleWait) + case err := <-failed: + // This host stopped taking the bytes: the checkpoint writer failed, + // or the store did. Named as what it is, so that a store outage is + // not reported as a guest that would not stream. + return fmt.Errorf("session %s: the workspace stream could not be taken: %w", sessionID, err) + case <-hub.Done(): + return fmt.Errorf("session %s lost its sandbox connection part way through its workspace stream "+ + "(%d bytes in); nothing may be checkpointed from it", sessionID, received.Load()) + case <-ctx.Done(): + return fmt.Errorf("streaming session %s's workspace: %w", sessionID, ctx.Err()) + case <-total.C: + return fmt.Errorf("session %s was still streaming its workspace after %s", sessionID, s.workspaceTotalWait) + case <-quiet.C: + if !started { + return fmt.Errorf("session %s did not begin streaming its workspace within %s of acknowledging the cold suspend", + sessionID, s.workspaceStartWait) + } + return fmt.Errorf("session %s's workspace stream stopped making progress for %s", sessionID, s.workspaceIdleWait) + } + } +} + +// CheckpointCommitted tells sessionID's guest that its workspace is durable. +// +// Best effort, and deliberately so: the guest acts on nothing and answers +// nothing, the VM is terminated moments later, and the value of the event is +// that the session's own log says whether its work made it out. A failure here +// is not a failed suspend — the manifest is already committed and verified. +// +// nonce is the suspend's, so a session's log ties the line to the stream it +// wrote rather than to "some checkpoint". +func (s *Server) CheckpointCommitted(sessionID string, nonce uint64) { + hub, ok := s.reg.hub(sessionID) + if !ok { + return + } + b, err := json.Marshal(relay.ControlEvent{Kind: relay.KindCheckpointCommitted, ID: nonce}) + if err != nil { + return + } + if err := hub.SendControl(b); err != nil { + log.Printf("session %s: telling the sandbox its checkpoint committed: %v", sessionID, err) + } +} diff --git a/internal/runnerd/workspacestream_test.go b/internal/runnerd/workspacestream_test.go new file mode 100644 index 0000000..b199d05 --- /dev/null +++ b/internal/runnerd/workspacestream_test.go @@ -0,0 +1,388 @@ +// internal/runnerd/workspacestream_test.go +package runnerd + +import ( + "bytes" + "context" + "errors" + "net/http/httptest" + "strings" + "testing" + "time" + + "github.com/coder/websocket" + + "github.com/tokencanopy/rainier/internal/driver" + "github.com/tokencanopy/rainier/internal/relay" +) + +// The runner's half of the cold-suspend checkpoint. Every test here is about +// the one rule that makes the barrier a barrier: this call returns success only +// when the WHOLE workspace arrived, and every other outcome is an error the +// driver above it turns into a suspend that did not happen. + +// streamScene is a runner with one registered session and the three stream +// budgets cut to something a test can spend. +func streamScene(t *testing.T) (*Server, *sandbox, string) { + t.Helper() + rd := New(driver.NewFake(4), "", "", "") + rd.suspendAckWait = 500 * time.Millisecond + rd.coldSuspendReadyWait = 500 * time.Millisecond + rd.workspaceStartWait = time.Second + rd.workspaceIdleWait = time.Second + rd.workspaceTotalWait = 10 * time.Second + srv := httptest.NewServer(rd.Handler()) + t.Cleanup(srv.Close) + id, sb := dialSandbox(t, rd, srv) + return rd, sb, id +} + +// sendStream writes one chunk of stream id up the session's conn, the way +// sessiond's ControlSender.SendStream does. +func (s *sandbox) sendStream(t *testing.T, id uint64, chunk []byte) { + t.Helper() + f, err := relay.Encode(relay.Frame{Type: relay.FrameStream, AttachID: id, Payload: chunk}) + if err != nil { + t.Fatal(err) + } + if err := s.c.Write(s.ctx, websocket.MessageText, f); err != nil { + t.Fatalf("write stream frame: %v", err) + } +} + +// coldNotice reads the cold suspend notice the runner sends and returns its +// nonce, which is also the id of the stream that answers it. +func coldNotice(t *testing.T, sb *sandbox) uint64 { + t.Helper() + ev := sb.nextControl(t) + if ev.Kind != relay.KindSuspending || !ev.Cold { + t.Fatalf("the sandbox was sent %+v, want a cold suspending notice", ev) + } + if ev.ID == 0 { + t.Fatal("the cold suspend notice carries no nonce; a late answer could satisfy the next suspend") + } + return ev.ID +} + +type streamOutcome struct { + rep driver.WorkspaceStream + err error +} + +// TestStreamWorkspaceReadsTheWholeTreeAndChecksTheCounts is the happy path, and +// the counts are the point: the guest says how much it wrote and this end knows +// how much arrived, which is what turns "the connection went quiet" into a +// refused suspend rather than a checkpoint of half a tree. +func TestStreamWorkspaceReadsTheWholeTreeAndChecksTheCounts(t *testing.T) { + rd, sb, id := streamScene(t) + + var got bytes.Buffer + done := make(chan streamOutcome, 1) + go func() { + rep, err := rd.StreamWorkspace(context.Background(), id, &got) + done <- streamOutcome{rep, err} + }() + + nonce := coldNotice(t, sb) + sb.answer(t, relay.KindSuspendAck, nonce) + sb.sendStream(t, nonce, []byte("workspace-")) + sb.sendStream(t, nonce, []byte("bytes")) + sb.send(t, relay.ControlEvent{Kind: relay.KindWorkspaceEnd, ID: nonce, OK: true, Entries: 7, Bytes: 15}) + sb.answer(t, relay.KindSuspendReady, nonce) + + out := <-done + if out.err != nil { + t.Fatalf("the stream failed: %v", out.err) + } + if got.String() != "workspace-bytes" { + t.Errorf("the runner received %q", got.String()) + } + if out.rep.Entries != 7 || out.rep.Bytes != 15 || out.rep.Nonce != nonce { + t.Errorf("the report is %+v, want 7 entries, 15 bytes and nonce %d", out.rep, nonce) + } +} + +// TestStreamWorkspaceRefusesCountsThatDisagree: the guest wrote more than +// arrived, which is a stream that lost bytes between the two ends. Nothing may +// be checkpointed from it. +func TestStreamWorkspaceRefusesCountsThatDisagree(t *testing.T) { + rd, sb, id := streamScene(t) + + done := make(chan streamOutcome, 1) + go func() { + rep, err := rd.StreamWorkspace(context.Background(), id, new(bytes.Buffer)) + done <- streamOutcome{rep, err} + }() + + nonce := coldNotice(t, sb) + sb.answer(t, relay.KindSuspendAck, nonce) + sb.sendStream(t, nonce, []byte("short")) + sb.send(t, relay.ControlEvent{Kind: relay.KindWorkspaceEnd, ID: nonce, OK: true, Entries: 3, Bytes: 4096}) + + out := <-done + if out.err == nil { + t.Fatal("a stream that lost bytes was accepted") + } + if !strings.Contains(out.err.Error(), "not complete") { + t.Errorf("the refusal is %v", out.err) + } +} + +// TestStreamWorkspaceRefusesTheGuestsOwnFailure: the sandbox could not read its +// own workspace. It reports rather than deciding — it does not know whether a +// checkpoint committed — and this end is what refuses. +func TestStreamWorkspaceRefusesTheGuestsOwnFailure(t *testing.T) { + rd, sb, id := streamScene(t) + + done := make(chan streamOutcome, 1) + go func() { + rep, err := rd.StreamWorkspace(context.Background(), id, new(bytes.Buffer)) + done <- streamOutcome{rep, err} + }() + + nonce := coldNotice(t, sb) + sb.answer(t, relay.KindSuspendAck, nonce) + sb.sendStream(t, nonce, []byte("partial")) + sb.send(t, relay.ControlEvent{Kind: relay.KindWorkspaceEnd, ID: nonce, Stage: "workspace_stream", RC: -1, + Entries: 41, Bytes: 7, Tail: "the runner is not taking the workspace stream"}) + + out := <-done + if out.err == nil { + t.Fatal("a guest that could not stream its workspace was accepted") + } + for _, want := range []string{"workspace_stream", "41 entries"} { + if !strings.Contains(out.err.Error(), want) { + t.Errorf("the refusal %q does not carry %q", out.err, want) + } + } +} + +// TestStreamWorkspaceRefusesASandboxThatCannotAnswer. A sessiond that predates +// this vocabulary answers nothing at all, and there is no checkpoint to be had +// from one — so unlike the warm path, which shrugs and freezes anyway, this is +// a refusal. +func TestStreamWorkspaceRefusesASandboxThatCannotAnswer(t *testing.T) { + rd, sb, id := streamScene(t) + + done := make(chan streamOutcome, 1) + go func() { + rep, err := rd.StreamWorkspace(context.Background(), id, new(bytes.Buffer)) + done <- streamOutcome{rep, err} + }() + + coldNotice(t, sb) // heard, and deliberately never answered + + out := <-done + if out.err == nil { + t.Fatal("a sandbox that never acknowledged the cold suspend was accepted") + } + if !strings.Contains(out.err.Error(), "did not acknowledge") { + t.Errorf("the refusal is %v", out.err) + } +} + +// TestStreamWorkspaceRefusesAStreamThatNeverStarts and +// TestStreamWorkspaceRefusesAStreamThatStalls are the two halves of the +// progress budget: one bounds the quiescing before the first chunk, the other +// bounds the gap between chunks. They are different numbers because they bound +// different work. +func TestStreamWorkspaceRefusesAStreamThatNeverStarts(t *testing.T) { + rd, sb, id := streamScene(t) + + done := make(chan streamOutcome, 1) + go func() { + rep, err := rd.StreamWorkspace(context.Background(), id, new(bytes.Buffer)) + done <- streamOutcome{rep, err} + }() + + nonce := coldNotice(t, sb) + sb.answer(t, relay.KindSuspendAck, nonce) + + out := <-done + if out.err == nil || !strings.Contains(out.err.Error(), "did not begin streaming") { + t.Fatalf("a sandbox that acknowledged and streamed nothing gave %v", out.err) + } +} + +func TestStreamWorkspaceRefusesAStreamThatStalls(t *testing.T) { + rd, sb, id := streamScene(t) + + done := make(chan streamOutcome, 1) + go func() { + rep, err := rd.StreamWorkspace(context.Background(), id, new(bytes.Buffer)) + done <- streamOutcome{rep, err} + }() + + nonce := coldNotice(t, sb) + sb.answer(t, relay.KindSuspendAck, nonce) + sb.sendStream(t, nonce, []byte("a start and then nothing")) + + out := <-done + if out.err == nil || !strings.Contains(out.err.Error(), "stopped making progress") { + t.Fatalf("a stalled stream gave %v", out.err) + } +} + +// TestStreamWorkspaceRefusesWhenThisHostStopsTakingBytes. The stream is +// back-pressured by whatever is consuming it, so a checkpoint writer or a blob +// store that failed stalls the stream — and the failure must be named as what +// it is rather than reported as a guest that would not stream. +func TestStreamWorkspaceRefusesWhenThisHostStopsTakingBytes(t *testing.T) { + rd, sb, id := streamScene(t) + + done := make(chan streamOutcome, 1) + go func() { + rep, err := rd.StreamWorkspace(context.Background(), id, failingWriter{}) + done <- streamOutcome{rep, err} + }() + + nonce := coldNotice(t, sb) + sb.answer(t, relay.KindSuspendAck, nonce) + sb.sendStream(t, nonce, []byte("anything at all")) + + out := <-done + if out.err == nil || !strings.Contains(out.err.Error(), "could not be taken") { + t.Fatalf("a sink that failed gave %v", out.err) + } +} + +// TestStreamWorkspaceIgnoresAnEndMarkerForAnotherSuspend: a late answer from a +// suspend that already gave up must not tell this one that a tree it never +// received is complete. That is what the nonce is for. +func TestStreamWorkspaceIgnoresAnEndMarkerForAnotherSuspend(t *testing.T) { + rd, sb, id := streamScene(t) + + done := make(chan streamOutcome, 1) + go func() { + rep, err := rd.StreamWorkspace(context.Background(), id, new(bytes.Buffer)) + done <- streamOutcome{rep, err} + }() + + nonce := coldNotice(t, sb) + sb.answer(t, relay.KindSuspendAck, nonce) + sb.sendStream(t, nonce, []byte("some bytes")) + // The end marker of a suspend that is not this one. + sb.send(t, relay.ControlEvent{Kind: relay.KindWorkspaceEnd, ID: nonce + 99, OK: true, Entries: 1, Bytes: 10}) + + out := <-done + if out.err == nil || !strings.Contains(out.err.Error(), "stopped making progress") { + t.Fatalf("an end marker from another suspend gave %v", out.err) + } +} + +// TestStreamWorkspaceRefusesASessionWithNoSandbox: a session whose sandbox +// never registered cannot be checkpointed, so it is not suspended either. +func TestStreamWorkspaceRefusesASessionWithNoSandbox(t *testing.T) { + rd := New(driver.NewFake(4), "", "", "") + if _, err := rd.StreamWorkspace(context.Background(), "sess-nobody", new(bytes.Buffer)); err == nil { + t.Fatal("a session with no sandbox connection was accepted") + } +} + +// TestStreamWorkspaceDropsChunksNobodyAskedFor. A guest may put a stream frame +// on the wire whenever it likes; one that matches no in-flight suspend is +// dropped rather than buffered, and the session carries on. +func TestStreamWorkspaceDropsChunksNobodyAskedFor(t *testing.T) { + rd, sb, id := streamScene(t) + + sb.sendStream(t, 12345, []byte("a stream nobody asked for")) + + // The conn is still live and still serving: the proof is that a suspend + // started afterwards still reaches this sandbox. + done := make(chan streamOutcome, 1) + go func() { + rep, err := rd.StreamWorkspace(context.Background(), id, new(bytes.Buffer)) + done <- streamOutcome{rep, err} + }() + nonce := coldNotice(t, sb) + sb.answer(t, relay.KindSuspendAck, nonce) + sb.sendStream(t, nonce, []byte("the real one")) + sb.send(t, relay.ControlEvent{Kind: relay.KindWorkspaceEnd, ID: nonce, OK: true, Entries: 1, Bytes: 12}) + sb.answer(t, relay.KindSuspendReady, nonce) + if out := <-done; out.err != nil { + t.Fatalf("the stream after an unsolicited chunk failed: %v", out.err) + } +} + +// TestAColdSuspendSendsOneNoticeWhenTheDriverCheckpoints. The notice and the +// stream that answers it are ONE handshake with one nonce, so a runner that +// sent its own notice as well would have the sandbox quiesce twice and stream +// into a suspend nobody is reading. +func TestAColdSuspendSendsOneNoticeWhenTheDriverCheckpoints(t *testing.T) { + rd := New(&checkpointingFake{Fake: driver.NewFake(4)}, "", "", "") + rd.suspendAckWait = 200 * time.Millisecond + rd.coldSuspendReadyWait = 200 * time.Millisecond + srv := httptest.NewServer(rd.Handler()) + t.Cleanup(srv.Close) + id, sb := dialSandbox(t, rd, srv) + + if err := rd.Op(context.Background(), id, "suspend", false); err != nil { + t.Fatalf("the cold suspend failed: %v", err) + } + + // Nothing was sent down this conn: the driver owns the handshake. + ctx, cancel := context.WithTimeout(sb.ctx, 300*time.Millisecond) + defer cancel() + if _, _, err := sb.c.Read(ctx); err == nil { + t.Fatal("the runner sent the sandbox a notice of its own; the driver already sends one") + } else if !errors.Is(err, context.DeadlineExceeded) { + t.Fatalf("reading from the runner: %v", err) + } +} + +// checkpointingFake is a driver that announces the microVM capability AND says +// it runs the cold-suspend handshake itself, which is the pair runnerd reads +// before deciding whether to send a notice of its own. +type checkpointingFake struct{ *driver.Fake } + +func (*checkpointingFake) Capabilities() []string { return []string{"microvm.v1"} } +func (*checkpointingFake) CheckpointsColdSuspend() bool { return true } +func (f *checkpointingFake) SetHost(h driver.MicrovmHost) {} + +// failingWriter is a sink that cannot take a byte: a checkpoint writer whose +// store has gone away. +type failingWriter struct{} + +func (failingWriter) Write([]byte) (int, error) { + return 0, errors.New("the store is not taking bytes") +} + +// TestClampGuestTextBoundsWhatTheSandboxChose. Both strings in the guest's own +// failure report are the SANDBOX's, and they reach an operator's log and a +// session's error column. sessiond is careful about what it puts in them; this +// hop is where that stops being something to rely on, so the bound is here and +// is tested here. +func TestClampGuestTextBoundsWhatTheSandboxChose(t *testing.T) { + short := "mkfs" + if got := clampGuestText(short); got != short { + t.Errorf("a short string came back as %q", got) + } + exact := strings.Repeat("a", maxGuestText) + if got := clampGuestText(exact); got != exact { + t.Errorf("a string at the bound was truncated (%d bytes)", len(got)) + } + + over := strings.Repeat("b", maxGuestText+1) + got := clampGuestText(over) + if len(got) <= maxGuestText { + t.Fatalf("the truncated form is %d bytes, which is not the truncation notice plus the bound", len(got)) + } + if !strings.HasPrefix(got, strings.Repeat("b", maxGuestText)) { + t.Errorf("the truncated form is not the first %d bytes: %q", maxGuestText, got) + } + if strings.Count(got, "b") != maxGuestText { + t.Errorf("the truncated form carries %d bytes of the sandbox's string, want %d", strings.Count(got, "b"), maxGuestText) + } + // It says that it did, so nobody reads a clamped stage name as the whole + // of what the guest reported. + if !strings.Contains(got, "truncated") { + t.Errorf("the truncated form does not say it was truncated: %q", got) + } + + // And a megabyte of it — the frame limit above this is 16 MiB — comes back + // bounded, which is the whole point. + huge := clampGuestText(strings.Repeat("c", 1<<20)) + if len(huge) > maxGuestText+64 { + t.Errorf("a 1 MiB guest string came back as %d bytes", len(huge)) + } +} diff --git a/internal/wstream/checkpoint_test.go b/internal/wstream/checkpoint_test.go new file mode 100644 index 0000000..68b18a6 --- /dev/null +++ b/internal/wstream/checkpoint_test.go @@ -0,0 +1,136 @@ +package wstream + +import ( + "bytes" + "context" + "io/fs" + "os" + "path/filepath" + "testing" + + "github.com/tokencanopy/rainier/checkpoint" +) + +// The claim this whole package exists for: a workspace that left a guest as a +// stream can be checkpointed, verified, and restored into a directory that is +// the workspace again. +// +// It uses the real library rather than a stand-in, because "the fs.FS the +// checkpoint writer walks" is not a property of an interface — it is a property +// of that walk, in the order it makes its calls, with the exclusions it applies +// and the refusals it raises. +func TestFSFeedsTheCheckpointWriter(t *testing.T) { + root := fixture(t) + // A file large enough to span several of the writer's copy buffers, so the + // body path is exercised rather than only the header path. + if err := os.WriteFile(filepath.Join(root, "dir/big.bin"), bytes.Repeat([]byte("x"), 200<<10), 0o644); err != nil { + t.Fatal(err) + } + exclude := checkpoint.DefaultExclusions() + + var buf bytes.Buffer + sent, err := Write(context.Background(), os.DirFS(root), &buf, Limits{Exclude: exclude}) + if err != nil { + t.Fatalf("streaming the workspace: %v", err) + } + + f, err := NewFS(bytes.NewReader(buf.Bytes()), Limits{Exclude: exclude}) + if err != nil { + t.Fatalf("reading the stream: %v", err) + } + if f.Indexed() != int(sent.Entries) { + t.Fatalf("the host indexed %d entries, the guest streamed %d", f.Indexed(), sent.Entries) + } + + store := checkpoint.NewMemoryStore() + keys, err := checkpoint.NewStaticKeyWrapper("test/checkpoint/v1", [32]byte{1, 2, 3}) + if err != nil { + t.Fatal(err) + } + w, err := checkpoint.NewWriter(store, keys, checkpoint.WriterOptions{KeyRef: "test/checkpoint/v1"}) + if err != nil { + t.Fatal(err) + } + c := checkpoint.Context{Workspace: "ws-example", Session: "sess-example", Generation: 1} + res, err := w.Write(context.Background(), c, checkpoint.Source{FS: f}) + if err != nil { + t.Fatalf("checkpointing the stream: %v", err) + } + if res.Manifest.Entries != int64(f.Entries()) { + t.Errorf("the checkpoint carries %d entries, the stream presented %d", res.Manifest.Entries, f.Entries()) + } + + r, err := checkpoint.NewReader(store, keys, checkpoint.ReaderOptions{ + Authorize: func(context.Context, checkpoint.Context, checkpoint.Manifest) error { return nil }, + }) + if err != nil { + t.Fatal(err) + } + if _, err := r.Verify(context.Background(), c); err != nil { + t.Fatalf("verifying the committed checkpoint: %v", err) + } + + target := filepath.Join(t.TempDir(), "restored") + if _, err := r.Restore(context.Background(), c, target); err != nil { + t.Fatalf("restoring the checkpoint: %v", err) + } + compareTrees(t, root, target, exclude) +} + +// compareTrees requires the restored tree to be the source tree, entry for +// entry: the same names in the same order, the same kinds, the same +// permissions, the same content and the same link targets. +func compareTrees(t *testing.T, want, got string, exclude []string) { + t.Helper() + walk := func(dir string) []string { + var out []string + err := fs.WalkDir(os.DirFS(dir), ".", func(name string, d fs.DirEntry, err error) error { + if err != nil { + return err + } + if name == "." { + return nil + } + for _, e := range exclude { + if name == e { + return fs.SkipDir + } + } + info, err := d.Info() + if err != nil { + return err + } + switch { + case d.IsDir(): + out = append(out, "d "+name+" "+info.Mode().Perm().String()) + case d.Type()&fs.ModeSymlink != 0: + target, err := os.Readlink(filepath.Join(dir, name)) + if err != nil { + return err + } + out = append(out, "l "+name+" -> "+target) + default: + body, err := os.ReadFile(filepath.Join(dir, name)) + if err != nil { + return err + } + out = append(out, "f "+name+" "+info.Mode().Perm().String()+" "+ + info.ModTime().UTC().Truncate(1e9).String()+" "+string(body[:min(len(body), 16)])) + } + return nil + }) + if err != nil { + t.Fatalf("walking %s: %v", dir, err) + } + return out + } + a, b := walk(want), walk(got) + if len(a) != len(b) { + t.Fatalf("the restored tree has %d entries, the source has %d:\n%v\n%v", len(b), len(a), b, a) + } + for i := range a { + if a[i] != b[i] { + t.Errorf("entry %d restored as %q, want %q", i, b[i], a[i]) + } + } +} diff --git a/internal/wstream/doc.go b/internal/wstream/doc.go new file mode 100644 index 0000000..cae0caf --- /dev/null +++ b/internal/wstream/doc.go @@ -0,0 +1,58 @@ +// Package wstream carries a microVM session's workspace out of the guest as +// one byte stream, and presents that stream to the host as the file tree the +// checkpoint writer walks. +// +// It exists because the two ends of a cold suspend see different things. The +// guest has the workspace MOUNTED — it is an ext4 image, and the guest's kernel +// is the only one that may parse it, because a host that loop-mounted a +// tenant's filesystem would be running the host kernel's ext4 driver over +// attacker-controlled metadata (see +// docs/design/2026-09-23-workspace-checkpoint-wiring.md §2). The host has the +// checkpoint library, which takes an fs.FS. So the guest writes the tree and +// the host reads it back as a tree. +// +// # The stream +// +// "rainier.wsstream.v1\n" +// index: repeated uvarint(len) record, terminated by a zero length +// record := kind byte ('d' | 'f' | 'l') || name +// tar: archive/tar, the same entries in the same order, bodies for files +// +// The index carries the tree's SHAPE and nothing else — a name and a kind. +// Every other attribute (mode, size, modification time, link target) comes from +// the tar header when the walk reaches that entry, so each fact has one source +// and there is nothing for the two sections to disagree about. +// +// The index exists because fs.WalkDir is not streamable: it asks for every one +// of a directory's children before it visits the first of them, and a child's +// subtree sits in the stream between that child and its next sibling. No +// emission order fixes that, and buffering entry CONTENT to paper over it is +// the one thing this design refuses to do — the host would then hold a +// tenant's workspace in memory, or on its disk in plaintext. So the host +// buffers names (bounded by Limits.MaxEntries and Limits.MaxIndexBytes, fail +// closed) and never a file byte: the tar reader is a one-way cursor the walk +// drags forward, and a skipped body costs nothing to discard. +// +// # The order +// +// Entries are emitted in fs.WalkDir order — depth first, lexical within a +// directory — because that is the order the checkpoint writer walks in. The +// host's cursor only ever moves forward, so a stream in any other order fails +// the suspend rather than producing a checkpoint of a tree nobody had. +// +// # Trust +// +// The guest is the untrusted end. Every limit in Limits is applied by the HOST +// as well as by the guest, because a check performed by the hop before you is a +// check you are trusting. The guest applies them too, so that an oversized tree +// fails inside the session with a sentence about the session rather than as a +// refused stream. +// +// # Errors +// +// No error here carries a path, a file name or a symlink target. These travel +// from sessiond through runnerd to a session's error column, where the tenancy +// specification's §15.1 prohibited-logging list applies. A refusal names the +// entry's ORDINAL and its kind, exactly as the checkpoint package's does, which +// is findable with a local walk and is not a path. +package wstream diff --git a/internal/wstream/format.go b/internal/wstream/format.go new file mode 100644 index 0000000..e448fa2 --- /dev/null +++ b/internal/wstream/format.go @@ -0,0 +1,212 @@ +package wstream + +import ( + "errors" + "fmt" + "path" + "strings" +) + +// Magic is the stream's first bytes. It is a line so that a reader which got +// something else — a JSON error page, a truncated frame, another protocol — +// fails on the first read with a sentence rather than on the tar header with a +// puzzle. +const Magic = "rainier.wsstream.v1\n" + +// The three entry kinds an index record can name. They are bytes rather than +// an enum because they are wire-visible: the guest that writes them ships +// inside a session image and the host that reads them ships with the runner, so +// the two are routinely different builds. +const ( + KindDir byte = 'd' + KindFile byte = 'f' + KindSymlink byte = 'l' +) + +// The limits below are the SAME numbers on both ends. The guest applies them so +// that an oversized workspace fails as a sentence about the session; the host +// applies them because the guest is the untrusted end of this hop. +const ( + // DefaultMaxEntries bounds the index, and therefore the one O(entries) + // structure this design has. Half a million names is a large monorepo with + // its dependencies installed; past that a checkpoint is not the right tool + // and a refusal is better than a host that allocates until it is killed. + DefaultMaxEntries = 500_000 + // DefaultMaxIndexBytes bounds the same structure by BYTES, because + // MaxEntries alone bounds it only if names are short — and it is charged + // what the structure COSTS rather than only what it reads (see + // indexEntryOverhead). 64 MiB is 500,000 entries with names averaging 38 + // bytes, and it is what one concurrent cold suspend may spend on this + // host: the design note's §10 keeps a host-wide bound on concurrent + // checkpoints open, and this number is why it matters. + DefaultMaxIndexBytes = 64 << 20 + // DefaultMaxEntryBytes bounds one file. 8 GiB is under the workspace disk + // (10 GiB) and over any plausible single artifact in one. + DefaultMaxEntryBytes = 8 << 30 + // DefaultMaxTotalBytes bounds the whole stream. The workspace disk is 10 + // GiB; 32 GiB leaves room for a disk that grows without making this the + // check that has to be edited first. + DefaultMaxTotalBytes = 32 << 30 + // maxNameLen and maxNameElement are the filesystem's own limits (PATH_MAX, + // NAME_MAX). A name past them could not be restored anywhere. + maxNameLen = 4096 + maxNameElement = 255 + // maxRecordLen bounds ONE index record before its name is even looked at, + // so a malformed length prefix cannot make a reader allocate. + maxRecordLen = 1 + maxNameLen + // indexEntryOverhead is what the host pays to HOLD one entry, beyond its + // name: a stream record, a tree node, a map entry and a slot in its + // parent's child list. It is a deliberate over-estimate — the point is that + // MaxIndexBytes bounds memory rather than bounding a number that correlates + // with memory. + indexEntryOverhead = 96 +) + +// Limits is what both ends enforce. A zero value means every default; a field +// set to a negative number is refused rather than read as "no limit", because +// "no limit" is not a thing either end of this hop may choose. +type Limits struct { + MaxEntries int64 + MaxIndexBytes int64 + MaxEntryBytes int64 + MaxTotalBytes int64 + + // Exclude is the set of paths that never travel: clean, relative, + // slash-separated, matching themselves and everything under them. The + // caller passes checkpoint.DefaultExclusions() (plus its own policy), and + // both ends apply it — the guest so the bytes never leave the session, the + // host so an excluded entry cannot reach the checkpoint writer even if a + // guest sent one anyway. + Exclude []string +} + +// DefaultLimits is the set a caller gets for passing a zero Limits. +func DefaultLimits() Limits { + return Limits{ + MaxEntries: DefaultMaxEntries, + MaxIndexBytes: DefaultMaxIndexBytes, + MaxEntryBytes: DefaultMaxEntryBytes, + MaxTotalBytes: DefaultMaxTotalBytes, + } +} + +// resolve folds the defaults in and refuses a nonsensical set. It is called +// once per stream, at the top, so that a limit failure is about configuration +// rather than about the tree. +func (l Limits) resolve() (Limits, error) { + d := DefaultLimits() + if l.MaxEntries == 0 { + l.MaxEntries = d.MaxEntries + } + if l.MaxIndexBytes == 0 { + l.MaxIndexBytes = d.MaxIndexBytes + } + if l.MaxEntryBytes == 0 { + l.MaxEntryBytes = d.MaxEntryBytes + } + if l.MaxTotalBytes == 0 { + l.MaxTotalBytes = d.MaxTotalBytes + } + for _, v := range [...]int64{l.MaxEntries, l.MaxIndexBytes, l.MaxEntryBytes, l.MaxTotalBytes} { + if v < 0 { + return Limits{}, fmt.Errorf("%w: a stream limit is negative", ErrLimit) + } + } + for i, e := range l.Exclude { + switch { + case e == "", e != path.Clean(e), path.IsAbs(e), e == ".", strings.HasPrefix(e, ".."): + // The index of the offending element, not its value: an exclusion + // is the caller's own configuration, but it is also a path. + return Limits{}, fmt.Errorf("%w: exclusion %d is not a clean path inside the tree", ErrLimit, i) + } + } + return l, nil +} + +// excluded reports whether name is an excluded path or sits under one. It is +// the same rule the checkpoint package applies, spelled here because this end +// must not depend on the other one having applied it. +func (l Limits) excluded(name string) bool { + for _, e := range l.Exclude { + if name == e || strings.HasPrefix(name, e+"/") { + return true + } + } + return false +} + +// The error vocabulary. Every one of them is a flat sentence with no path, no +// name and no target in it — see the package comment. +var ( + // ErrLimit is a tree, or a stream, past one of the limits above. + ErrLimit = errors.New("wstream: the workspace is past a limit this stream enforces") + + // ErrFormat is a stream that is not one: a bad magic, a length prefix that + // does not parse, an index record with an unknown kind, a tar header where + // the index said something else, an entry out of walk order. + ErrFormat = errors.New("wstream: the workspace stream is malformed") + + // ErrEntry is an entry the format does not carry: a name that is not a + // clean relative path, a symlink that leaves the tree, a device node. + ErrEntry = errors.New("wstream: an entry is not one a workspace stream carries") + + // ErrSource is the guest's own tree refusing to be read: a file that + // changed size under the walk (the workspace was not quiesced), a read + // that failed. + ErrSource = errors.New("wstream: the workspace could not be read as it stands") + + // ErrTruncated is a stream that ended before its tar did. + ErrTruncated = errors.New("wstream: the workspace stream ended early") +) + +// validName is the name rule, applied by the guest on the way out and by the +// host on the way in. It is the checkpoint package's rule, spelled again here +// for the reason that package spells protocol/workspace's again: a check the +// previous hop performed is a check you are trusting. +func validName(name string) error { + switch { + case name == "": + return errors.New("the name is empty") + case len(name) > maxNameLen: + return fmt.Errorf("the name is longer than %d bytes", maxNameLen) + case strings.ContainsRune(name, 0): + return errors.New("the name contains a NUL byte") + case path.IsAbs(name): + return errors.New("the name is absolute") + case name != path.Clean(name): + return errors.New("the name is not a clean relative path") + case name == ".": + return errors.New("the name is the tree root itself") + } + for _, elem := range strings.Split(name, "/") { + switch { + case elem == "", elem == ".", elem == "..": + return errors.New("the name has an empty, \".\" or \"..\" element") + case len(elem) > maxNameElement: + return fmt.Errorf("the name has an element longer than %d bytes", maxNameElement) + } + } + return nil +} + +// validLink is the symlink rule: a target must be relative and must resolve, +// lexically, to somewhere still inside the tree. Lexical on purpose — a +// dangling but contained link is ordinary in a real tree (git worktrees, +// node_modules/.bin) and a stream that refused them could not be taken. +func validLink(name, target string) error { + switch { + case target == "": + return errors.New("the symlink target is empty") + case len(target) > maxNameLen: + return fmt.Errorf("the symlink target is longer than %d bytes", maxNameLen) + case strings.ContainsRune(target, 0): + return errors.New("the symlink target contains a NUL byte") + case path.IsAbs(target): + return errors.New("the symlink target is absolute") + } + resolved := path.Join(path.Dir(name), target) + if resolved == ".." || strings.HasPrefix(resolved, "../") { + return errors.New("the symlink target leaves the tree") + } + return nil +} diff --git a/internal/wstream/fs.go b/internal/wstream/fs.go new file mode 100644 index 0000000..fc8710d --- /dev/null +++ b/internal/wstream/fs.go @@ -0,0 +1,661 @@ +package wstream + +import ( + "archive/tar" + "bufio" + "encoding/binary" + "errors" + "fmt" + "io" + "io/fs" + "path" + "slices" + "strings" + "time" +) + +// FS is the host's end: a workspace stream, presented as the fs.FS the +// checkpoint writer walks. +// +// It is a CURSOR, not a file system. The stream is sequential and one-way, so +// this type can answer a question about the entry the cursor is at or about any +// entry ahead of it, and never about one behind it. That is sound only because +// the walk order is known in advance: the checkpoint writer walks in fs.WalkDir +// order, this type's ReadDir hands back children in the order that produces, +// and the index at the head of the stream is required to be in exactly that +// order — so every question the writer asks is about the entry the cursor is at +// or the next one. Anything else is a stream that was not in walk order, and is +// refused rather than answered wrongly. +// +// What it holds: the tree's shape (a name and a kind per entry, bounded by +// Limits), one tar header, and the underlying reader. NO file content, ever, +// and no buffer proportional to a file. A workspace of any size passes through +// a 32 KiB copy buffer in the writer above it. +// +// It is NOT safe for concurrent use. One walk, one goroutine — which is what +// the checkpoint writer does, and the cursor could not mean anything otherwise. +type FS struct { + lim Limits + + // order is every entry the index named, in stream order, INCLUDING the ones + // this host excludes: the tar carries them too, and the cursor's position is + // a position in the tar. + order []idxEntry + // nodes is the tree the walk sees, which is order minus the exclusions. + // Keyed by name; "." is the root and is synthesized. + nodes map[string]*node + + // src is the stream under the tar reader, kept so that Drain can read the + // rest of it through the SAME byte limit everything else passed. + src io.Reader + tr *tar.Reader + // pos is how many tar entries have been consumed. After seek(i) it is i+1 + // and cur is entry i. + pos int + cur *tar.Header + // err is sticky: a stream that has been found malformed stays malformed, + // and every later call reports the same sentence rather than a second, + // stranger one derived from a desynchronized cursor. + err error +} + +type idxEntry struct { + name string + kind byte + excluded bool +} + +// node is one entry of the tree the walk sees. It holds no content and no +// header: everything but the shape comes from the tar when the cursor arrives. +type node struct { + name string + kind byte + pos int // its position in the stream, which is what the cursor seeks to + children []*node +} + +var ( + _ fs.FS = (*FS)(nil) + _ fs.StatFS = (*FS)(nil) + _ fs.ReadDirFS = (*FS)(nil) + _ fs.ReadLinkFS = (*FS)(nil) +) + +// NewFS reads the stream's header and index, and returns the tree they +// describe. It reads no tar entry: the content follows the index on the same +// reader, and the first seek is what starts consuming it. +// +// r is read to exactly the end of the index and no further, so a caller that +// wants the rest of the stream for itself (to count its bytes, say) wraps r +// rather than sharing it. +func NewFS(r io.Reader, lim Limits) (*FS, error) { + lim, err := lim.resolve() + if err != nil { + return nil, err + } + if r == nil { + return nil, fmt.Errorf("%w: there is no stream to read", ErrFormat) + } + // Bounded before anything is parsed: every limit below is about a number the + // GUEST chose, and the guest is the untrusted end of this hop. + br := bufio.NewReaderSize(&limitReader{r: r, left: lim.MaxTotalBytes}, 64<<10) + + magic := make([]byte, len(Magic)) + if _, err := io.ReadFull(br, magic); err != nil { + return nil, fmt.Errorf("%w: the stream ended before its header", ErrTruncated) + } + if string(magic) != Magic { + return nil, fmt.Errorf("%w: the stream does not begin with this format's header", ErrFormat) + } + + f := &FS{lim: lim, nodes: map[string]*node{}} + f.nodes["."] = &node{name: ".", kind: KindDir, pos: -1} + if err := f.readIndex(br); err != nil { + return nil, err + } + f.src = br + f.tr = tar.NewReader(br) + return f, nil +} + +// Drain reads whatever is left of the stream and discards it. +// +// A walk stops at the last entry it was interested in, which leaves at least +// the tar's two trailing zero blocks unread, and leaves more than that when the +// host excluded entries the guest still sent. Those bytes are not optional to +// consume: the guest is blocked writing them, and a guest that is blocked +// writing cannot send the marker that says its tree was complete. +// +// It reads through this stream's own byte limit, so a guest that keeps writing +// forever is stopped by Limits.MaxTotalBytes — the bound that already governs +// everything else it sends — rather than by a second, smaller number that would +// turn an ordinary exclusion into a failed suspend. +func (f *FS) Drain() error { + if f.err != nil { + return f.err + } + if _, err := io.Copy(io.Discard, f.src); err != nil { + if errors.Is(err, errStreamTooLarge) { + return f.fail(fmt.Errorf("%w: the stream is over %d bytes", ErrLimit, f.lim.MaxTotalBytes)) + } + return f.fail(fmt.Errorf("%w: the stream could not be read to its end", ErrTruncated)) + } + return nil +} + +// readIndex parses the index section and builds the tree, refusing anything +// that is not a strict fs.WalkDir ordering of a valid tree. +// +// The ordering is CHECKED here rather than discovered later, and that is what +// makes the cursor safe: after this returns, the order the walk will ask for +// entries in is known to be the order they arrive in. +func (f *FS) readIndex(br *bufio.Reader) error { + // last is the previous entry's name, which is how depth-first order is + // checked: an entry's parent must be the previous entry or one of its + // ancestors, or the walk jumped sideways into a directory it had left. + last := "" + var indexBytes int64 + for { + n, err := binary.ReadUvarint(br) + if err != nil { + return fmt.Errorf("%w: the stream ended inside its index", ErrTruncated) + } + if n == 0 { + return nil + } + if n > maxRecordLen { + return fmt.Errorf("%w: an index record claims %d bytes", ErrFormat, n) + } + // The record's bytes PLUS what holding it costs. MaxIndexBytes is the + // host's memory bound, and counting only the names would understate it + // by more than half: every entry also carries a stream record, a tree + // node, a map entry and a child pointer. indexEntryOverhead is a + // generous estimate of the four, so the limit bounds what this type + // actually holds rather than what it reads. + indexBytes += int64(n) + indexEntryOverhead + if indexBytes > f.lim.MaxIndexBytes { + return fmt.Errorf("%w: the index is over %d bytes", ErrLimit, f.lim.MaxIndexBytes) + } + if int64(len(f.order)) >= f.lim.MaxEntries { + return fmt.Errorf("%w: the index names more than %d entries", ErrLimit, f.lim.MaxEntries) + } + rec := make([]byte, n) + if _, err := io.ReadFull(br, rec); err != nil { + return fmt.Errorf("%w: the stream ended inside an index record", ErrTruncated) + } + kind, name := rec[0], string(rec[1:]) + ord := int64(len(f.order)) + 1 + switch kind { + case KindDir, KindFile, KindSymlink: + default: + return fmt.Errorf("%w: entry %d names a kind this format does not carry", ErrFormat, ord) + } + if err := validName(name); err != nil { + return fmt.Errorf("%w: entry %d is not a valid name (%s)", ErrEntry, ord, err) + } + if err := f.addIndexEntry(ord, name, kind, last); err != nil { + return err + } + last = name + } +} + +// addIndexEntry places one index record in the tree, or refuses it. +func (f *FS) addIndexEntry(ord int64, name string, kind byte, last string) error { + pos := len(f.order) + excluded := f.lim.excluded(name) + f.order = append(f.order, idxEntry{name: name, kind: kind, excluded: excluded}) + if excluded { + // Dropped from the tree entirely — it is not something the checkpoint + // writer may ever see — but kept in `order`, because the tar carries it + // and the cursor has to walk past it. + return nil + } + if _, dup := f.nodes[name]; dup { + return fmt.Errorf("%w: entry %d is named twice", ErrFormat, ord) + } + parent := path.Dir(name) + p, ok := f.nodes[parent] + if !ok || p.kind != KindDir { + // Either the guest named a child before its directory, or the directory + // is excluded and this entry claims not to be — both are a tree this + // host will not assemble. + return fmt.Errorf("%w: entry %d is not inside a directory the stream already named", ErrFormat, ord) + } + // Depth first: the parent must be the previous entry or one of its + // ancestors. Without this, "a/x, b, a/y" would build a tree whose walk + // order is not the stream's, and the cursor would be asked to go back. + if last != "" && parent != "." && parent != last && !strings.HasPrefix(last, parent+"/") { + return fmt.Errorf("%w: entry %d is out of depth-first order", ErrFormat, ord) + } + // And lexical within the directory, which is the order fs.WalkDir visits + // children in and therefore the order ReadDir must return them in. + if len(p.children) > 0 && path.Base(name) <= path.Base(p.children[len(p.children)-1].name) { + return fmt.Errorf("%w: entry %d is out of lexical order within its directory", ErrFormat, ord) + } + n := &node{name: name, kind: kind, pos: pos} + f.nodes[name] = n + p.children = append(p.children, n) + return nil +} + +// Indexed is how many entries the guest's index named, excluded ones included. +// It is what a caller compares against the count in the guest's end marker: the +// two are statements about the same list, made by the two ends of the hop. +func (f *FS) Indexed() int { return len(f.order) } + +// Entries is how many entries the TREE presents — Indexed minus whatever this +// host excluded. It is smaller than Indexed exactly when a guest sent something +// the host would not carry, which is worth a line in a log. +func (f *FS) Entries() int { return len(f.nodes) - 1 } + +// Open returns the entry at name. For a regular file it is the file's bytes, +// readable exactly once and only while the cursor is on it; for a directory it +// is a listing. +func (f *FS) Open(name string) (fs.File, error) { + n, err := f.lookup("open", name) + if err != nil { + return nil, err + } + if n.name == "." { + return &dirFile{fs: f, node: n, info: rootInfo{}}, nil + } + if err := f.seek(n.pos); err != nil { + return nil, &fs.PathError{Op: "open", Path: name, Err: err} + } + info, err := f.info(n) + if err != nil { + return nil, &fs.PathError{Op: "open", Path: name, Err: err} + } + if n.kind != KindFile { + return &dirFile{fs: f, node: n, info: info}, nil + } + return &streamFile{fs: f, node: n, info: info, at: f.pos, left: f.cur.Size}, nil +} + +// Stat is the entry's metadata, read off the tar header when the cursor +// arrives at it. It does not follow symbolic links — there is nothing to +// follow to, and the writer above wants the link. +func (f *FS) Stat(name string) (fs.FileInfo, error) { return f.Lstat(name) } + +// Lstat is fs.ReadLinkFS's half of the same thing. +func (f *FS) Lstat(name string) (fs.FileInfo, error) { + n, err := f.lookup("stat", name) + if err != nil { + return nil, err + } + if n.name == "." { + return rootInfo{}, nil + } + if err := f.seek(n.pos); err != nil { + return nil, &fs.PathError{Op: "stat", Path: name, Err: err} + } + info, err := f.info(n) + if err != nil { + return nil, &fs.PathError{Op: "stat", Path: name, Err: err} + } + return info, nil +} + +// ReadLink is what makes a symbolic link travel as a link: without it the +// checkpoint writer refuses the entry rather than following it. +func (f *FS) ReadLink(name string) (string, error) { + n, err := f.lookup("readlink", name) + if err != nil { + return "", err + } + if n.kind != KindSymlink { + return "", &fs.PathError{Op: "readlink", Path: name, Err: fs.ErrInvalid} + } + if err := f.seek(n.pos); err != nil { + return "", &fs.PathError{Op: "readlink", Path: name, Err: err} + } + // The cursor is ON this entry, checked here rather than assumed. seek + // refuses to go backwards, but a caller that asked about an entry the + // stream had already passed would otherwise be handed the CURRENT entry's + // target — a symlink checkpointed with somebody else's destination, which + // is the quietest way this design could go wrong. + if f.cur == nil || f.pos != n.pos+1 { + return "", &fs.PathError{Op: "readlink", Path: name, + Err: f.fail(fmt.Errorf("%w: the tree was walked out of the order the stream is in", ErrFormat))} + } + return f.cur.Linkname, nil +} + +// ReadDir lists a directory's children FROM THE INDEX, without moving the +// cursor. That is the whole reason the index exists: fs.WalkDir asks for every +// child of a directory before it visits the first of them, and the answer is +// spread across the entire stream. +func (f *FS) ReadDir(name string) ([]fs.DirEntry, error) { + n, err := f.lookup("readdir", name) + if err != nil { + return nil, err + } + if n.kind != KindDir { + return nil, &fs.PathError{Op: "readdir", Path: name, Err: errors.New("not a directory")} + } + out := make([]fs.DirEntry, 0, len(n.children)) + for _, c := range n.children { + out = append(out, &dirEntry{fs: f, node: c}) + } + // Already lexical by construction (addIndexEntry refuses anything else); + // sorted again because fs.ReadDir's contract is the sort, not our + // construction, and a contract the caller relies on is not a place to save + // a comparison. + slices.SortFunc(out, func(a, b fs.DirEntry) int { return strings.Compare(a.Name(), b.Name()) }) + return out, nil +} + +func (f *FS) lookup(op, name string) (*node, error) { + if f.err != nil { + return nil, &fs.PathError{Op: op, Path: name, Err: f.err} + } + if !fs.ValidPath(name) { + return nil, &fs.PathError{Op: op, Path: name, Err: fs.ErrInvalid} + } + n, ok := f.nodes[name] + if !ok { + return nil, &fs.PathError{Op: op, Path: name, Err: fs.ErrNotExist} + } + return n, nil +} + +// seek moves the cursor to the entry at stream position i, consuming — and +// discarding — everything between. It never moves backwards: a stream is +// one-way, and a request for an entry already passed is a walk that did not +// happen in the order the index promised. +func (f *FS) seek(i int) error { + if f.err != nil { + return f.err + } + if f.pos > i+1 { + return f.fail(fmt.Errorf("%w: the tree was walked out of the order the stream is in", ErrFormat)) + } + for f.pos <= i { + if err := f.next(); err != nil { + return err + } + } + return nil +} + +// next consumes one tar entry and checks it against the index. tar.Reader.Next +// discards whatever is left of the previous entry's body, which is what makes +// skipping an excluded entry free. +func (f *FS) next() error { + hdr, err := f.tr.Next() + if errors.Is(err, io.EOF) { + return f.fail(fmt.Errorf("%w: the stream ended before the entries its index named", ErrTruncated)) + } + if err != nil { + if errors.Is(err, errStreamTooLarge) { + return f.fail(fmt.Errorf("%w: the stream is over %d bytes", ErrLimit, f.lim.MaxTotalBytes)) + } + return f.fail(fmt.Errorf("%w: an entry's header could not be read", ErrFormat)) + } + want := f.order[f.pos] + ord := int64(f.pos) + 1 + // A directory's name is compared with a trailing slash removed: tar's own + // convention allows one and this format's writer emits none, and a + // disagreement about a slash is not worth failing a suspend over. + got := hdr.Name + if hdr.Typeflag == tar.TypeDir { + got = strings.TrimSuffix(got, "/") + } + if got != want.name { + return f.fail(fmt.Errorf("%w: entry %d is not the entry the index named there", ErrFormat, ord)) + } + if err := f.checkHeader(hdr, want, ord); err != nil { + return f.fail(err) + } + f.cur = hdr + f.pos++ + return nil +} + +// checkHeader applies every rule this host enforces on one entry. It runs for +// EXCLUDED entries too: an excluded entry's body is still bytes on this +// connection, and its size still counts against the stream. +func (f *FS) checkHeader(hdr *tar.Header, want idxEntry, ord int64) error { + var kind byte + switch hdr.Typeflag { + case tar.TypeDir: + kind = KindDir + case tar.TypeReg: + kind = KindFile + case tar.TypeSymlink: + kind = KindSymlink + default: + // A hard link, a device node, a FIFO, a GNU extension. The writer of + // this format never produces one, and none of them is a thing a + // workspace checkpoint carries. + return fmt.Errorf("%w: entry %d is not a regular file, directory or symbolic link", ErrEntry, ord) + } + if kind != want.kind { + return fmt.Errorf("%w: entry %d is not the kind the index named", ErrFormat, ord) + } + for k := range hdr.PAXRecords { + // A sparse entry's defining property is a header size far larger than + // the bytes behind it, which is the one thing a bounded reader cannot + // tolerate. archive/tar implements them; this format does not have them. + if strings.HasPrefix(k, "GNU.sparse.") { + return fmt.Errorf("%w: entry %d is a sparse entry", ErrEntry, ord) + } + } + if hdr.Mode < 0 || hdr.Mode&^0o777 != 0 { + // Setuid, setgid and sticky are dropped by the writer of this format + // and by the checkpoint writer above it. One arriving here did not come + // from either. + return fmt.Errorf("%w: entry %d has mode bits outside the permission bits", ErrEntry, ord) + } + switch kind { + case KindFile: + if hdr.Size < 0 || hdr.Size > f.lim.MaxEntryBytes { + return fmt.Errorf("%w: entry %d is larger than %d bytes", ErrLimit, ord, f.lim.MaxEntryBytes) + } + case KindSymlink: + if err := validLink(want.name, hdr.Linkname); err != nil { + return fmt.Errorf("%w: entry %d is a symbolic link that is not allowed (%s)", ErrEntry, ord, err) + } + if hdr.Size != 0 { + // archive/tar treats a symlink header as header-only and would + // re-read the claimed body as the next header, so this is refused + // here rather than left to fail as a mysterious format error one + // entry later. + return fmt.Errorf("%w: entry %d is a symbolic link with a body", ErrFormat, ord) + } + case KindDir: + if hdr.Size != 0 { + return fmt.Errorf("%w: entry %d is a directory with a body", ErrFormat, ord) + } + } + return nil +} + +// fail records a stream as unusable. Everything afterwards reports the same +// sentence: a cursor that has lost its place cannot produce a second, more +// specific diagnosis, only a stranger one. +func (f *FS) fail(err error) error { + if f.err == nil { + f.err = err + } + return f.err +} + +// info is the FileInfo for the entry the cursor is on. +func (f *FS) info(n *node) (fs.FileInfo, error) { + if f.cur == nil || f.pos != n.pos+1 { + return nil, f.fail(fmt.Errorf("%w: the tree was walked out of the order the stream is in", ErrFormat)) + } + return &entryInfo{hdr: f.cur, name: path.Base(n.name), kind: n.kind}, nil +} + +// --------------------------------------------------------------------------- +// the fs.File and fs.DirEntry implementations +// --------------------------------------------------------------------------- + +// streamFile is one regular file's bytes, read straight off the tar reader. +// There is no buffer: what the caller reads is what the connection delivers. +type streamFile struct { + fs *FS + node *node + info fs.FileInfo + // at is the cursor position this file was opened at. A read after the + // cursor has moved on is refused rather than served from whatever the + // stream is now pointing at. + at int + left int64 +} + +func (s *streamFile) Stat() (fs.FileInfo, error) { return s.info, nil } +func (s *streamFile) Close() error { return nil } + +func (s *streamFile) Read(p []byte) (int, error) { + if s.fs.err != nil { + return 0, s.fs.err + } + if s.fs.pos != s.at { + return 0, s.fs.fail(fmt.Errorf("%w: an entry was read after the stream had moved past it", ErrFormat)) + } + if s.left == 0 { + return 0, io.EOF + } + if int64(len(p)) > s.left { + p = p[:s.left] + } + n, err := s.fs.tr.Read(p) + s.left -= int64(n) + if errors.Is(err, io.EOF) && s.left > 0 { + return n, s.fs.fail(fmt.Errorf("%w: an entry's body ended before the length its header promised", ErrTruncated)) + } + if err != nil && !errors.Is(err, io.EOF) { + if errors.Is(err, errStreamTooLarge) { + return n, s.fs.fail(fmt.Errorf("%w: the stream is over %d bytes", ErrLimit, s.fs.lim.MaxTotalBytes)) + } + return n, s.fs.fail(fmt.Errorf("%w: an entry's body could not be read", ErrTruncated)) + } + return n, nil +} + +// dirFile is a directory (and, for completeness, a symbolic link) opened as a +// file. Reading one is an error rather than an empty file, which is what an +// operating system does and what keeps a caller that opened the wrong thing +// from silently checkpointing nothing. +type dirFile struct { + fs *FS + node *node + info fs.FileInfo + read int +} + +func (d *dirFile) Stat() (fs.FileInfo, error) { return d.info, nil } +func (d *dirFile) Close() error { return nil } + +func (d *dirFile) Read([]byte) (int, error) { + return 0, &fs.PathError{Op: "read", Path: d.node.name, Err: errors.New("is a directory")} +} + +func (d *dirFile) ReadDir(n int) ([]fs.DirEntry, error) { + all, err := d.fs.ReadDir(d.node.name) + if err != nil { + return nil, err + } + if n <= 0 { + d.read = len(all) + return all, nil + } + if d.read >= len(all) { + return nil, io.EOF + } + end := min(d.read+n, len(all)) + out := all[d.read:end] + d.read = end + return out, nil +} + +// dirEntry is one child in a listing. Its Info is LAZY — it moves the cursor — +// because the walk asks for a directory's children long before it visits them, +// and a listing that stat'ed each child would have to read the whole stream to +// answer. +type dirEntry struct { + fs *FS + node *node +} + +func (d *dirEntry) Name() string { return path.Base(d.node.name) } +func (d *dirEntry) IsDir() bool { return d.node.kind == KindDir } + +func (d *dirEntry) Type() fs.FileMode { + switch d.node.kind { + case KindDir: + return fs.ModeDir + case KindSymlink: + return fs.ModeSymlink + default: + return 0 + } +} + +func (d *dirEntry) Info() (fs.FileInfo, error) { return d.fs.Lstat(d.node.name) } + +// entryInfo is a tar header, as an fs.FileInfo. Nothing here is computed: the +// mode, the size and the modification time are what the header said, so that +// what the checkpoint records is what the guest sent. +type entryInfo struct { + hdr *tar.Header + name string + kind byte +} + +func (e *entryInfo) Name() string { return e.name } +func (e *entryInfo) Size() int64 { return e.hdr.Size } +func (e *entryInfo) ModTime() time.Time { return e.hdr.ModTime } +func (e *entryInfo) IsDir() bool { return e.kind == KindDir } +func (e *entryInfo) Sys() any { return nil } + +func (e *entryInfo) Mode() fs.FileMode { + mode := fs.FileMode(e.hdr.Mode) & fs.ModePerm + switch e.kind { + case KindDir: + mode |= fs.ModeDir + case KindSymlink: + mode |= fs.ModeSymlink + } + return mode +} + +// rootInfo is the tree's root, which the stream does not carry an entry for — +// the root is the destination rather than an entry, in this format as in the +// checkpoint one. Its mode is a plain 0755 directory, and nothing reads it: the +// checkpoint writer's walk skips ".". +type rootInfo struct{} + +func (rootInfo) Name() string { return "." } +func (rootInfo) Size() int64 { return 0 } +func (rootInfo) Mode() fs.FileMode { return fs.ModeDir | 0o755 } +func (rootInfo) ModTime() time.Time { return time.Unix(0, 0).UTC() } +func (rootInfo) IsDir() bool { return true } +func (rootInfo) Sys() any { return nil } + +// limitReader bounds the whole stream, headers and bodies alike, before any of +// it is parsed. It is the outermost thing on this path on purpose: a guest that +// sends bytes forever must be stopped by a counter that no parser's opinion can +// move. +type limitReader struct { + r io.Reader + left int64 +} + +var errStreamTooLarge = errors.New("wstream: the stream is over its byte limit") + +func (l *limitReader) Read(p []byte) (int, error) { + if l.left <= 0 { + return 0, errStreamTooLarge + } + if int64(len(p)) > l.left { + p = p[:l.left] + } + n, err := l.r.Read(p) + l.left -= int64(n) + return n, err +} diff --git a/internal/wstream/fs_test.go b/internal/wstream/fs_test.go new file mode 100644 index 0000000..def6265 --- /dev/null +++ b/internal/wstream/fs_test.go @@ -0,0 +1,551 @@ +package wstream + +import ( + "archive/tar" + "bytes" + "context" + "encoding/binary" + "errors" + "io" + "io/fs" + "os" + "path/filepath" + "strings" + "testing" + "testing/fstest" + "time" +) + +// The host's end of the stream. Every test here reads a stream the writer in +// this package produced, or one built by hand to be something the writer never +// produces — which is the half that matters, because the guest is the untrusted +// end of this hop. + +func mustStream(t *testing.T, fsys fs.FS, lim Limits) []byte { + t.Helper() + var buf bytes.Buffer + if _, err := Write(context.Background(), fsys, &buf, lim); err != nil { + t.Fatalf("writing the fixture stream: %v", err) + } + return buf.Bytes() +} + +// walkLikeTheWriter walks f the way checkpoint.Writer's own walk does: it asks +// every entry for its info, reads every regular file, and reads every symbolic +// link's target. +// +// The distinction matters more than it looks. fs.WalkDir alone touches nothing +// but the index — ReadDir is answered from the tree and DirEntry.Info is lazy — +// so a test that only walked would consume no tar entry at all and would prove +// nothing about the cursor or about any rule applied when it moves. +func walkLikeTheWriter(f *FS) error { + return fs.WalkDir(f, ".", func(name string, d fs.DirEntry, err error) error { + if err != nil { + return err + } + if name == "." { + return nil + } + if _, err := d.Info(); err != nil { + return err + } + switch { + case d.IsDir(): + return nil + case d.Type()&fs.ModeSymlink != 0: + _, err := fs.ReadLink(f, name) + return err + default: + _, err := fs.ReadFile(f, name) + return err + } + }) +} + +func newFS(t *testing.T, raw []byte, lim Limits) *FS { + t.Helper() + f, err := NewFS(bytes.NewReader(raw), lim) + if err != nil { + t.Fatalf("NewFS: %v", err) + } + return f +} + +// TestFSPresentsTheTreeAWalkExpects is the round trip: what fs.WalkDir sees +// over a stream is what it would have seen over the guest's own directory. +func TestFSPresentsTheTreeAWalkExpects(t *testing.T) { + root := fixture(t) + f := newFS(t, mustStream(t, os.DirFS(root), Limits{Exclude: []string{".rainier"}}), Limits{Exclude: []string{".rainier"}}) + + var visited []string + err := fs.WalkDir(f, ".", func(name string, d fs.DirEntry, err error) error { + if err != nil { + return err + } + if name == "." { + return nil + } + info, err := d.Info() + if err != nil { + return err + } + switch { + case d.IsDir(): + visited = append(visited, "d "+name+" "+info.Mode().String()) + case d.Type()&fs.ModeSymlink != 0: + target, err := fs.ReadLink(f, name) + if err != nil { + return err + } + visited = append(visited, "l "+name+" -> "+target) + default: + body, err := fs.ReadFile(f, name) + if err != nil { + return err + } + visited = append(visited, "f "+name+" "+string(body)+" "+info.Mode().Perm().String()) + } + return nil + }) + if err != nil { + t.Fatalf("walking the stream: %v", err) + } + want := []string{ + "f a.txt alpha -rw-r--r--", + "d dir drwxr-xr-x", + "f dir/b.txt bravo -rw-------", + "l link -> dir/b.txt", + } + if len(visited) != len(want) { + t.Fatalf("the walk saw %v, want %v", visited, want) + } + for i := range want { + if visited[i] != want[i] { + t.Fatalf("the walk saw %v, want %v", visited, want) + } + } + if f.Indexed() != 4 || f.Entries() != 4 { + t.Errorf("the stream indexed %d entries and presents %d, want 4 and 4", f.Indexed(), f.Entries()) + } +} + +// TestFSCarriesModificationTimes: the checkpoint's tree digest commits to them, +// so a restored workspace is only the same workspace if they survive this hop. +func TestFSCarriesModificationTimes(t *testing.T) { + root := t.TempDir() + if err := os.WriteFile(filepath.Join(root, "a.txt"), []byte("alpha"), 0o644); err != nil { + t.Fatal(err) + } + when := time.Date(2026, 9, 23, 10, 30, 0, 0, time.UTC) + if err := os.Chtimes(filepath.Join(root, "a.txt"), when, when); err != nil { + t.Fatal(err) + } + f := newFS(t, mustStream(t, os.DirFS(root), Limits{}), Limits{}) + info, err := fs.Stat(f, "a.txt") + if err != nil { + t.Fatal(err) + } + if !info.ModTime().Equal(when) { + t.Errorf("a.txt's modification time came back as %v, want %v", info.ModTime(), when) + } +} + +// TestFSExcludesWhatTheHostOwns. The host applies the exclusions too, because a +// check the previous hop performed is a check you are trusting: an entry the +// guest sent anyway is dropped from the tree AND skipped in the stream, which +// is what keeps the cursor in step. +func TestFSExcludesWhatTheHostOwns(t *testing.T) { + // A stream written with NO exclusions, read by a host that has them. + root := fixture(t) + f := newFS(t, mustStream(t, os.DirFS(root), Limits{}), Limits{Exclude: []string{".rainier"}}) + + if f.Indexed() != 6 || f.Entries() != 4 { + t.Fatalf("the stream indexed %d entries and presents %d, want 6 and 4", f.Indexed(), f.Entries()) + } + if _, err := fs.Stat(f, ".rainier"); !errors.Is(err, fs.ErrNotExist) { + t.Errorf("an excluded directory is visible: %v", err) + } + // And the tree after it is still readable, which is the part a naive + // implementation gets wrong: skipping an entry means skipping its BODY too. + if body, err := fs.ReadFile(f, "dir/b.txt"); err != nil || string(body) != "bravo" { + t.Errorf("reading past an excluded entry gave %q, %v", body, err) + } +} + +// TestFSRefusesAStreamThatIsNotOne. +func TestFSRefusesAStreamThatIsNotOne(t *testing.T) { + cases := []struct { + name string + raw []byte + want error + }{ + {"empty", nil, ErrTruncated}, + {"another protocol", []byte("{\"kind\":\"suspend_ready\"}\n"), ErrFormat}, + {"a header and nothing else", []byte(Magic), ErrTruncated}, + {"an index that never ends", append([]byte(Magic), 0x05, 'f', 'a', '.', 't'), ErrTruncated}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + if _, err := NewFS(bytes.NewReader(tc.raw), Limits{}); !errors.Is(err, tc.want) { + t.Fatalf("NewFS gave %v, want %v", err, tc.want) + } + }) + } +} + +// index builds an index section by hand, which is how every "a guest that does +// not behave" test below states its case. +func index(entries ...string) []byte { + out := []byte(Magic) + var hdr [binary.MaxVarintLen64]byte + for _, e := range entries { + n := binary.PutUvarint(hdr[:], uint64(len(e))) + out = append(out, hdr[:n]...) + out = append(out, e...) + } + return append(out, 0) +} + +// tarball appends a tar section carrying exactly these headers. +func tarball(t *testing.T, prefix []byte, headers ...*tar.Header) []byte { + t.Helper() + var buf bytes.Buffer + buf.Write(prefix) + tw := tar.NewWriter(&buf) + for _, h := range headers { + if err := tw.WriteHeader(h); err != nil { + t.Fatal(err) + } + if h.Size > 0 { + if _, err := tw.Write(bytes.Repeat([]byte("x"), int(h.Size))); err != nil { + t.Fatal(err) + } + } + } + if err := tw.Close(); err != nil { + t.Fatal(err) + } + return buf.Bytes() +} + +// TestFSRefusesAnIndexThatIsNotATree is the path-traversal table. Every one of +// these is a name a guest can put on the wire and none of them may become a +// tree entry, because the checkpoint they would land in is restored into a +// directory on a host. +func TestFSRefusesAnIndexThatIsNotATree(t *testing.T) { + cases := []struct { + name string + entry string + want error + }{ + {"an absolute path", "f/etc/shadow", ErrEntry}, + {"a parent escape", "f../../etc/shadow", ErrEntry}, + {"a dot-dot element", "fa/../../b", ErrEntry}, + {"an unclean path", "fa//b", ErrEntry}, + {"the root itself", "f.", ErrEntry}, + {"an empty name", "f", ErrEntry}, + {"a NUL", "fa\x00b", ErrEntry}, + {"a kind this format does not carry", "ba.txt", ErrFormat}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + if _, err := NewFS(bytes.NewReader(index(tc.entry)), Limits{}); !errors.Is(err, tc.want) { + t.Fatalf("NewFS gave %v, want %v", err, tc.want) + } + }) + } +} + +// TestFSRefusesAnIndexOutOfWalkOrder. The cursor is one-way, so an index in any +// order but the walk's is a stream whose entries would be asked for backwards. +// Refused at the head, where there is something to say, rather than half way +// through a checkpoint. +func TestFSRefusesAnIndexOutOfWalkOrder(t *testing.T) { + cases := []struct { + name string + entries []string + }{ + {"a child before its directory", []string{"fdir/b.txt", "ddir"}}, + {"a directory re-entered later", []string{"ddir", "fdir/a", "fz", "fdir/b"}}, + {"children out of lexical order", []string{"fb.txt", "fa.txt"}}, + {"the same name twice", []string{"fa.txt", "fa.txt"}}, + {"a child of a file", []string{"fa.txt", "fa.txt/b"}}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + if _, err := NewFS(bytes.NewReader(index(tc.entries...)), Limits{}); !errors.Is(err, ErrFormat) { + t.Fatalf("NewFS gave %v, want ErrFormat", err) + } + }) + } +} + +// TestFSRefusesEachLimit, on the host's side of the hop this time. +func TestFSRefusesEachLimit(t *testing.T) { + root := fixture(t) + raw := mustStream(t, os.DirFS(root), Limits{Exclude: []string{".rainier"}}) + + t.Run("entries", func(t *testing.T) { + if _, err := NewFS(bytes.NewReader(raw), Limits{MaxEntries: 2}); !errors.Is(err, ErrLimit) { + t.Fatalf("NewFS gave %v, want ErrLimit", err) + } + }) + t.Run("index bytes", func(t *testing.T) { + if _, err := NewFS(bytes.NewReader(raw), Limits{MaxIndexBytes: 4}); !errors.Is(err, ErrLimit) { + t.Fatalf("NewFS gave %v, want ErrLimit", err) + } + }) + t.Run("a record longer than any name", func(t *testing.T) { + // A length prefix claiming more than a name could ever be, which is + // what an allocation attack looks like from here. + raw := append([]byte(Magic), 0xff, 0xff, 0xff, 0xff, 0x0f) + if _, err := NewFS(bytes.NewReader(raw), Limits{}); !errors.Is(err, ErrFormat) { + t.Fatalf("NewFS gave %v, want ErrFormat", err) + } + }) + t.Run("one entry's bytes", func(t *testing.T) { + f := newFS(t, raw, Limits{MaxEntryBytes: 2, Exclude: []string{".rainier"}}) + err := walkLikeTheWriter(f) + if !errors.Is(err, ErrLimit) { + t.Fatalf("the walk gave %v, want ErrLimit", err) + } + }) + t.Run("total bytes", func(t *testing.T) { + if _, err := NewFS(bytes.NewReader(raw), Limits{MaxTotalBytes: 8}); err == nil { + t.Fatal("a stream past the total limit was accepted") + } + }) +} + +// TestFSRefusesAnEntryTheIndexDidNotName is the cross-check between the two +// sections: a guest that named one tree and sent another must not be able to +// checkpoint the one the host never validated. +func TestFSRefusesTarThatDisagreesWithTheIndex(t *testing.T) { + cases := []struct { + name string + raw []byte + want error + }{ + { + "a different name", + tarball(t, index("fa.txt"), &tar.Header{Typeflag: tar.TypeReg, Name: "b.txt", Mode: 0o644}), + ErrFormat, + }, + { + "a different kind", + tarball(t, index("fa.txt"), &tar.Header{Typeflag: tar.TypeDir, Name: "a.txt", Mode: 0o755}), + ErrFormat, + }, + { + "a hard link", + tarball(t, index("fa.txt"), &tar.Header{Typeflag: tar.TypeLink, Name: "a.txt", Linkname: "b", Mode: 0o644}), + ErrEntry, + }, + { + "a device node", + tarball(t, index("fa.txt"), &tar.Header{Typeflag: tar.TypeChar, Name: "a.txt", Mode: 0o644}), + ErrEntry, + }, + { + "a setuid bit", + tarball(t, index("fa.txt"), &tar.Header{Typeflag: tar.TypeReg, Name: "a.txt", Mode: 0o4755}), + ErrEntry, + }, + { + "a symlink out of the tree", + tarball(t, index("lcreds"), &tar.Header{Typeflag: tar.TypeSymlink, Name: "creds", Linkname: "/etc/shadow"}), + ErrEntry, + }, + { + "a symlink climbing out of the tree", + tarball(t, index("ldir", "ldir/creds"), &tar.Header{Typeflag: tar.TypeSymlink, Name: "dir", Linkname: "x"}), + ErrFormat, // the index says two symlinks, one of which has children + }, + { + "nothing at all", + index("fa.txt"), + ErrTruncated, + }, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + f, err := NewFS(bytes.NewReader(tc.raw), Limits{}) + if err != nil { + if !errors.Is(err, tc.want) { + t.Fatalf("NewFS gave %v, want %v", err, tc.want) + } + return + } + err = walkLikeTheWriter(f) + if !errors.Is(err, tc.want) { + t.Fatalf("the walk gave %v, want %v", err, tc.want) + } + }) + } +} + +// TestFSRefusesASecondReadOfAnEntryTheStreamHasPassed. The cursor is the whole +// design; a caller that went back would otherwise be served whatever the stream +// is pointing at now, which is another file's bytes. +func TestFSRefusesAReadAfterTheCursorMovedOn(t *testing.T) { + fsys := fstest.MapFS{ + "a.txt": &fstest.MapFile{Data: []byte("alpha"), Mode: 0o644}, + "b.txt": &fstest.MapFile{Data: []byte("bravo"), Mode: 0o644}, + } + f := newFS(t, mustStream(t, fsys, Limits{}), Limits{}) + + first, err := f.Open("a.txt") + if err != nil { + t.Fatal(err) + } + if _, err := fs.ReadFile(f, "b.txt"); err != nil { + t.Fatal(err) + } + if _, err := io.ReadAll(first); !errors.Is(err, ErrFormat) { + t.Fatalf("reading a passed entry gave %v, want ErrFormat", err) + } + // And the stream stays refused: a cursor that has lost its place cannot + // produce a better answer to the next question either. + if _, err := fs.Stat(f, "b.txt"); !errors.Is(err, ErrFormat) { + t.Fatalf("a later call gave %v, want the same refusal", err) + } +} + +// TestFSRefusesAQuestionAboutAnEntryTheStreamHasPassed is the cursor's other +// half, and the quietest way this design could go wrong: a caller that asked +// about an earlier entry must not be answered from the CURRENT one. A ReadLink +// served that way would checkpoint a symbolic link with somebody else's +// destination, in a checkpoint that then verified. +func TestFSRefusesAQuestionAboutAnEntryTheStreamHasPassed(t *testing.T) { + fsys := fstest.MapFS{ + "a-link": &fstest.MapFile{Data: []byte("first-target"), Mode: fs.ModeSymlink | 0o777}, + "b-link": &fstest.MapFile{Data: []byte("second-target"), Mode: fs.ModeSymlink | 0o777}, + "c.txt": &fstest.MapFile{Data: []byte("charlie"), Mode: 0o644}, + } + raw := mustStream(t, fsys, Limits{}) + + t.Run("readlink", func(t *testing.T) { + f := newFS(t, raw, Limits{}) + if target, err := fs.ReadLink(f, "b-link"); err != nil || target != "second-target" { + t.Fatalf("reading the second link gave %q, %v", target, err) + } + target, err := fs.ReadLink(f, "a-link") + if err == nil { + t.Fatalf("reading a passed link returned %q", target) + } + if !errors.Is(err, ErrFormat) { + t.Fatalf("reading a passed link gave %v, want ErrFormat", err) + } + }) + t.Run("stat", func(t *testing.T) { + f := newFS(t, raw, Limits{}) + if _, err := fs.Stat(f, "c.txt"); err != nil { + t.Fatal(err) + } + if _, err := fs.Stat(f, "a-link"); !errors.Is(err, ErrFormat) { + t.Fatalf("stat of a passed entry gave %v, want ErrFormat", err) + } + }) + t.Run("open", func(t *testing.T) { + f := newFS(t, raw, Limits{}) + if _, err := fs.ReadFile(f, "c.txt"); err != nil { + t.Fatal(err) + } + if _, err := f.Open("a-link"); !errors.Is(err, ErrFormat) { + t.Fatalf("opening a passed entry gave %v, want ErrFormat", err) + } + }) +} + +// TestFSDrainConsumesWhatTheWalkDidNotWant. A walk stops at the last entry it +// was interested in, and the guest is blocked writing whatever follows — the +// tar's trailing blocks at least, and everything the HOST excluded that sorts +// after that. A guest blocked writing cannot send the marker that says its tree +// was complete, so those bytes are not optional to consume. +func TestFSDrainConsumesWhatTheWalkDidNotWant(t *testing.T) { + fsys := fstest.MapFS{ + "a.txt": &fstest.MapFile{Data: []byte("alpha"), Mode: 0o644}, + "node_modules/x": &fstest.MapFile{Data: bytes.Repeat([]byte("x"), 128<<10), Mode: 0o644}, + } + // The guest sends everything; this host excludes a directory the guest + // never heard of, and it is the LAST thing in the stream. + raw := mustStream(t, fsys, Limits{}) + f := newFS(t, raw, Limits{Exclude: []string{"node_modules"}}) + if err := walkLikeTheWriter(f); err != nil { + t.Fatalf("walking the stream: %v", err) + } + if err := f.Drain(); err != nil { + t.Fatalf("draining the rest of the stream: %v", err) + } + // And the drain is bounded by the stream's own byte limit rather than by a + // second, smaller number: the same stream against a total limit it exceeds + // is refused rather than silently half-read. + tight := newFS(t, raw, Limits{Exclude: []string{"node_modules"}, MaxTotalBytes: 64 << 10}) + err := walkLikeTheWriter(tight) + if err == nil { + err = tight.Drain() + } + if !errors.Is(err, ErrLimit) { + t.Fatalf("draining past the total limit gave %v, want ErrLimit", err) + } +} + +// TestFSRefusesAnEntryThatEndsEarly: a stream cut off inside a file's body. +func TestFSRefusesABodyThatEndsEarly(t *testing.T) { + fsys := fstest.MapFS{"a.txt": &fstest.MapFile{Data: bytes.Repeat([]byte("x"), 4096), Mode: 0o644}} + raw := mustStream(t, fsys, Limits{}) + f := newFS(t, raw[:len(raw)-2048], Limits{}) + if _, err := fs.ReadFile(f, "a.txt"); !errors.Is(err, ErrTruncated) { + t.Fatalf("reading a truncated body gave %v, want ErrTruncated", err) + } +} + +// TestFSHoldsNoContent is the bounded-memory claim, as a claim about the type +// rather than about a measurement: the only per-entry state is a name and a +// kind, so a stream of any SIZE costs the same as a stream of any other, for +// the same number of entries. +func TestFSHoldsNoContent(t *testing.T) { + big := 8 << 20 + fsys := fstest.MapFS{"big.bin": &fstest.MapFile{Data: bytes.Repeat([]byte("x"), big), Mode: 0o644}} + raw := mustStream(t, fsys, Limits{}) + + before := heapInUse() + f := newFS(t, raw, Limits{}) + after := heapInUse() + // The index of one entry, and nothing that grew with the 8 MiB behind it. + if grew := int64(after) - int64(before); grew > int64(big)/4 { + t.Fatalf("building the tree grew the heap by %d bytes for an %d byte stream", grew, big) + } + entry, err := f.Open("big.bin") + if err != nil { + t.Fatal(err) + } + if n, err := io.Copy(io.Discard, entry); err != nil || n != int64(big) { + t.Fatalf("reading the entry gave %d bytes, %v", n, err) + } +} + +// TestFSErrorsNameNoPath: these travel to a session's error column. +func TestFSErrorsNameNoPath(t *testing.T) { + raw := tarball(t, index("fsecret-project/api-key.txt"), + &tar.Header{Typeflag: tar.TypeChar, Name: "secret-project/api-key.txt", Mode: 0o644}) + f, err := NewFS(bytes.NewReader(raw), Limits{}) + if err == nil { + err = walkLikeTheWriter(f) + } + if err == nil { + t.Fatal("the stream was accepted") + } + // fs.PathError carries the path by construction, so the assertion is about + // the sentence this package produced, which is what a caller logs. + var pe *fs.PathError + msg := err.Error() + if errors.As(err, &pe) { + msg = pe.Err.Error() + } + for _, leak := range []string{"secret-project", "api-key"} { + if strings.Contains(msg, leak) { + t.Errorf("the refusal quotes %q: %q", leak, msg) + } + } +} diff --git a/internal/wstream/fuzz_test.go b/internal/wstream/fuzz_test.go new file mode 100644 index 0000000..5598cdb --- /dev/null +++ b/internal/wstream/fuzz_test.go @@ -0,0 +1,93 @@ +package wstream + +import ( + "archive/tar" + "bytes" + "context" + "errors" + "io/fs" + "os" + "testing" + "testing/fstest" +) + +// FuzzFS is the host's parser against bytes a guest chose. +// +// The guest is the untrusted end of this hop and everything it sends is parsed +// before anything is authenticated: the magic, the index's length prefixes and +// names, every tar header, every body length. So the property under test is the +// one a suspend depends on — a stream this parser cannot make sense of is +// REFUSED, with one of this package's own sentinels, rather than panicking, +// hanging, or returning a tree that is not what the bytes said. +// +// The limits are deliberately tiny: a fuzzer that could ask for 8 GiB would +// spend its time allocating instead of finding parses. +func FuzzFS(f *testing.F) { + small := Limits{MaxEntries: 64, MaxIndexBytes: 4 << 10, MaxEntryBytes: 1 << 16, MaxTotalBytes: 1 << 20} + + f.Add([]byte(nil)) + f.Add([]byte(Magic)) + f.Add(index("fa.txt")) + f.Add(index("ddir", "fdir/a.txt", "llink")) + f.Add(tarballNoT(index("fa.txt"), &tar.Header{Typeflag: tar.TypeReg, Name: "a.txt", Mode: 0o644, Size: 3})) + f.Add(tarballNoT(index("ddir", "fdir/a.txt"), + &tar.Header{Typeflag: tar.TypeDir, Name: "dir", Mode: 0o755}, + &tar.Header{Typeflag: tar.TypeReg, Name: "dir/a.txt", Mode: 0o644, Size: 5})) + // A whole stream the writer in this package produced, which is the shape + // every mutation starts from. + var buf bytes.Buffer + if _, err := Write(context.Background(), fstest.MapFS{ + "a.txt": &fstest.MapFile{Data: []byte("alpha"), Mode: 0o644}, + "dir/b.txt": &fstest.MapFile{Data: []byte("bravo"), Mode: 0o600}, + "link": &fstest.MapFile{Data: []byte("dir/b.txt"), Mode: fs.ModeSymlink | 0o777}, + }, &buf, small); err == nil { + f.Add(buf.Bytes()) + } + + f.Fuzz(func(t *testing.T, raw []byte) { + fsys, err := NewFS(bytes.NewReader(raw), small) + if err != nil { + requireSentinel(t, err) + return + } + if err := walkLikeTheWriter(fsys); err != nil { + requireSentinel(t, err) + } + }) +} + +// requireSentinel is the property: every refusal is one of this package's own, +// so that a caller can tell "this stream is not one" from "this host is broken" +// — and so that no error escapes with a message nobody wrote. +func requireSentinel(t *testing.T, err error) { + t.Helper() + for _, s := range []error{ErrFormat, ErrEntry, ErrLimit, ErrSource, ErrTruncated, fs.ErrNotExist, fs.ErrInvalid, os.ErrDeadlineExceeded} { + if errors.Is(err, s) { + return + } + } + t.Fatalf("a refusal that is not one of this package's: %#v (%v)", err, err) +} + +// tarballNoT is tarball without a *testing.T, for the seed corpus: a seed that +// cannot be built is a seed that is silently dropped rather than a test that +// fails, which is the right trade for a corpus entry. +func tarballNoT(prefix []byte, headers ...*tar.Header) []byte { + var buf bytes.Buffer + buf.Write(prefix) + tw := tar.NewWriter(&buf) + for _, h := range headers { + if tw.WriteHeader(h) != nil { + return prefix + } + if h.Size > 0 { + if _, err := tw.Write(bytes.Repeat([]byte("x"), int(h.Size))); err != nil { + return prefix + } + } + } + if tw.Close() != nil { + return prefix + } + return buf.Bytes() +} diff --git a/internal/wstream/heap_test.go b/internal/wstream/heap_test.go new file mode 100644 index 0000000..dc3670a --- /dev/null +++ b/internal/wstream/heap_test.go @@ -0,0 +1,13 @@ +package wstream + +import "runtime" + +// heapInUse is a settled reading of the live heap. The garbage collector runs +// first because what is being measured is what a structure HOLDS, not what its +// construction allocated on the way. +func heapInUse() uint64 { + runtime.GC() + var m runtime.MemStats + runtime.ReadMemStats(&m) + return m.HeapAlloc +} diff --git a/internal/wstream/write.go b/internal/wstream/write.go new file mode 100644 index 0000000..8021f53 --- /dev/null +++ b/internal/wstream/write.go @@ -0,0 +1,403 @@ +package wstream + +import ( + "archive/tar" + "bytes" + "context" + "crypto/sha256" + "encoding/binary" + "errors" + "fmt" + "hash" + "io" + "io/fs" + "os" + "time" +) + +// Report is what one stream carried: the counts the guest states in its end +// marker, and the host checks against what it received. +// +// Skipped is the entries that are neither a directory, a regular file nor a +// symbolic link — a socket a dev server left behind, a FIFO, a device node. +// They are skipped and COUNTED rather than refused, for the reason the +// checkpoint package skips them: a stray socket must not be able to defeat the +// durability barrier, and a count in the end marker means it is never silent. +type Report struct { + Entries int64 + Bytes int64 + Skipped int64 +} + +// Write streams fsys to w in the format this package's comment describes, and +// returns what it carried. +// +// It walks the tree TWICE — once for the index, once for the bodies — rather +// than buffering the index, because a guest holding its own workspace's shape +// in memory is the same O(entries) cost this design refuses on the host, paid +// inside a VM with less memory. The workspace is quiesced by the time this +// runs (the agent is stopped, the execs are dead, the disks are synced), so the +// two walks see the same tree; a tree that changed between them is caught here +// rather than left for the host, by a digest over the names each pass visited. +// +// Nothing here follows a symbolic link. fs.WalkDir does not, fs.ReadLink reads +// the link itself, and a link whose target leaves the tree is refused rather +// than resolved — which is what keeps a workspace containing +// "creds -> /etc/shadow" from streaming /etc/shadow out of the guest. +// The Report is filled in as far as the stream got, on the failure paths as +// well as on the successful one: the counts travel to the host in the end +// marker, and "it failed after 900 entries and 40 MB" is the only thing a +// person can act on in a diagnostic that may not name a file. +func Write(ctx context.Context, fsys fs.FS, w io.Writer, lim Limits) (Report, error) { + cw := &countWriter{w: &sinkWriter{w: w}} + var entries, skipped int64 + rep := func() Report { return Report{Entries: entries, Bytes: cw.n, Skipped: skipped} } + + lim, err := lim.resolve() + if err != nil { + return rep(), err + } + if fsys == nil { + return rep(), fmt.Errorf("%w: there is no tree to stream", ErrSource) + } + + if _, err := io.WriteString(cw, Magic); err != nil { + return rep(), writeErr(err) + } + + indexDigest := sha256.New() + var indexBytes int64 + var rec []byte + skipped, err = walkTree(ctx, fsys, lim, func(ord int64, name string, kind byte, _ fs.DirEntry) error { + entries = ord + if ord > lim.MaxEntries { + return fmt.Errorf("%w: the tree has more than %d entries", ErrLimit, lim.MaxEntries) + } + rec = append(rec[:0], kind) + rec = append(rec, name...) + // Charged the same way the HOST charges it (fs.go's readIndex): the + // record's bytes plus what holding the entry costs. Both ends applying + // the same arithmetic is what keeps a tree that this end streamed + // happily from being refused at the other end's index. + indexBytes += int64(len(rec)) + indexEntryOverhead + if indexBytes > lim.MaxIndexBytes { + return fmt.Errorf("%w: the tree's names are more than %d bytes", ErrLimit, lim.MaxIndexBytes) + } + hashRecord(indexDigest, kind, name) + return writeRecord(cw, rec) + }) + if err != nil { + return rep(), err + } + if _, err := cw.Write([]byte{0}); err != nil { // the zero-length record that ends the index + return rep(), writeErr(err) + } + + bodyDigest := sha256.New() + tw := tar.NewWriter(cw) + buf := make([]byte, copyBufSize) + if _, err := walkTree(ctx, fsys, lim, func(ord int64, name string, kind byte, d fs.DirEntry) error { + hashRecord(bodyDigest, kind, name) + return writeEntry(ctx, tw, cw, fsys, ord, name, kind, d, lim, buf) + }); err != nil { + return rep(), err + } + if err := tw.Close(); err != nil { + return rep(), writeErr(err) + } + // The two passes must have seen the same tree. They ran seconds apart on a + // quiesced workspace, so a difference is a process still writing in there — + // and a stream whose index and bodies disagree is one the host refuses + // halfway through, with nothing useful to say about why. + if !bytes.Equal(indexDigest.Sum(nil), bodyDigest.Sum(nil)) { + return rep(), fmt.Errorf("%w: the workspace changed while it was being streamed; it was not quiesced", ErrSource) + } + if cw.n > lim.MaxTotalBytes { + return rep(), fmt.Errorf("%w: the stream is over %d bytes", ErrLimit, lim.MaxTotalBytes) + } + return rep(), nil +} + +// copyBufSize is the one copy buffer a stream holds, the same size the +// checkpoint package uses for the same job. +const copyBufSize = 32 << 10 + +// writeEntry writes one entry's header and, for a regular file, its bytes. +func writeEntry(ctx context.Context, tw *tar.Writer, cw *countWriter, fsys fs.FS, + ord int64, name string, kind byte, d fs.DirEntry, lim Limits, buf []byte) error { + switch kind { + case KindDir: + info, err := d.Info() + if err != nil { + return fmt.Errorf("%w: entry %d could not be inspected (%s)", ErrSource, ord, category(err)) + } + return headerErr(ord, tw.WriteHeader(&tar.Header{ + Typeflag: tar.TypeDir, + Name: name, + Mode: int64(info.Mode().Perm()), + ModTime: modTime(info), + })) + + case KindSymlink: + target, err := fs.ReadLink(fsys, name) + if err != nil { + return fmt.Errorf("%w: entry %d is a symbolic link that could not be read (%s)", ErrSource, ord, category(err)) + } + if err := validLink(name, target); err != nil { + return fmt.Errorf("%w: entry %d is a symbolic link that is not allowed (%s)", ErrEntry, ord, err) + } + return headerErr(ord, tw.WriteHeader(&tar.Header{ + Typeflag: tar.TypeSymlink, + Name: name, + Linkname: target, + ModTime: time.Unix(0, 0).UTC(), + })) + + default: // KindFile + info, err := d.Info() + if err != nil { + return fmt.Errorf("%w: entry %d could not be inspected (%s)", ErrSource, ord, category(err)) + } + size := info.Size() + if size > lim.MaxEntryBytes { + return fmt.Errorf("%w: entry %d is larger than %d bytes", ErrLimit, ord, lim.MaxEntryBytes) + } + if cw.n+size > lim.MaxTotalBytes { + return fmt.Errorf("%w: the stream is over %d bytes", ErrLimit, lim.MaxTotalBytes) + } + if err := headerErr(ord, tw.WriteHeader(&tar.Header{ + Typeflag: tar.TypeReg, + Name: name, + Mode: int64(info.Mode().Perm()), + Size: size, + ModTime: modTime(info), + })); err != nil { + return err + } + return copyFile(ctx, tw, fsys, ord, name, size, buf) + } +} + +// copyFile writes exactly the length the header promised, and fails on either +// side of that promise. +// +// A file that SHRANK would desynchronize the tar stream; a file that GREW would +// be silently truncated into a checkpoint that verified. Both mean the +// workspace was not quiesced, which is a failed suspend rather than a +// checkpoint of a torn file — the same choice, for the same reason, that +// checkpoint.writeFileEntry makes on the host's side of a local tree. +func copyFile(ctx context.Context, tw io.Writer, fsys fs.FS, ord int64, name string, size int64, buf []byte) error { + f, err := fsys.Open(name) + if err != nil { + return fmt.Errorf("%w: entry %d could not be opened (%s)", ErrSource, ord, category(err)) + } + defer f.Close() + + n, err := io.CopyBuffer(tw, io.LimitReader(&ctxReader{ctx: ctx, r: f}, size), buf) + if err != nil { + if ctx.Err() != nil { + return ctx.Err() + } + // io.Copy reports the DESTINATION's failures too, and a destination + // that stopped taking bytes is not a workspace that could not be read. + // Reporting it as one would put "this file is unreadable" in a session's + // error column for a connection that died. + if d := destErr(err); d != nil { + return d + } + return fmt.Errorf("%w: entry %d could not be read (%s)", ErrSource, ord, category(err)) + } + if n != size { + return fmt.Errorf("%w: entry %d shrank while it was being read; the workspace was not quiesced", ErrSource, ord) + } + var probe [1]byte + if k, _ := f.Read(probe[:]); k > 0 { + return fmt.Errorf("%w: entry %d grew while it was being read; the workspace was not quiesced", ErrSource, ord) + } + return nil +} + +// walkTree visits every entry this format carries, in fs.WalkDir order, and is +// the ONE place the two passes agree about what an entry is: the same +// exclusions, the same kinds, the same ordinals, the same skips. +func walkTree(ctx context.Context, fsys fs.FS, lim Limits, fn func(ord int64, name string, kind byte, d fs.DirEntry) error) (skipped int64, err error) { + ord := int64(0) + err = fs.WalkDir(fsys, ".", func(name string, d fs.DirEntry, err error) error { + if ctxErr := ctx.Err(); ctxErr != nil { + return ctxErr + } + if err != nil { + // fs.WalkDir hands back the file system's own *fs.PathError, which + // carries the path. Reduced to its errno, never forwarded. + return fmt.Errorf("%w: entry %d could not be read (%s)", ErrSource, ord+1, category(err)) + } + if name == "." { + return nil // the root is the destination, not an entry + } + if lim.excluded(name) { + if d.IsDir() { + // Pruned rather than filtered: an excluded directory is never + // opened and nothing inside it is ever read, which is what + // makes an exclusion a promise about the bytes that left the + // guest rather than about the archive. + return fs.SkipDir + } + return nil + } + var kind byte + switch { + case d.IsDir(): + kind = KindDir + case d.Type()&fs.ModeSymlink != 0: + kind = KindSymlink + case d.Type().IsRegular(): + kind = KindFile + default: + // A socket, a FIFO, a device node. Skipped and counted rather than + // refused, and skipped BEFORE an ordinal is spent on it, so that the + // ordinals in an error name entries the stream actually carries. + skipped++ + return nil + } + ord++ + if err := validName(name); err != nil { + return fmt.Errorf("%w: entry %d is not a valid name (%s)", ErrEntry, ord, err) + } + return fn(ord, name, kind, d) + }) + return skipped, err +} + +// writeRecord writes one index record: its length as a uvarint, then its bytes. +func writeRecord(w io.Writer, rec []byte) error { + var hdr [binary.MaxVarintLen64]byte + n := binary.PutUvarint(hdr[:], uint64(len(rec))) + if _, err := w.Write(hdr[:n]); err != nil { + return writeErr(err) + } + if _, err := w.Write(rec); err != nil { + return writeErr(err) + } + return nil +} + +// hashRecord folds one (kind, name) into a digest. The separator is a NUL, +// which no name may contain, so two different trees cannot hash the same. +func hashRecord(h hash.Hash, kind byte, name string) { + h.Write([]byte{kind, 0}) + h.Write([]byte(name)) + h.Write([]byte{0}) +} + +func headerErr(ord int64, err error) error { + if err == nil { + return nil + } + if d := destErr(err); d != nil { + return d + } + return fmt.Errorf("%w: entry %d could not be written (%s)", ErrSource, ord, category(err)) +} + +// sinkWriter marks the errors that came from the DESTINATION — the caller's +// transport — rather than from the tree, and destErr reads the mark off again. +// +// The distinction is not cosmetic. Everything this package writes goes through +// archive/tar and io.Copy, both of which return a destination's error as their +// own, so without the mark a connection that died would be reported as "entry 7 +// could not be read" — which sends a person looking at a file when the problem +// is a socket. With it, the caller's own sentence about its own transport +// survives the trip out unchanged, which is what lets sessiond promise that +// nothing in that sentence is a path. +type sinkWriter struct{ w io.Writer } + +type sinkError struct{ err error } + +func (e *sinkError) Error() string { return e.err.Error() } +func (e *sinkError) Unwrap() error { return e.err } + +func (s *sinkWriter) Write(p []byte) (int, error) { + n, err := s.w.Write(p) + if err != nil { + return n, &sinkError{err: err} + } + return n, nil +} + +// destErr returns the destination's own error when err came from the +// destination, and nil when it did not. +func destErr(err error) error { + var se *sinkError + if errors.As(err, &se) { + return se.err + } + return nil +} + +// writeErr is destErr with a fallback, for the places where the error can only +// have come from the destination but the code must not depend on that: a nil +// returned from destErr there would swallow a failure whole. +func writeErr(err error) error { + if d := destErr(err); d != nil { + return d + } + return fmt.Errorf("%w: the stream could not be written (%s)", ErrSource, category(err)) +} + +// modTime is the modification time an entry carries: truncated to the second, +// deliberately and here rather than by archive/tar. +// +// tar's own header holds whole seconds, and sub-second precision is +// representable only as a PAX record — which costs a 1 KiB block PER ENTRY, so +// a large tree would pay megabytes for nanoseconds that the checkpoint format +// truncates away at the other end anyway (checkpoint.entryModTime). +func modTime(info fs.FileInfo) time.Time { return info.ModTime().Truncate(time.Second) } + +// countWriter counts what passes through it, which is how the stream's byte +// total is known without a second pass. +type countWriter struct { + w io.Writer + n int64 +} + +func (c *countWriter) Write(p []byte) (int, error) { + n, err := c.w.Write(p) + c.n += int64(n) + return n, err +} + +// ctxReader makes a long copy cancellable: the context is checked once per copy +// buffer, so one enormous file does not make a cancelled suspend run to +// completion. +type ctxReader struct { + ctx context.Context + r io.Reader +} + +func (c *ctxReader) Read(p []byte) (int, error) { + if err := c.ctx.Err(); err != nil { + return 0, err + } + return c.r.Read(p) +} + +// category reduces a file-system error to the part that is safe to print. An +// *fs.PathError and an *os.LinkError both carry the path that failed — which is +// session content — wrapped around an errno that carries nothing. +func category(err error) string { + var pe *fs.PathError + if errors.As(err, &pe) && pe.Err != nil { + return pe.Err.Error() + } + var le *os.LinkError + if errors.As(err, &le) && le.Err != nil { + return le.Err.Error() + } + if err == nil { + return "unknown error" + } + // Anything else comes from a file system this package did not write, whose + // text is not known to be path-free. Reduced rather than forwarded: a leak + // is worse than a vague sentence. + return "input/output error" +} diff --git a/internal/wstream/write_test.go b/internal/wstream/write_test.go new file mode 100644 index 0000000..a08582c --- /dev/null +++ b/internal/wstream/write_test.go @@ -0,0 +1,464 @@ +package wstream + +import ( + "archive/tar" + "bytes" + "context" + "encoding/binary" + "errors" + "io" + "io/fs" + "os" + "path/filepath" + "strings" + "testing" + "testing/fstest" + "time" +) + +// fixture builds the tree every test below streams: a file, a directory with a +// file in it, a symbolic link, and a Rainier-owned path that must never travel. +// +// It is a real directory rather than an fstest.MapFS because the thing under +// test is what a guest's own workspace produces — modes, modification times and +// symbolic links as the file system reports them. +func fixture(t *testing.T) string { + t.Helper() + root := t.TempDir() + write := func(rel, content string, mode os.FileMode) { + t.Helper() + p := filepath.Join(root, rel) + if err := os.MkdirAll(filepath.Dir(p), 0o755); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(p, []byte(content), mode); err != nil { + t.Fatal(err) + } + if err := os.Chmod(p, mode); err != nil { + t.Fatal(err) + } + } + write("a.txt", "alpha", 0o644) + write("dir/b.txt", "bravo", 0o600) + write(".rainier/session.json", "{}", 0o600) + if err := os.Symlink("dir/b.txt", filepath.Join(root, "link")); err != nil { + t.Fatal(err) + } + return root +} + +// entry is one decoded stream entry, index side and tar side together, which is +// how every assertion below is written: the two sections are one statement +// about one tree or the stream is wrong. +type entry struct { + kind byte + name string + hdr *tar.Header + content string +} + +// decode reads a whole stream back: the magic, the index, then the tar. It is +// deliberately a SECOND implementation of the reader rather than a call into +// fs.go, so that the format these tests pin is the one on the wire and not +// whatever the host's parser happens to accept. +func decode(t *testing.T, b []byte) []entry { + t.Helper() + if !bytes.HasPrefix(b, []byte(Magic)) { + t.Fatalf("the stream does not begin with the magic line") + } + r := bytes.NewReader(b[len(Magic):]) + var out []entry + for { + n, err := binary.ReadUvarint(r) + if err != nil { + t.Fatalf("reading an index length: %v", err) + } + if n == 0 { + break + } + rec := make([]byte, n) + if _, err := io.ReadFull(r, rec); err != nil { + t.Fatalf("reading an index record: %v", err) + } + out = append(out, entry{kind: rec[0], name: string(rec[1:])}) + } + tr := tar.NewReader(r) + for i := 0; ; i++ { + hdr, err := tr.Next() + if errors.Is(err, io.EOF) { + if i != len(out) { + t.Fatalf("the tar carries %d entries, the index names %d", i, len(out)) + } + return out + } + if err != nil { + t.Fatalf("reading tar entry %d: %v", i, err) + } + if i >= len(out) { + t.Fatalf("the tar carries more entries than the index names") + } + body, err := io.ReadAll(tr) + if err != nil { + t.Fatalf("reading tar body %d: %v", i, err) + } + out[i].hdr = hdr + out[i].content = string(body) + } +} + +func stream(t *testing.T, fsys fs.FS, lim Limits) ([]byte, Report, error) { + t.Helper() + var buf bytes.Buffer + rep, err := Write(context.Background(), fsys, &buf, lim) + return buf.Bytes(), rep, err +} + +// TestWriteCarriesTheTreeInWalkOrder is the whole format in one test: the index +// names every entry in fs.WalkDir order, the tar carries the same entries in +// the same order, and the two agree about what each one is. +// +// The ORDER is the load-bearing part. The checkpoint writer walks in fs.WalkDir +// order and the host's cursor only moves forward, so a stream in any other +// order is a suspend that fails rather than a checkpoint of a tree nobody had. +func TestWriteCarriesTheTreeInWalkOrder(t *testing.T) { + root := fixture(t) + raw, rep, err := stream(t, os.DirFS(root), Limits{Exclude: []string{".rainier"}}) + if err != nil { + t.Fatal(err) + } + got := decode(t, raw) + + want := []struct { + kind byte + name string + }{ + {KindFile, "a.txt"}, + {KindDir, "dir"}, + {KindFile, "dir/b.txt"}, + {KindSymlink, "link"}, + } + if len(got) != len(want) { + t.Fatalf("the stream carries %d entries, want %d: %+v", len(got), len(want), got) + } + for i, w := range want { + if got[i].kind != w.kind || got[i].name != w.name { + t.Errorf("entry %d is %c %q, want %c %q", i, got[i].kind, got[i].name, w.kind, w.name) + } + if got[i].hdr.Name != w.name { + t.Errorf("tar entry %d names %q, want %q", i, got[i].hdr.Name, w.name) + } + } + if rep.Entries != 4 { + t.Errorf("the report counts %d entries, want 4", rep.Entries) + } + if rep.Bytes != int64(len(raw)) { + t.Errorf("the report counts %d bytes, the stream is %d", rep.Bytes, len(raw)) + } +} + +// TestWriteCarriesModesContentAndLinkTargets: the attributes a restored +// workspace is the same workspace because of. +func TestWriteCarriesModesContentAndLinkTargets(t *testing.T) { + root := fixture(t) + info, err := os.Stat(filepath.Join(root, "a.txt")) + if err != nil { + t.Fatal(err) + } + raw, _, err := stream(t, os.DirFS(root), Limits{Exclude: []string{".rainier"}}) + if err != nil { + t.Fatal(err) + } + byName := map[string]entry{} + for _, e := range decode(t, raw) { + byName[e.name] = e + } + + if got := byName["a.txt"]; got.content != "alpha" || got.hdr.Mode != 0o644 { + t.Errorf("a.txt is mode %o with content %q, want 0644 and \"alpha\"", got.hdr.Mode, got.content) + } + if got := byName["dir/b.txt"]; got.content != "bravo" || got.hdr.Mode != 0o600 { + t.Errorf("dir/b.txt is mode %o with content %q, want 0600 and \"bravo\"", got.hdr.Mode, got.content) + } + // The link travels as a LINK with its target recorded, never followed and + // never resolved into a second copy of the file. + if got := byName["link"]; got.hdr.Linkname != "dir/b.txt" || got.content != "" { + t.Errorf("link is %q with %d bytes of body, want a target of dir/b.txt and no body", + got.hdr.Linkname, len(got.content)) + } + // The modification time survives to the second, which is what tar holds and + // what the checkpoint's tree digest commits to. + if got, want := byName["a.txt"].hdr.ModTime, info.ModTime().Truncate(time.Second); !got.Equal(want) { + t.Errorf("a.txt's modification time is %v, want %v", got, want) + } +} + +// TestWriteExcludesThePathsTheHostWouldRefuse. The exclusion is a promise about +// the bytes that LEAVE the guest, so the excluded directory must not be in the +// index, the tar, or the byte count. +func TestWriteExcludesThePathsTheHostWouldRefuse(t *testing.T) { + root := fixture(t) + raw, _, err := stream(t, os.DirFS(root), Limits{Exclude: []string{".rainier"}}) + if err != nil { + t.Fatal(err) + } + for _, e := range decode(t, raw) { + if strings.HasPrefix(e.name, ".rainier") { + t.Fatalf("the stream carries %q, which is excluded", e.name) + } + } + if bytes.Contains(raw, []byte("session.json")) { + t.Error("an excluded file's name appears in the stream's bytes") + } +} + +// TestWriteSkipsAndCountsAnEntryTheFormatCannotCarry: a socket left behind by a +// dev server must not be able to defeat the durability barrier, and must not be +// silent either. +func TestWriteSkipsAndCountsAnEntryTheFormatCannotCarry(t *testing.T) { + fsys := fstest.MapFS{ + "a.txt": &fstest.MapFile{Data: []byte("alpha"), Mode: 0o644}, + "a.sock": &fstest.MapFile{Mode: fs.ModeSocket | 0o600}, + } + raw, rep, err := stream(t, fsys, Limits{}) + if err != nil { + t.Fatal(err) + } + if rep.Entries != 1 || rep.Skipped != 1 { + t.Fatalf("the report counts %d entries and %d skipped, want 1 and 1", rep.Entries, rep.Skipped) + } + for _, e := range decode(t, raw) { + if e.name == "a.sock" { + t.Fatal("the stream carries a socket") + } + } +} + +// TestWriteRefusesASymlinkThatLeavesTheTree is the containment rule at the +// guest's end: a workspace containing "creds -> /etc/shadow" must not stream +// the host's — or the guest's — /etc/shadow anywhere. +func TestWriteRefusesASymlinkThatLeavesTheTree(t *testing.T) { + for _, target := range []string{"/etc/shadow", "../../etc/shadow"} { + root := t.TempDir() + if err := os.Symlink(target, filepath.Join(root, "creds")); err != nil { + t.Fatal(err) + } + _, _, err := stream(t, os.DirFS(root), Limits{}) + if !errors.Is(err, ErrEntry) { + t.Fatalf("a symlink to %q streamed with error %v, want ErrEntry", target, err) + } + if err != nil && strings.Contains(err.Error(), "shadow") { + t.Errorf("the refusal quotes the symlink target: %v", err) + } + } +} + +// TestWriteRefusesEachLimit walks the table of limits, because a limit nothing +// tests is a limit that is one typo from being absent. +func TestWriteRefusesEachLimit(t *testing.T) { + root := fixture(t) + cases := []struct { + name string + lim Limits + }{ + {"entries", Limits{MaxEntries: 2}}, + {"index bytes", Limits{MaxIndexBytes: 8}}, + {"one entry's bytes", Limits{MaxEntryBytes: 2}}, + {"total bytes", Limits{MaxTotalBytes: 64}}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + tc.lim.Exclude = []string{".rainier"} + _, _, err := stream(t, os.DirFS(root), tc.lim) + if !errors.Is(err, ErrLimit) { + t.Fatalf("the stream ended with %v, want ErrLimit", err) + } + }) + } + t.Run("a negative limit is configuration, not permission", func(t *testing.T) { + _, _, err := stream(t, os.DirFS(root), Limits{MaxEntries: -1}) + if !errors.Is(err, ErrLimit) { + t.Fatalf("a negative limit gave %v, want ErrLimit", err) + } + }) + t.Run("an exclusion that is not a clean relative path", func(t *testing.T) { + _, _, err := stream(t, os.DirFS(root), Limits{Exclude: []string{"/etc"}}) + if !errors.Is(err, ErrLimit) { + t.Fatalf("an absolute exclusion gave %v, want ErrLimit", err) + } + }) +} + +// TestWriteReportsTheCountsItReachedOnAFailure. The counts travel in the end +// marker, and a failure is exactly when somebody needs them: a stream that +// stopped at entry 900 is a different problem from one that never started. +func TestWriteRefusalCarriesTheCountsItReached(t *testing.T) { + root := fixture(t) + _, rep, err := stream(t, os.DirFS(root), Limits{MaxEntries: 2, Exclude: []string{".rainier"}}) + if err == nil { + t.Fatal("the stream did not refuse") + } + if rep.Entries == 0 || rep.Bytes == 0 { + t.Fatalf("the refusal reports %d entries and %d bytes, want the counts it reached", rep.Entries, rep.Bytes) + } +} + +// TestWriteRefusesAFileThatChangedUnderIt is the quiesce check. A file that +// shrank would desynchronize the tar; one that grew would be silently truncated +// into a checkpoint that VERIFIED, which is the one failure the barrier must +// not have. Both are a failed suspend. +func TestWriteRefusesAFileThatChangedUnderIt(t *testing.T) { + for _, tc := range []struct { + name string + claim int64 + data string + }{ + {"shrank", 10, "short"}, + {"grew", 2, "longer"}, + } { + t.Run(tc.name, func(t *testing.T) { + fsys := &lyingFS{ + MapFS: fstest.MapFS{"a.txt": &fstest.MapFile{Data: []byte(tc.data), Mode: 0o644}}, + name: "a.txt", + size: tc.claim, + } + _, _, err := stream(t, fsys, Limits{}) + if !errors.Is(err, ErrSource) { + t.Fatalf("a file that %s gave %v, want ErrSource", tc.name, err) + } + if !strings.Contains(err.Error(), "quiesced") { + t.Errorf("the refusal does not name the cause: %v", err) + } + }) + } +} + +// TestWriteRefusesATreeThatChangedBetweenThePasses. The index and the bodies +// come from two walks; a tree that changed between them would produce a stream +// whose two halves disagree, which the host can only refuse halfway through. +// Caught here, where there is something to say about it. +func TestWriteRefusesATreeThatChangedBetweenThePasses(t *testing.T) { + fsys := &mutatingFS{MapFS: fstest.MapFS{ + "a.txt": &fstest.MapFile{Data: []byte("alpha"), Mode: 0o644}, + }} + _, _, err := stream(t, fsys, Limits{}) + if !errors.Is(err, ErrSource) { + t.Fatalf("a tree that changed between the passes gave %v, want ErrSource", err) + } + if !strings.Contains(err.Error(), "quiesced") { + t.Errorf("the refusal does not name the cause: %v", err) + } +} + +// TestWriteStopsOnACancelledContext: a cold suspend whose budget expired must +// not run a 10 GiB copy to completion first. +func TestWriteStopsOnACancelledContext(t *testing.T) { + root := fixture(t) + ctx, cancel := context.WithCancel(context.Background()) + cancel() + if _, err := Write(ctx, os.DirFS(root), new(bytes.Buffer), Limits{}); !errors.Is(err, context.Canceled) { + t.Fatalf("a cancelled stream ended with %v, want context.Canceled", err) + } +} + +// TestWriteErrorsNameNoPath. These sentences travel to a session's error +// column, where the tenancy specification's §15.1 prohibited-logging list +// applies: an ordinal is findable with a local walk and is not a path. +func TestWriteErrorsNameNoPath(t *testing.T) { + root := fixture(t) + _, _, err := stream(t, os.DirFS(root), Limits{MaxEntryBytes: 1, Exclude: []string{".rainier"}}) + if err == nil { + t.Fatal("the stream did not refuse") + } + for _, leak := range []string{root, "a.txt", "b.txt", "link"} { + if strings.Contains(err.Error(), leak) { + t.Errorf("the refusal quotes %q: %v", leak, err) + } + } +} + +// lyingFS reports a size its file's bytes do not agree with, which is what a +// file being written to under the walk looks like from here. +type lyingFS struct { + fstest.MapFS + name string + size int64 +} + +func (l *lyingFS) Open(name string) (fs.File, error) { + f, err := l.MapFS.Open(name) + if err != nil || name != l.name { + return f, err + } + return &lyingFile{File: f, size: l.size}, nil +} + +func (l *lyingFS) Stat(name string) (fs.FileInfo, error) { + info, err := l.MapFS.Stat(name) + if err != nil || name != l.name { + return info, err + } + return &lyingInfo{FileInfo: info, size: l.size}, nil +} + +func (l *lyingFS) ReadDir(name string) ([]fs.DirEntry, error) { + entries, err := l.MapFS.ReadDir(name) + if err != nil { + return nil, err + } + for i, e := range entries { + if e.Name() == l.name { + entries[i] = &lyingDirEntry{DirEntry: e, size: l.size} + } + } + return entries, nil +} + +type lyingFile struct { + fs.File + size int64 +} + +func (l *lyingFile) Stat() (fs.FileInfo, error) { + info, err := l.File.Stat() + if err != nil { + return nil, err + } + return &lyingInfo{FileInfo: info, size: l.size}, nil +} + +type lyingInfo struct { + fs.FileInfo + size int64 +} + +func (l *lyingInfo) Size() int64 { return l.size } + +type lyingDirEntry struct { + fs.DirEntry + size int64 +} + +func (l *lyingDirEntry) Info() (fs.FileInfo, error) { + info, err := l.DirEntry.Info() + if err != nil { + return nil, err + } + return &lyingInfo{FileInfo: info, size: l.size}, nil +} + +// mutatingFS grows a file between the two walks, which is a process still +// writing in a workspace that was supposed to be quiesced. +type mutatingFS struct { + fstest.MapFS + walks int +} + +func (m *mutatingFS) ReadDir(name string) ([]fs.DirEntry, error) { + if name == "." { + m.walks++ + if m.walks == 2 { + m.MapFS["b.txt"] = &fstest.MapFile{Data: []byte("bravo"), Mode: 0o644} + } + } + return m.MapFS.ReadDir(name) +}