sandboxd: workspace filecache — cross-node NAS-backed workspace sync

sandboxd/config/config.go

@@ -171,6 +171,22 @@ type Config struct {
 	DataDir   string `json:"data_dir"`
 	CocoonBin string `json:"cocoon_bin"`
 
+	// WorkspaceRoot enables the workspace filecache: the host path (a shared
+	// NAS mount) under which a claim's workspace token resolves to
+	// <WorkspaceRoot>/<token>. Empty disables the feature — claims that carry a
+	// workspace are served normally, just without host-side sync. The mount
+	// should use a low attribute cache (actimeo=1) so cross-node journal
+	// changes appear within the visibility budget.
+	WorkspaceRoot string `json:"workspace_root,omitempty"`
+	// WorkspaceDiskMB, when > 0, puts each workspace on a dedicated read-write
+	// ext4 virtio-blk disk of this size (image under WorkspaceDiskDir) instead
+	// of the guest rootfs layer, isolating it from the rootfs COW. Requires
+	// WorkspaceRoot.
+	WorkspaceDiskMB int `json:"workspace_disk_mb,omitempty"`
+	// WorkspaceDiskDir holds the raw disk images on local NVMe; defaults to
+	// <DataDir>/workspaces.
+	WorkspaceDiskDir string `json:"workspace_disk_dir,omitempty"`
+
 	// AdvertiseAddr is the host:port the data plane reaches this node at; it
 	// is returned as a claim's owner address (and, at M2c, gossiped). Defaults
 	// to Listen, which is correct when Listen is a routable host:port.

sandboxd/engine/guestfs.go

@@ -0,0 +1,148 @@
+package engine
+
+import (
+	"bytes"
+	"context"
+	"errors"
+	"fmt"
+	"io"
+	"slices"
+
+	"github.com/cocoonstack/sandbox/protocol/wire"
+)
+
+// GuestFS-style helpers over silkd for the workspace filecache. They reuse the
+// same session plumbing as the volume and CA helpers; all operate on a claimed
+// VM's vsock UDS and stay off the warm-claim hot path.
+
+// GuestRun executes argv in the guest and returns its stdout (stderr folds into
+// the error on non-zero exit). silkd starts the child with an empty
+// environment, so PATH is set like silkdExec.
+func (e *Engine) GuestRun(ctx context.Context, vsockSocket string, argv ...string) (string, error) {
+	s, err := e.dialSilkdSession(ctx, vsockSocket)
+	if err != nil {
+		return "", err
+	}
+	defer s.close()
+	sendErr := s.send(wire.Exec{Argv: argv, Env: map[string]string{"PATH": guestExecPATH}})
+	if sendErr == nil {
+		sendErr = s.send(wire.StdinClose{})
+	}
+	var stdout, stderr bytes.Buffer
+	for {
+		frame, err := s.recv()
+		if err != nil {
+			return "", errors.Join(sendErr, err)
+		}
+		switch resp := frame.(type) {
+		case *wire.Started:
+		case *wire.Stdout:
+			stdout.Write(resp.Data)
+		case *wire.Stderr:
+			stderr.Write(resp.Data)
+		case *wire.Exit:
+			if resp.Code != 0 {
+				return stdout.String(), fmt.Errorf("exit code %d: %s", resp.Code, bytes.TrimSpace(stderr.Bytes()))
+			}
+			return stdout.String(), nil
+		case *wire.ErrorResp:
+			return "", fmt.Errorf("silkd %w", resp)
+		default:
+			return "", fmt.Errorf("unexpected silkd frame %q", resp.RespType())
+		}
+	}
+}
+
+// GuestWriteFile writes data to path in the guest (mode applied at create).
+func (e *Engine) GuestWriteFile(ctx context.Context, vsockSocket, path string, mode uint32, data []byte) error {
+	return e.silkdWriteFile(ctx, vsockSocket, path, mode, data)
+}
+
+// GuestReadFile reads a guest file, returning its bytes.
+func (e *Engine) GuestReadFile(ctx context.Context, vsockSocket, path string) ([]byte, error) {
+	return e.silkdReadFile(ctx, vsockSocket, path)
+}
+
+// GuestRemove deletes a guest path (recursive for trees).
+func (e *Engine) GuestRemove(ctx context.Context, vsockSocket, path string, recursive bool) error {
+	return e.silkdStream(ctx, vsockSocket, wire.FsRm{Path: path, Recursive: recursive}, func(wire.Response) error {
+		return nil
+	})
+}
+
+// GuestPushTar extracts a tar stream under dest in the guest (silkd runs
+// `tar -x`). The reader supplies tar bytes.
+func (e *Engine) GuestPushTar(ctx context.Context, vsockSocket, dest string, r io.Reader) error {
+	s, err := e.dialSilkdSession(ctx, vsockSocket)
+	if err != nil {
+		return err
+	}
+	defer s.close()
+	sendErr := func() error {
+		if serr := s.send(wire.FsPush{Dest: dest}); serr != nil {
+			return serr
+		}
+		buf := make([]byte, silkdChunk)
+		for {
+			n, rerr := r.Read(buf)
+			if n > 0 {
+				if serr := s.send(wire.Data{Data: slices.Clone(buf[:n])}); serr != nil {
+					return serr
+				}
+			}
+			if rerr == io.EOF {
+				break
+			}
+			if rerr != nil {
+				return rerr
+			}
+		}
+		return s.send(wire.DataEnd{})
+	}()
+	frame, err := s.recv()
+	if err != nil {
+		return errors.Join(sendErr, err)
+	}
+	switch resp := frame.(type) {
+	case *wire.Done:
+		return nil
+	case *wire.ErrorResp:
+		return fmt.Errorf("silkd %w", resp)
+	default:
+		return fmt.Errorf("unexpected silkd frame %q", resp.RespType())
+	}
+}
+
+// WorkspaceDiskAttach hot-attaches a read-write ext4 image to vmName as a
+// virtio-blk device with serial name (guest device discovered by that serial).
+// Unlike DiskAttach (operator catalog volumes are read-only), the workspace
+// disk is writable so the filecache can stage the guest's working set on it.
+func (e *Engine) WorkspaceDiskAttach(ctx context.Context, vmName, rawPath, name string) error {
+	_, err := e.run(ctx, "vm", "disk", "attach", vmName,
+		"--path", rawPath, argName, name, "--directio", "auto")
+	return err
+}
+
+// WorkspaceDiskMount discovers the attached workspace disk by serial and mounts
+// it read-write at mount inside the guest.
+func (e *Engine) WorkspaceDiskMount(ctx context.Context, vsockSocket, name, mount string) error {
+	ctx, cancel := context.WithTimeout(ctx, volumeSetupTimeout)
+	defer cancel()
+	device, err := e.waitForVolumeDevice(ctx, vsockSocket, name)
+	if err != nil {
+		return fmt.Errorf("wait for workspace device %s: %w", name, err)
+	}
+	if err := e.silkdExec(ctx, vsockSocket, "mkdir", "-p", "--", mount); err != nil {
+		return fmt.Errorf("create workspace mount point %s: %w", mount, err)
+	}
+	if err := e.silkdExec(ctx, vsockSocket, "mount", "-t", "ext4", "--", device, mount); err != nil {
+		return fmt.Errorf("mount workspace disk %s: %w", name, err)
+	}
+	return nil
+}
+
+// WorkspaceDiskDetach unmounts and detaches the workspace disk from vmName.
+func (e *Engine) WorkspaceDiskDetach(ctx context.Context, vmName, name string) error {
+	_, err := e.run(ctx, "vm", "disk", "detach", vmName, argName, name)
+	return err
+}

sandboxd/filecache/disk.go

@@ -0,0 +1,95 @@
+package filecache
+
+import (
+	"context"
+	"fmt"
+	"os"
+	"os/exec"
+	"path/filepath"
+	"time"
+)
+
+// Disk drives VM-level workspace-disk hotplug for the dedicated-disk mode; the
+// engine implements it, reusing the same cocoon disk-attach and by-serial
+// device discovery as operator catalog volumes, but read-write. Separate from
+// Guest (in-guest silkd ops) because attach/detach/mount act on the VM.
+type Disk interface {
+	// Attach hot-attaches a read-write raw image to vmName as a virtio-blk
+	// device with the given serial name.
+	Attach(ctx context.Context, vmName, rawPath, name string) error
+	// Mount discovers the device by serial and mounts it read-write at mount.
+	Mount(ctx context.Context, vsockSocket, name, mount string) error
+	// Detach unmounts and detaches the disk from vmName.
+	Detach(ctx context.Context, vmName, name string) error
+}
+
+// diskSerial is the attach serial and by-id key for a sandbox's workspace disk.
+// One per sandbox, so a constant is fine.
+const diskSerial = "fcws"
+
+// diskProvisioner creates the host-side raw ext4 image, hot-attaches it
+// read-write, and mounts it in the guest at the workspace mount before
+// hydration. On barrier it unmounts, detaches, and removes the image. A nil
+// disk driver means the feature is off (workspace stays on the rootfs layer).
+type diskProvisioner struct {
+	disk   Disk
+	guest  Guest
+	root   string // host dir for raw images
+	sizeMB int
+}
+
+func (d *diskProvisioner) rawPath(id string) string {
+	return filepath.Join(d.root, id+".raw")
+}
+
+// attachAndMount creates a fresh ext4 image, attaches it read-write to vmName,
+// and mounts it at mount inside the guest before hydration.
+func (d *diskProvisioner) attachAndMount(ctx context.Context, id, vmName, vsockSocket, mount string) error {
+	raw := d.rawPath(id)
+	if err := os.MkdirAll(d.root, 0o755); err != nil {
+		return err
+	}
+	if _, err := os.Stat(raw); os.IsNotExist(err) {
+		if err := createExt4(ctx, raw, d.sizeMB); err != nil {
+			return fmt.Errorf("create workspace disk: %w", err)
+		}
+	}
+	if err := d.disk.Attach(ctx, vmName, raw, diskSerial); err != nil {
+		return fmt.Errorf("attach workspace disk: %w", err)
+	}
+	if err := d.disk.Mount(ctx, vsockSocket, diskSerial, mount); err != nil {
+		return fmt.Errorf("mount workspace disk: %w", err)
+	}
+	return nil
+}
+
+// unmountAndDetach reverses attachAndMount for barrier. Best-effort per step so
+// one failure does not strand the rest; a gone VM (already reaped) is fine.
+func (d *diskProvisioner) unmountAndDetach(ctx context.Context, id, vmName, vsockSocket, mount string) {
+	d.guest.Run(ctx, vsockSocket, "/bin/sh", "-c", "/usr/bin/umount "+mount+" 2>/dev/null || true")
+	_ = d.disk.Detach(ctx, vmName, diskSerial)
+	os.Remove(d.rawPath(id))
+}
+
+// createExt4 makes a sparse raw image of sizeMB and formats it ext4. mkfs is
+// host tooling the sandbox rootfs already relies on at bake time.
+func createExt4(ctx context.Context, path string, sizeMB int) error {
+	f, err := os.OpenFile(path, os.O_CREATE|os.O_WRONLY|os.O_TRUNC, 0o644)
+	if err != nil {
+		return err
+	}
+	if err := f.Truncate(int64(sizeMB) * 1024 * 1024); err != nil {
+		f.Close()
+		os.Remove(path)
+		return err
+	}
+	f.Close()
+	ctx, cancel := context.WithTimeout(ctx, 30*time.Second)
+	defer cancel()
+	cmd := exec.CommandContext(ctx, "mkfs.ext4", "-qF", path)
+	if out, err := cmd.CombinedOutput(); err != nil {
+		os.Remove(path)
+		return fmt.Errorf("mkfs.ext4: %w: %s", err, out)
+	}
+	return nil
+}

sandboxd/filecache/filecache.go

@@ -0,0 +1,49 @@
+// Package filecache keeps a sandbox's workspace directory in sync with a
+// shared NAS workspace under a session-granular, multi-writer contract.
+//
+// The guest workspace lives on the sandbox's own local disk, so every file
+// operation runs at local latency with no network client and no FUSE inside
+// the guest. This package (host side, in sandboxd) moves deltas between the
+// guest — over silkd via the engine — and the NAS workspace (a host mount).
+//
+// Coordination is through the NAS itself: each writer appends journal entries
+// under <ws>/.filecache/journal/ and freshens <ws>/.filecache/seq; pullers
+// poll seq (an O(1) GETATTR) and fetch only the paths named in unseen entries.
+// Concurrent edits resolve last-writer-wins by NAS-observed divergence; the
+// peer's version is preserved as <path>.fc-conflict-<ts>, never silently lost.
+package filecache
+
+import (
+	"context"
+	"io"
+)
+
+const fcDir = ".filecache"
+
+// Guest is the subset of guest operations the sync engine needs; the engine
+// package implements it over silkd. All paths are absolute guest paths.
+type Guest interface {
+	Run(ctx context.Context, vsockSocket string, argv ...string) (string, error)
+	WriteFile(ctx context.Context, vsockSocket, path string, mode uint32, data []byte) error
+	ReadFile(ctx context.Context, vsockSocket, path string) ([]byte, error)
+	PushTar(ctx context.Context, vsockSocket, dest string, r io.Reader) error
+	Remove(ctx context.Context, vsockSocket, path string, recursive bool) error
+}
+
+// entMeta is a workspace entry's identity for change detection.
+type entMeta struct {
+	Kind   string `json:"kind"` // f | l
+	Size   int64  `json:"size,omitempty"`
+	MtimeS int64  `json:"mtime_s,omitempty"`
+	Target string `json:"target,omitempty"` // symlink target
+}
+
+// journalEntry is one writer's published delta, serialized under
+// <ws>/.filecache/journal/<writer>-<seq>.json.
+type journalEntry struct {
+	Writer string             `json:"writer"`
+	Seq    uint64             `json:"seq"`
+	TsNs   int64              `json:"ts_ns"`
+	Puts   map[string]entMeta `json:"puts,omitempty"`
+	Dels   []string           `json:"dels,omitempty"`
+}