diff --git a/sandboxd/config/config.go b/sandboxd/config/config.go index 61addf8..2fc7350 100644 --- a/sandboxd/config/config.go +++ b/sandboxd/config/config.go @@ -171,6 +171,22 @@ type Config struct { DataDir string `json:"data_dir"` CocoonBin string `json:"cocoon_bin"` + // WorkspaceRoot enables the workspace filecache: the host path (a shared + // NAS mount) under which a claim's workspace token resolves to + // /. Empty disables the feature — claims that carry a + // workspace are served normally, just without host-side sync. The mount + // should use a low attribute cache (actimeo=1) so cross-node journal + // changes appear within the visibility budget. + WorkspaceRoot string `json:"workspace_root,omitempty"` + // WorkspaceDiskMB, when > 0, puts each workspace on a dedicated read-write + // ext4 virtio-blk disk of this size (image under WorkspaceDiskDir) instead + // of the guest rootfs layer, isolating it from the rootfs COW. Requires + // WorkspaceRoot. + WorkspaceDiskMB int `json:"workspace_disk_mb,omitempty"` + // WorkspaceDiskDir holds the raw disk images on local NVMe; defaults to + // /workspaces. + WorkspaceDiskDir string `json:"workspace_disk_dir,omitempty"` + // AdvertiseAddr is the host:port the data plane reaches this node at; it // is returned as a claim's owner address (and, at M2c, gossiped). Defaults // to Listen, which is correct when Listen is a routable host:port. diff --git a/sandboxd/engine/guestfs.go b/sandboxd/engine/guestfs.go new file mode 100644 index 0000000..7549285 --- /dev/null +++ b/sandboxd/engine/guestfs.go @@ -0,0 +1,148 @@ +package engine + +import ( + "bytes" + "context" + "errors" + "fmt" + "io" + "slices" + + "github.com/cocoonstack/sandbox/protocol/wire" +) + +// GuestFS-style helpers over silkd for the workspace filecache. They reuse the +// same session plumbing as the volume and CA helpers; all operate on a claimed +// VM's vsock UDS and stay off the warm-claim hot path. + +// GuestRun executes argv in the guest and returns its stdout (stderr folds into +// the error on non-zero exit). silkd starts the child with an empty +// environment, so PATH is set like silkdExec. +func (e *Engine) GuestRun(ctx context.Context, vsockSocket string, argv ...string) (string, error) { + s, err := e.dialSilkdSession(ctx, vsockSocket) + if err != nil { + return "", err + } + defer s.close() + sendErr := s.send(wire.Exec{Argv: argv, Env: map[string]string{"PATH": guestExecPATH}}) + if sendErr == nil { + sendErr = s.send(wire.StdinClose{}) + } + var stdout, stderr bytes.Buffer + for { + frame, err := s.recv() + if err != nil { + return "", errors.Join(sendErr, err) + } + switch resp := frame.(type) { + case *wire.Started: + case *wire.Stdout: + stdout.Write(resp.Data) + case *wire.Stderr: + stderr.Write(resp.Data) + case *wire.Exit: + if resp.Code != 0 { + return stdout.String(), fmt.Errorf("exit code %d: %s", resp.Code, bytes.TrimSpace(stderr.Bytes())) + } + return stdout.String(), nil + case *wire.ErrorResp: + return "", fmt.Errorf("silkd %w", resp) + default: + return "", fmt.Errorf("unexpected silkd frame %q", resp.RespType()) + } + } +} + +// GuestWriteFile writes data to path in the guest (mode applied at create). +func (e *Engine) GuestWriteFile(ctx context.Context, vsockSocket, path string, mode uint32, data []byte) error { + return e.silkdWriteFile(ctx, vsockSocket, path, mode, data) +} + +// GuestReadFile reads a guest file, returning its bytes. +func (e *Engine) GuestReadFile(ctx context.Context, vsockSocket, path string) ([]byte, error) { + return e.silkdReadFile(ctx, vsockSocket, path) +} + +// GuestRemove deletes a guest path (recursive for trees). +func (e *Engine) GuestRemove(ctx context.Context, vsockSocket, path string, recursive bool) error { + return e.silkdStream(ctx, vsockSocket, wire.FsRm{Path: path, Recursive: recursive}, func(wire.Response) error { + return nil + }) +} + +// GuestPushTar extracts a tar stream under dest in the guest (silkd runs +// `tar -x`). The reader supplies tar bytes. +func (e *Engine) GuestPushTar(ctx context.Context, vsockSocket, dest string, r io.Reader) error { + s, err := e.dialSilkdSession(ctx, vsockSocket) + if err != nil { + return err + } + defer s.close() + sendErr := func() error { + if serr := s.send(wire.FsPush{Dest: dest}); serr != nil { + return serr + } + buf := make([]byte, silkdChunk) + for { + n, rerr := r.Read(buf) + if n > 0 { + if serr := s.send(wire.Data{Data: slices.Clone(buf[:n])}); serr != nil { + return serr + } + } + if rerr == io.EOF { + break + } + if rerr != nil { + return rerr + } + } + return s.send(wire.DataEnd{}) + }() + frame, err := s.recv() + if err != nil { + return errors.Join(sendErr, err) + } + switch resp := frame.(type) { + case *wire.Done: + return nil + case *wire.ErrorResp: + return fmt.Errorf("silkd %w", resp) + default: + return fmt.Errorf("unexpected silkd frame %q", resp.RespType()) + } +} + +// WorkspaceDiskAttach hot-attaches a read-write ext4 image to vmName as a +// virtio-blk device with serial name (guest device discovered by that serial). +// Unlike DiskAttach (operator catalog volumes are read-only), the workspace +// disk is writable so the filecache can stage the guest's working set on it. +func (e *Engine) WorkspaceDiskAttach(ctx context.Context, vmName, rawPath, name string) error { + _, err := e.run(ctx, "vm", "disk", "attach", vmName, + "--path", rawPath, argName, name, "--directio", "auto") + return err +} + +// WorkspaceDiskMount discovers the attached workspace disk by serial and mounts +// it read-write at mount inside the guest. +func (e *Engine) WorkspaceDiskMount(ctx context.Context, vsockSocket, name, mount string) error { + ctx, cancel := context.WithTimeout(ctx, volumeSetupTimeout) + defer cancel() + device, err := e.waitForVolumeDevice(ctx, vsockSocket, name) + if err != nil { + return fmt.Errorf("wait for workspace device %s: %w", name, err) + } + if err := e.silkdExec(ctx, vsockSocket, "mkdir", "-p", "--", mount); err != nil { + return fmt.Errorf("create workspace mount point %s: %w", mount, err) + } + if err := e.silkdExec(ctx, vsockSocket, "mount", "-t", "ext4", "--", device, mount); err != nil { + return fmt.Errorf("mount workspace disk %s: %w", name, err) + } + return nil +} + +// WorkspaceDiskDetach unmounts and detaches the workspace disk from vmName. +func (e *Engine) WorkspaceDiskDetach(ctx context.Context, vmName, name string) error { + _, err := e.run(ctx, "vm", "disk", "detach", vmName, argName, name) + return err +} diff --git a/sandboxd/filecache/disk.go b/sandboxd/filecache/disk.go new file mode 100644 index 0000000..a30b32a --- /dev/null +++ b/sandboxd/filecache/disk.go @@ -0,0 +1,95 @@ +package filecache + +import ( + "context" + "fmt" + "os" + "os/exec" + "path/filepath" + "time" +) + +// Disk drives VM-level workspace-disk hotplug for the dedicated-disk mode; the +// engine implements it, reusing the same cocoon disk-attach and by-serial +// device discovery as operator catalog volumes, but read-write. Separate from +// Guest (in-guest silkd ops) because attach/detach/mount act on the VM. +type Disk interface { + // Attach hot-attaches a read-write raw image to vmName as a virtio-blk + // device with the given serial name. + Attach(ctx context.Context, vmName, rawPath, name string) error + // Mount discovers the device by serial and mounts it read-write at mount. + Mount(ctx context.Context, vsockSocket, name, mount string) error + // Detach unmounts and detaches the disk from vmName. + Detach(ctx context.Context, vmName, name string) error +} + +// diskSerial is the attach serial and by-id key for a sandbox's workspace disk. +// One per sandbox, so a constant is fine. +const diskSerial = "fcws" + +// diskProvisioner creates the host-side raw ext4 image, hot-attaches it +// read-write, and mounts it in the guest at the workspace mount before +// hydration. On barrier it unmounts, detaches, and removes the image. A nil +// disk driver means the feature is off (workspace stays on the rootfs layer). +type diskProvisioner struct { + disk Disk + guest Guest + root string // host dir for raw images + sizeMB int +} + +func (d *diskProvisioner) rawPath(id string) string { + return filepath.Join(d.root, id+".raw") +} + +// attachAndMount creates a fresh ext4 image, attaches it read-write to vmName, +// and mounts it at mount inside the guest before hydration. +func (d *diskProvisioner) attachAndMount(ctx context.Context, id, vmName, vsockSocket, mount string) error { + raw := d.rawPath(id) + if err := os.MkdirAll(d.root, 0o755); err != nil { + return err + } + if _, err := os.Stat(raw); os.IsNotExist(err) { + if err := createExt4(ctx, raw, d.sizeMB); err != nil { + return fmt.Errorf("create workspace disk: %w", err) + } + } + if err := d.disk.Attach(ctx, vmName, raw, diskSerial); err != nil { + return fmt.Errorf("attach workspace disk: %w", err) + } + if err := d.disk.Mount(ctx, vsockSocket, diskSerial, mount); err != nil { + return fmt.Errorf("mount workspace disk: %w", err) + } + return nil +} + +// unmountAndDetach reverses attachAndMount for barrier. Best-effort per step so +// one failure does not strand the rest; a gone VM (already reaped) is fine. +func (d *diskProvisioner) unmountAndDetach(ctx context.Context, id, vmName, vsockSocket, mount string) { + d.guest.Run(ctx, vsockSocket, "/bin/sh", "-c", "/usr/bin/umount "+mount+" 2>/dev/null || true") + _ = d.disk.Detach(ctx, vmName, diskSerial) + os.Remove(d.rawPath(id)) +} + +// createExt4 makes a sparse raw image of sizeMB and formats it ext4. mkfs is +// host tooling the sandbox rootfs already relies on at bake time. +func createExt4(ctx context.Context, path string, sizeMB int) error { + f, err := os.OpenFile(path, os.O_CREATE|os.O_WRONLY|os.O_TRUNC, 0o644) + if err != nil { + return err + } + if err := f.Truncate(int64(sizeMB) * 1024 * 1024); err != nil { + f.Close() + os.Remove(path) + return err + } + f.Close() + ctx, cancel := context.WithTimeout(ctx, 30*time.Second) + defer cancel() + cmd := exec.CommandContext(ctx, "mkfs.ext4", "-qF", path) + if out, err := cmd.CombinedOutput(); err != nil { + os.Remove(path) + return fmt.Errorf("mkfs.ext4: %w: %s", err, out) + } + return nil +} diff --git a/sandboxd/filecache/filecache.go b/sandboxd/filecache/filecache.go new file mode 100644 index 0000000..77605e6 --- /dev/null +++ b/sandboxd/filecache/filecache.go @@ -0,0 +1,49 @@ +// Package filecache keeps a sandbox's workspace directory in sync with a +// shared NAS workspace under a session-granular, multi-writer contract. +// +// The guest workspace lives on the sandbox's own local disk, so every file +// operation runs at local latency with no network client and no FUSE inside +// the guest. This package (host side, in sandboxd) moves deltas between the +// guest — over silkd via the engine — and the NAS workspace (a host mount). +// +// Coordination is through the NAS itself: each writer appends journal entries +// under /.filecache/journal/ and freshens /.filecache/seq; pullers +// poll seq (an O(1) GETATTR) and fetch only the paths named in unseen entries. +// Concurrent edits resolve last-writer-wins by NAS-observed divergence; the +// peer's version is preserved as .fc-conflict-, never silently lost. +package filecache + +import ( + "context" + "io" +) + +const fcDir = ".filecache" + +// Guest is the subset of guest operations the sync engine needs; the engine +// package implements it over silkd. All paths are absolute guest paths. +type Guest interface { + Run(ctx context.Context, vsockSocket string, argv ...string) (string, error) + WriteFile(ctx context.Context, vsockSocket, path string, mode uint32, data []byte) error + ReadFile(ctx context.Context, vsockSocket, path string) ([]byte, error) + PushTar(ctx context.Context, vsockSocket, dest string, r io.Reader) error + Remove(ctx context.Context, vsockSocket, path string, recursive bool) error +} + +// entMeta is a workspace entry's identity for change detection. +type entMeta struct { + Kind string `json:"kind"` // f | l + Size int64 `json:"size,omitempty"` + MtimeS int64 `json:"mtime_s,omitempty"` + Target string `json:"target,omitempty"` // symlink target +} + +// journalEntry is one writer's published delta, serialized under +// /.filecache/journal/-.json. +type journalEntry struct { + Writer string `json:"writer"` + Seq uint64 `json:"seq"` + TsNs int64 `json:"ts_ns"` + Puts map[string]entMeta `json:"puts,omitempty"` + Dels []string `json:"dels,omitempty"` +} diff --git a/sandboxd/filecache/session.go b/sandboxd/filecache/session.go new file mode 100644 index 0000000..329b6b5 --- /dev/null +++ b/sandboxd/filecache/session.go @@ -0,0 +1,206 @@ +package filecache + +import ( + "context" + "fmt" + "os" + "path/filepath" + "sync" + "time" + + "github.com/projecteru2/core/log" +) + +// Config tunes the sync cadence. Zero values fall back to defaults that keep +// cross-writer visibility within a 30s budget (push ≤8s + poll ≤4s + NAS +// attribute cache; the workspace NAS mount should use actimeo=1). +type Config struct { + PushInterval time.Duration + PullInterval time.Duration + Mount string // guest workspace dir; default /workspace + + // VMName is the cocoon VM the sandbox runs as; required only for the + // dedicated-disk mode (attach acts on the VM, not the guest). + VMName string + // DedicatedDisk, when true, hot-attaches a fresh ext4 virtio-blk disk and + // mounts it at Mount before hydration, so the workspace is isolated from + // the guest rootfs layer. Requires the Manager to have a Disk driver. + DedicatedDisk bool +} + +func (c Config) withDefaults() Config { + if c.PushInterval <= 0 { + c.PushInterval = 8 * time.Second + } + if c.PullInterval <= 0 { + c.PullInterval = 4 * time.Second + } + if c.Mount == "" { + c.Mount = "/workspace" + } + return c +} + +// Session runs the push/pull loops for one sandbox↔workspace binding. A +// Manager owns one per claimed sandbox that requested a workspace. +type Session struct { + sy *syncer + cfg Config + stop context.CancelFunc + done chan struct{} + id string + dedicated bool + + mu sync.Mutex + stopped bool +} + +// Manager tracks live workspace sessions, keyed by sandbox id. It mirrors the +// egress-listener bookkeeping in the pool Manager: arm on claim, barrier on +// release, all guarded by mu. +type Manager struct { + guest Guest + disk *diskProvisioner // nil disables dedicated-disk mode + mu sync.Mutex + byID map[string]*Session +} + +// NewManager returns a filecache manager driving the guest via g. +func NewManager(g Guest) *Manager { + return &Manager{guest: g, byID: map[string]*Session{}} +} + +// EnableDedicatedDisk turns on the dedicated-workspace-disk mode: sessions that +// request it get a fresh ext4 virtio-blk disk (backed by an image under root, +// sized sizeMB) attached and mounted before hydration. Call once before +// serving. Without this, DedicatedDisk requests fall back to the rootfs layer. +func (m *Manager) EnableDedicatedDisk(d Disk, root string, sizeMB int) { + m.disk = &diskProvisioner{disk: d, guest: m.guest, root: root, sizeMB: sizeMB} +} + +// Arm binds sandbox id (reachable at vsockSocket) to the NAS workspace ws as +// writer, hydrates the guest, and starts the sync loops. Bootstrap runs +// synchronously so the guest workspace is populated before Arm returns; the +// loops then run in the background until Barrier. Arming an id already armed +// is a no-op. +func (m *Manager) Arm(ctx context.Context, id, vsockSocket, ws, writer string, cfg Config) error { + cfg = cfg.withDefaults() + m.mu.Lock() + if _, ok := m.byID[id]; ok { + m.mu.Unlock() + return nil + } + m.mu.Unlock() + + if err := os.MkdirAll(ws, 0o755); err != nil { + return fmt.Errorf("workspace dir: %w", err) + } + // Dedicated-disk mode: attach and mount a fresh ext4 virtio-blk disk at the + // mount before hydration, so the workspace is isolated from the rootfs + // layer. Falls back to the rootfs layer if the feature is off. + dedicated := cfg.DedicatedDisk && m.disk != nil + if dedicated { + if err := m.disk.attachAndMount(ctx, id, cfg.VMName, vsockSocket, cfg.Mount); err != nil { + return fmt.Errorf("workspace disk: %w", err) + } + } + sy := newSyncer(m.guest, vsockSocket, cfg.Mount, ws, writer) + if err := sy.bootstrap(ctx); err != nil { + if dedicated { + m.disk.unmountAndDetach(ctx, id, cfg.VMName, vsockSocket, cfg.Mount) + } + return fmt.Errorf("bootstrap: %w", err) + } + + loopCtx, cancel := context.WithCancel(context.WithoutCancel(ctx)) + sess := &Session{sy: sy, cfg: cfg, stop: cancel, done: make(chan struct{}), id: id, dedicated: dedicated} + m.mu.Lock() + m.byID[id] = sess + m.mu.Unlock() + go sess.run(loopCtx) + log.WithFunc("filecache.Arm").Infof(ctx, "workspace armed for %s (ws=%s writer=%s)", id, ws, writer) + return nil +} + +// Barrier stops the sync loops for id and runs a final push so every local +// change is on the NAS and visible to other clients (close-to-open) before it +// returns. A no-op if id has no session. Bounded internally so a hung guest +// cannot wedge release. +func (m *Manager) Barrier(ctx context.Context, id string) { + m.mu.Lock() + sess := m.byID[id] + delete(m.byID, id) + m.mu.Unlock() + if sess == nil { + return + } + sess.mu.Lock() + sess.stopped = true + sess.mu.Unlock() + sess.stop() + <-sess.done // loops exited; no concurrent cycle + + bctx, cancel := context.WithTimeout(context.WithoutCancel(ctx), 60*time.Second) + defer cancel() + if puts, dels, err := sess.sy.pushCycle(bctx); err != nil { + log.WithFunc("filecache.Barrier").Errorf(ctx, err, "final push for %s", id) + } else if puts+dels > 0 { + log.WithFunc("filecache.Barrier").Infof(ctx, "barrier %s: %d puts %d dels", id, puts, dels) + } + // Dedicated disk: after the final push has drained the workspace to the + // NAS, unmount and detach it (the VM is about to be torn down anyway). + if sess.dedicated && m.disk != nil { + m.disk.unmountAndDetach(bctx, id, sess.cfg.VMName, sess.sy.sock, sess.cfg.Mount) + } +} + +// Has reports whether id currently has a workspace session. +func (m *Manager) Has(id string) bool { + m.mu.Lock() + defer m.mu.Unlock() + _, ok := m.byID[id] + return ok +} + +// run drives the push/pull/heartbeat tickers until ctx is cancelled. +func (s *Session) run(ctx context.Context) { + defer close(s.done) + pushT := time.NewTicker(s.cfg.PushInterval) + pullT := time.NewTicker(s.cfg.PullInterval) + hbT := time.NewTicker(10 * time.Second) + defer pushT.Stop() + defer pullT.Stop() + defer hbT.Stop() + logger := log.WithFunc("filecache.run") + for { + select { + case <-ctx.Done(): + return + case <-pushT.C: + if puts, dels, err := s.sy.pushCycle(ctx); err != nil { + if ctx.Err() == nil { + logger.Warnf(ctx, "push %s: %v", s.id, err) + } + } else if puts+dels > 0 { + logger.Debugf(ctx, "push %s: %d puts %d dels", s.id, puts, dels) + } + case <-pullT.C: + if n, puts, dels, err := s.sy.pullCycle(ctx); err != nil { + if ctx.Err() == nil { + logger.Warnf(ctx, "pull %s: %v", s.id, err) + } + } else if n > 0 { + logger.Debugf(ctx, "pull %s: %d entries (%d puts %d dels)", s.id, n, puts, dels) + } + case <-hbT.C: + s.sy.heartbeat() + } + } +} + +// WorkspaceDir returns the on-NAS path a sandbox's workspace token maps to +// under root. Kept here so the arming call site and any inspector agree on +// layout: /. +func WorkspaceDir(root, token string) string { + return filepath.Join(root, token) +} diff --git a/sandboxd/filecache/sync.go b/sandboxd/filecache/sync.go new file mode 100644 index 0000000..50d4092 --- /dev/null +++ b/sandboxd/filecache/sync.go @@ -0,0 +1,500 @@ +package filecache + +import ( + "archive/tar" + "bytes" + "context" + "encoding/json" + "fmt" + "io" + "os" + "path/filepath" + "sort" + "strconv" + "strings" + "time" +) + +// syncer binds one sandbox (identified by its vsock socket) to one NAS +// workspace as a single writer. It is not safe for concurrent use; the owning +// Session serializes push and pull cycles. +type syncer struct { + guest Guest + sock string // sandbox vsock socket + mount string // guest workspace dir + ws string // NAS workspace dir (host mount) + writer string + + seq uint64 + applied map[string]uint64 // per-peer high-water of journal seq applied + manifest map[string]entMeta // last state common to guest and NAS +} + +func newSyncer(g Guest, sock, mount, ws, writer string) *syncer { + return &syncer{guest: g, sock: sock, mount: mount, ws: ws, writer: writer, + applied: map[string]uint64{}, manifest: map[string]entMeta{}} +} + +// bootstrap hydrates the guest workspace from the NAS tree and records that +// tree as the shared baseline. A restarted writer recovers its seq high-water +// from the journal so it never reuses a seq number. +func (s *syncer) bootstrap(ctx context.Context) error { + for _, d := range []string{filepath.Join(s.ws, fcDir, "journal"), filepath.Join(s.ws, fcDir, "writers")} { + if err := os.MkdirAll(d, 0o755); err != nil { + return err + } + } + if _, err := s.guest.Run(ctx, s.sock, "/bin/sh", "-c", "/usr/bin/mkdir -p "+s.mount); err != nil { + return fmt.Errorf("guest mkdir: %w", err) + } + nas, err := scanNAS(s.ws) + if err != nil { + return err + } + if len(nas) > 0 { + pr, pw := io.Pipe() + go func() { pw.CloseWithError(writeTar(pw, s.ws, nas)) }() + if err := s.guest.PushTar(ctx, s.sock, s.mount, pr); err != nil { + return fmt.Errorf("hydrate: %w", err) + } + } + s.manifest = nas + for w, seq := range maxSeqs(filepath.Join(s.ws, fcDir, "journal")) { + if w == s.writer { + s.seq = seq + } else { + s.applied[w] = seq + } + } + return nil +} + +// pushCycle diffs the guest tree against the baseline and publishes local +// changes to the NAS plus a journal entry. Returns the counts for logging. +func (s *syncer) pushCycle(ctx context.Context) (puts, dels int, err error) { + listing, err := s.guestListing(ctx) + if err != nil { + return 0, 0, err + } + changed := map[string]entMeta{} + for p, m := range listing { + old, ok := s.manifest[p] + if !ok || old != m { + changed[p] = m + } + } + var removed []string + for p := range s.manifest { + if _, ok := listing[p]; !ok { + removed = append(removed, p) + } + } + if len(changed) == 0 && len(removed) == 0 { + return 0, 0, nil + } + + var files []string + for p, m := range changed { + if m.Kind == "f" { + files = append(files, p) + } + } + if len(files) > 0 { + if err := s.publishFiles(ctx, files, changed); err != nil { + return 0, 0, err + } + } + for p, m := range changed { + if m.Kind == "l" { + dst := filepath.Join(s.ws, p) + os.MkdirAll(filepath.Dir(dst), 0o755) + os.Remove(dst) + if err := os.Symlink(m.Target, dst); err != nil { + return 0, 0, fmt.Errorf("symlink %s: %w", p, err) + } + } + } + for _, p := range removed { + if err := os.Remove(filepath.Join(s.ws, p)); err != nil && !os.IsNotExist(err) { + return 0, 0, fmt.Errorf("nas del %s: %w", p, err) + } + } + + s.seq++ + je := journalEntry{Writer: s.writer, Seq: s.seq, TsNs: time.Now().UnixNano(), Puts: changed, Dels: removed} + if err := s.appendJournal(je); err != nil { + return 0, 0, err + } + for p, m := range changed { + s.manifest[p] = m + } + for _, p := range removed { + delete(s.manifest, p) + } + return len(changed), len(removed), nil +} + +// publishFiles tars the changed regular files out of the guest and lands them +// on the NAS with per-file atomic visibility, preserving any peer version that +// diverged from our baseline as a conflict copy. +func (s *syncer) publishFiles(ctx context.Context, files []string, changed map[string]entMeta) error { + confTs := time.Now().Unix() + for _, p := range files { + dst := filepath.Join(s.ws, p) + fi, err := os.Lstat(dst) + if err != nil { + continue // not on NAS yet — clean create + } + base, known := s.manifest[p] + if known && base.Kind == "f" && fi.Size() == base.Size && fi.ModTime().Unix() == base.MtimeS { + continue // NAS matches our baseline — clean fast-forward + } + cp := fmt.Sprintf("%s.fc-conflict-%d", p, confTs) + if err := os.Rename(dst, filepath.Join(s.ws, cp)); err != nil { + continue + } + cfi, _ := os.Lstat(filepath.Join(s.ws, cp)) + changed[cp] = entMeta{Kind: "f", Size: cfi.Size(), MtimeS: cfi.ModTime().Unix()} + // Materialize into our own guest so the user sees it and it does not + // read as a deletion on the next push. Peers get it via the journal. + if b, err := os.ReadFile(filepath.Join(s.ws, cp)); err == nil { + s.guest.WriteFile(ctx, s.sock, filepath.Join(s.mount, cp), 0o644, b) + } + } + sort.Strings(files) + list := strings.Join(files, "\n") + "\n" + if err := s.guest.WriteFile(ctx, s.sock, "/tmp/.fc-list", 0o644, []byte(list)); err != nil { + return err + } + if _, err := s.guest.Run(ctx, s.sock, "/bin/sh", "-c", + "/usr/bin/tar --format=pax -cf /tmp/.fc-push.tar -C "+s.mount+" -T /tmp/.fc-list"); err != nil { + return fmt.Errorf("guest tar: %w", err) + } + data, err := s.guest.ReadFile(ctx, s.sock, "/tmp/.fc-push.tar") + if err != nil { + return err + } + if err := applyTarToNAS(s.ws, bytes.NewReader(data)); err != nil { + return err + } + s.guest.Run(ctx, s.sock, "/bin/sh", "-c", "/bin/rm -f /tmp/.fc-push.tar /tmp/.fc-list") + return nil +} + +// pullCycle applies other writers' journal entries into the guest. +func (s *syncer) pullCycle(ctx context.Context) (entries, puts, dels int, err error) { + pending, err := s.unseenJournal() + if err != nil || len(pending) == 0 { + return 0, 0, 0, err + } + putSet := map[string]entMeta{} + delSet := map[string]bool{} + for _, je := range pending { + for p, m := range je.Puts { + putSet[p] = m + delete(delSet, p) + } + for _, p := range je.Dels { + delSet[p] = true + delete(putSet, p) + } + } + + var tarBuf bytes.Buffer + tw := tar.NewWriter(&tarBuf) + nfiles := 0 + for p, m := range putSet { + switch m.Kind { + case "f": + src := filepath.Join(s.ws, p) + fi, err := os.Lstat(src) + if err != nil { + continue // deleted on NAS since; a later entry covers it + } + hdr := &tar.Header{Name: p, Mode: int64(fi.Mode().Perm()), Size: fi.Size(), + ModTime: fi.ModTime(), Format: tar.FormatPAX} + if err := tw.WriteHeader(hdr); err != nil { + return 0, 0, 0, err + } + f, err := os.Open(src) + if err != nil { + return 0, 0, 0, err + } + if _, err := io.Copy(tw, f); err != nil { + f.Close() + return 0, 0, 0, err + } + f.Close() + m.Size, m.MtimeS = fi.Size(), fi.ModTime().Unix() + putSet[p] = m + nfiles++ + case "l": + s.guest.Run(ctx, s.sock, "/bin/sh", "-c", + "/bin/ln -sfn "+shq(m.Target)+" "+shq(filepath.Join(s.mount, p))) + } + } + tw.Close() + if nfiles > 0 { + if err := s.guest.PushTar(ctx, s.sock, s.mount, &tarBuf); err != nil { + return 0, 0, 0, fmt.Errorf("apply: %w", err) + } + } + for p := range delSet { + s.guest.Remove(ctx, s.sock, filepath.Join(s.mount, p), false) + } + + for p, m := range putSet { + s.manifest[p] = m + } + for p := range delSet { + delete(s.manifest, p) + } + for _, je := range pending { + if je.Seq > s.applied[je.Writer] { + s.applied[je.Writer] = je.Seq + } + } + return len(pending), nfiles, len(delSet), nil +} + +// guestListing walks the guest workspace: files and symlinks with metadata. +func (s *syncer) guestListing(ctx context.Context) (map[string]entMeta, error) { + out, err := s.guest.Run(ctx, s.sock, "/bin/sh", "-c", + "cd "+s.mount+" && /usr/bin/find . \\( -type f -o -type l \\) -printf '%y\\t%P\\t%T@\\t%s\\t%l\\n'") + if err != nil { + return nil, fmt.Errorf("guest find: %w", err) + } + return parseListing(out), nil +} + +func parseListing(out string) map[string]entMeta { + m := map[string]entMeta{} + for _, line := range strings.Split(out, "\n") { + parts := strings.SplitN(line, "\t", 5) + if len(parts) < 4 || parts[1] == "" { + continue + } + p := parts[1] + if p == fcDir || strings.HasPrefix(p, fcDir+"/") || strings.HasPrefix(p, ".fc-") { + continue + } + mt, _ := strconv.ParseFloat(parts[2], 64) + sz, _ := strconv.ParseInt(parts[3], 10, 64) + e := entMeta{Kind: parts[0], Size: sz, MtimeS: int64(mt)} + if e.Kind == "l" { + e.Size = 0 + if len(parts) == 5 { + e.Target = parts[4] + } + } + m[p] = e + } + return m +} + +// applyTarToNAS extracts a guest-produced tar onto the NAS workspace with +// per-file atomic visibility (temp name + rename). +func applyTarToNAS(ws string, r io.Reader) error { + tr := tar.NewReader(r) + for { + hdr, err := tr.Next() + if err == io.EOF { + return nil + } + if err != nil { + return err + } + rel := filepath.Clean(strings.TrimPrefix(hdr.Name, "./")) + if rel == "." || strings.HasPrefix(rel, fcDir) { + continue + } + dst := filepath.Join(ws, rel) + switch hdr.Typeflag { + case tar.TypeDir: + os.MkdirAll(dst, 0o755) + case tar.TypeSymlink: + os.MkdirAll(filepath.Dir(dst), 0o755) + os.Remove(dst) + if err := os.Symlink(hdr.Linkname, dst); err != nil { + return err + } + case tar.TypeReg: + os.MkdirAll(filepath.Dir(dst), 0o755) + tmp := fmt.Sprintf("%s.fc-tmp.%d", dst, os.Getpid()) + f, err := os.OpenFile(tmp, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, os.FileMode(hdr.Mode).Perm()) + if err != nil { + return err + } + if _, err := io.Copy(f, tr); err != nil { //nolint:gosec // guest-produced tar, bounded by workspace + f.Close() + os.Remove(tmp) + return err + } + if err := f.Close(); err != nil { + os.Remove(tmp) + return err + } + os.Chtimes(tmp, hdr.ModTime, hdr.ModTime) + if err := os.Rename(tmp, dst); err != nil { + os.Remove(tmp) + return err + } + } + } +} + +// writeTar streams the NAS entries as a tar (hydration). +func writeTar(w io.Writer, root string, ents map[string]entMeta) error { + tw := tar.NewWriter(w) + paths := make([]string, 0, len(ents)) + for p := range ents { + paths = append(paths, p) + } + sort.Strings(paths) + for _, p := range paths { + m := ents[p] + src := filepath.Join(root, p) + switch m.Kind { + case "l": + if err := tw.WriteHeader(&tar.Header{Name: p, Typeflag: tar.TypeSymlink, + Linkname: m.Target, Format: tar.FormatPAX}); err != nil { + return err + } + case "f": + fi, err := os.Lstat(src) + if err != nil { + return err + } + if err := tw.WriteHeader(&tar.Header{Name: p, Mode: int64(fi.Mode().Perm()), + Size: fi.Size(), ModTime: fi.ModTime(), Format: tar.FormatPAX}); err != nil { + return err + } + f, err := os.Open(src) + if err != nil { + return err + } + if _, err := io.Copy(tw, f); err != nil { + f.Close() + return err + } + f.Close() + } + } + return tw.Close() +} + +// scanNAS walks the NAS workspace (excluding .filecache): files + symlinks. +func scanNAS(root string) (map[string]entMeta, error) { + out := map[string]entMeta{} + err := filepath.WalkDir(root, func(p string, d os.DirEntry, err error) error { + if err != nil { + return err + } + rel, _ := filepath.Rel(root, p) + if rel == "." { + return nil + } + if d.IsDir() { + if rel == fcDir { + return filepath.SkipDir + } + return nil + } + fi, err := d.Info() + if err != nil { + return err + } + if fi.Mode()&os.ModeSymlink != 0 { + tgt, _ := os.Readlink(p) + out[rel] = entMeta{Kind: "l", Target: tgt} + } else if fi.Mode().IsRegular() { + out[rel] = entMeta{Kind: "f", Size: fi.Size(), MtimeS: fi.ModTime().Unix()} + } + return nil + }) + return out, err +} + +func (s *syncer) appendJournal(je journalEntry) error { + b, _ := json.Marshal(je) + dir := filepath.Join(s.ws, fcDir, "journal") + name := fmt.Sprintf("%s-%016d.json", s.writer, je.Seq) + tmp := filepath.Join(dir, "."+name+".tmp") + if err := os.WriteFile(tmp, b, 0o644); err != nil { + return err + } + if err := os.Rename(tmp, filepath.Join(dir, name)); err != nil { + return err + } + seqTmp := filepath.Join(s.ws, fcDir, ".seq.tmp."+s.writer) + os.WriteFile(seqTmp, []byte(fmt.Sprintf("%d %s\n", je.TsNs, s.writer)), 0o644) + return os.Rename(seqTmp, filepath.Join(s.ws, fcDir, "seq")) +} + +// unseenJournal lists journal entries from other writers newer than applied. +func (s *syncer) unseenJournal() ([]journalEntry, error) { + dir := filepath.Join(s.ws, fcDir, "journal") + ents, err := os.ReadDir(dir) + if err != nil { + if os.IsNotExist(err) { + return nil, nil + } + return nil, err + } + var out []journalEntry + for _, e := range ents { + w, seq, ok := parseJournalName(e.Name()) + if !ok || w == s.writer || seq <= s.applied[w] { + continue + } + b, err := os.ReadFile(filepath.Join(dir, e.Name())) + if err != nil { + continue + } + var je journalEntry + if json.Unmarshal(b, &je) == nil { + out = append(out, je) + } + } + sort.Slice(out, func(i, j int) bool { return out[i].TsNs < out[j].TsNs }) + return out, nil +} + +func maxSeqs(dir string) map[string]uint64 { + res := map[string]uint64{} + ents, err := os.ReadDir(dir) + if err != nil { + return res + } + for _, e := range ents { + if w, seq, ok := parseJournalName(e.Name()); ok && seq > res[w] { + res[w] = seq + } + } + return res +} + +func parseJournalName(name string) (writer string, seq uint64, ok bool) { + i := strings.LastIndex(name, "-") + if i < 0 || !strings.HasSuffix(name, ".json") { + return "", 0, false + } + seq, err := strconv.ParseUint(strings.TrimSuffix(name[i+1:], ".json"), 10, 64) + if err != nil { + return "", 0, false + } + return name[:i], seq, true +} + +func (s *syncer) heartbeat() { + p := filepath.Join(s.ws, fcDir, "writers", s.writer+".json") + b, _ := json.Marshal(map[string]any{"writer": s.writer, "ts": time.Now().UnixNano()}) + tmp := p + ".tmp" + if os.WriteFile(tmp, b, 0o644) == nil { + os.Rename(tmp, p) + } +} + +func shq(s string) string { + return "'" + strings.ReplaceAll(s, "'", `'\''`) + "'" +} diff --git a/sandboxd/filecache/sync_test.go b/sandboxd/filecache/sync_test.go new file mode 100644 index 0000000..12a15a5 --- /dev/null +++ b/sandboxd/filecache/sync_test.go @@ -0,0 +1,160 @@ +package filecache + +import ( + "bytes" + "os" + "path/filepath" + "testing" +) + +func TestParseListing(t *testing.T) { + out := "f\tREADME.md\t1700000000.5\t12\t\n" + + "l\tlink\t1700000001\t0\ttarget/path\n" + + "f\t.filecache/journal/x\t1700000002\t3\t\n" + // excluded + "f\tsrc/main.go\t1700000003.9\t20\t\n" + m := parseListing(out) + if len(m) != 3 { + t.Fatalf("want 3 entries, got %d: %v", len(m), m) + } + if e := m["README.md"]; e.Kind != "f" || e.Size != 12 || e.MtimeS != 1700000000 { + t.Errorf("README.md: %+v", e) + } + if e := m["link"]; e.Kind != "l" || e.Target != "target/path" || e.Size != 0 { + t.Errorf("link: %+v", e) + } + if _, ok := m[".filecache/journal/x"]; ok { + t.Error(".filecache entries must be excluded") + } +} + +func TestParseJournalName(t *testing.T) { + cases := []struct { + name string + writer string + seq uint64 + ok bool + }{ + {"sb_abc-0000000000000007.json", "sb_abc", 7, true}, + {"sb_with-dash-0000000000000012.json", "sb_with-dash", 12, true}, + {"seq", "", 0, false}, + {"no-suffix", "", 0, false}, + {"writer-notanumber.json", "", 0, false}, + } + for _, c := range cases { + w, s, ok := parseJournalName(c.name) + if ok != c.ok || w != c.writer || s != c.seq { + t.Errorf("%q: got (%q,%d,%v) want (%q,%d,%v)", c.name, w, s, ok, c.writer, c.seq, c.ok) + } + } +} + +// TestJournalRoundTrip covers appendJournal → unseenJournal → maxSeqs and the +// per-writer applied high-water filter (the core of cross-writer convergence). +func TestJournalRoundTrip(t *testing.T) { + ws := t.TempDir() + must(t, os.MkdirAll(filepath.Join(ws, fcDir, "journal"), 0o755)) + + a := &syncer{ws: ws, writer: "wA", applied: map[string]uint64{}} + b := &syncer{ws: ws, writer: "wB", applied: map[string]uint64{}} + + a.seq = 1 + must(t, a.appendJournal(journalEntry{Writer: "wA", Seq: 1, TsNs: 100, Puts: map[string]entMeta{"f1": {Kind: "f", Size: 3}}})) + a.seq = 2 + must(t, a.appendJournal(journalEntry{Writer: "wA", Seq: 2, TsNs: 200, Dels: []string{"f1"}})) + + // B has seen nothing yet: both entries are unseen, ordered by ts. + got, err := b.unseenJournal() + must(t, err) + if len(got) != 2 || got[0].Seq != 1 || got[1].Seq != 2 { + t.Fatalf("unseen: %+v", got) + } + // A never pulls its own entries. + if own, _ := a.unseenJournal(); len(own) != 0 { + t.Errorf("a saw its own entries: %+v", own) + } + // After recording seq 1 applied, only seq 2 remains. + b.applied["wA"] = 1 + got, _ = b.unseenJournal() + if len(got) != 1 || got[0].Seq != 2 { + t.Fatalf("after applied=1: %+v", got) + } + // maxSeqs recovers the high-water for resume. + if ms := maxSeqs(filepath.Join(ws, fcDir, "journal")); ms["wA"] != 2 { + t.Errorf("maxSeqs wA = %d, want 2", ms["wA"]) + } + // The seq file is freshened for the puller fast-path. + if _, err := os.Stat(filepath.Join(ws, fcDir, "seq")); err != nil { + t.Errorf("seq file: %v", err) + } +} + +// TestTarRoundTrip covers scanNAS → writeTar → applyTarToNAS: the workspace +// tree survives a hydrate/publish round-trip byte-for-byte, and .filecache is +// never carried. +func TestTarRoundTrip(t *testing.T) { + src := t.TempDir() + must(t, os.MkdirAll(filepath.Join(src, "src"), 0o755)) + must(t, os.MkdirAll(filepath.Join(src, fcDir, "journal"), 0o755)) + must(t, os.WriteFile(filepath.Join(src, "README.md"), []byte("seed"), 0o644)) + must(t, os.WriteFile(filepath.Join(src, "src", "main.go"), []byte("package main\n"), 0o600)) + must(t, os.Symlink("README.md", filepath.Join(src, "readme-link"))) + must(t, os.WriteFile(filepath.Join(src, fcDir, "journal", "wA-0000000000000001.json"), []byte("{}"), 0o644)) + + ents, err := scanNAS(src) + must(t, err) + if len(ents) != 3 { // README, src/main.go, readme-link; .filecache excluded + t.Fatalf("scanNAS: %d entries %v", len(ents), ents) + } + + var buf bytes.Buffer + must(t, writeTar(&buf, src, ents)) + + dst := t.TempDir() + must(t, applyTarToNAS(dst, &buf)) + + if b, _ := os.ReadFile(filepath.Join(dst, "README.md")); string(b) != "seed" { + t.Errorf("README.md content: %q", b) + } + if b, _ := os.ReadFile(filepath.Join(dst, "src", "main.go")); string(b) != "package main\n" { + t.Errorf("main.go content: %q", b) + } + if tgt, err := os.Readlink(filepath.Join(dst, "readme-link")); err != nil || tgt != "README.md" { + t.Errorf("symlink: %q %v", tgt, err) + } + fi, _ := os.Stat(filepath.Join(dst, "src", "main.go")) + if fi.Mode().Perm() != 0o600 { + t.Errorf("mode not preserved: %v", fi.Mode()) + } + if _, err := os.Stat(filepath.Join(dst, fcDir)); !os.IsNotExist(err) { + t.Error(".filecache must not be carried in the tar") + } +} + +// TestApplyTarAtomicOverwrite verifies a file lands via rename with no +// leftover temp file and correct final content. +func TestApplyTarAtomicOverwrite(t *testing.T) { + dst := t.TempDir() + must(t, os.WriteFile(filepath.Join(dst, "f"), []byte("old"), 0o644)) + + src := t.TempDir() + must(t, os.WriteFile(filepath.Join(src, "f"), []byte("new-content"), 0o644)) + ents, _ := scanNAS(src) + var buf bytes.Buffer + must(t, writeTar(&buf, src, ents)) + must(t, applyTarToNAS(dst, &buf)) + + if b, _ := os.ReadFile(filepath.Join(dst, "f")); string(b) != "new-content" { + t.Errorf("overwrite: %q", b) + } + matches, _ := filepath.Glob(filepath.Join(dst, "*.fc-tmp.*")) + if len(matches) != 0 { + t.Errorf("temp files left behind: %v", matches) + } +} + +func must(t *testing.T, err error) { + t.Helper() + if err != nil { + t.Fatal(err) + } +} diff --git a/sandboxd/main.go b/sandboxd/main.go index 965fd48..58a0a48 100644 --- a/sandboxd/main.go +++ b/sandboxd/main.go @@ -13,6 +13,7 @@ import ( "net/http" "os" "os/signal" + "path/filepath" "runtime/debug" "slices" "syscall" @@ -89,6 +90,15 @@ func main() { if err != nil { logger.Fatalf(ctx, err, "init pool manager") } + if cfg.WorkspaceRoot != "" { + mgr.EnableWorkspaceSync(guestFS{eng}, cfg.WorkspaceRoot) + logger.Infof(ctx, "workspace filecache enabled (root %s)", cfg.WorkspaceRoot) + if cfg.WorkspaceDiskMB > 0 { + diskDir := cmp.Or(cfg.WorkspaceDiskDir, filepath.Join(cfg.DataDir, "workspaces")) + mgr.EnableWorkspaceDisk(diskFS{eng}, diskDir, cfg.WorkspaceDiskMB) + logger.Infof(ctx, "workspace dedicated disk enabled (%d MB, dir %s)", cfg.WorkspaceDiskMB, diskDir) + } + } ctx, stop := signal.NotifyContext(ctx, os.Interrupt, syscall.SIGTERM) defer stop() @@ -238,3 +248,37 @@ func versionString() string { } return version + "-" + info.Settings[i].Value[:12] } + +// guestFS adapts *engine.Engine to filecache.Guest (the engine namespaces its +// guest ops as Guest*). +type guestFS struct{ e *engine.Engine } + +func (g guestFS) Run(ctx context.Context, sock string, argv ...string) (string, error) { + return g.e.GuestRun(ctx, sock, argv...) +} +func (g guestFS) WriteFile(ctx context.Context, sock, path string, mode uint32, data []byte) error { + return g.e.GuestWriteFile(ctx, sock, path, mode, data) +} +func (g guestFS) ReadFile(ctx context.Context, sock, path string) ([]byte, error) { + return g.e.GuestReadFile(ctx, sock, path) +} +func (g guestFS) PushTar(ctx context.Context, sock, dest string, r io.Reader) error { + return g.e.GuestPushTar(ctx, sock, dest, r) +} +func (g guestFS) Remove(ctx context.Context, sock, path string, recursive bool) error { + return g.e.GuestRemove(ctx, sock, path, recursive) +} + +// diskFS adapts *engine.Engine to filecache.Disk (read-write workspace disks, +// reusing cocoon's disk-attach and by-serial device discovery). +type diskFS struct{ e *engine.Engine } + +func (d diskFS) Attach(ctx context.Context, vmName, rawPath, name string) error { + return d.e.WorkspaceDiskAttach(ctx, vmName, rawPath, name) +} +func (d diskFS) Mount(ctx context.Context, sock, name, mount string) error { + return d.e.WorkspaceDiskMount(ctx, sock, name, mount) +} +func (d diskFS) Detach(ctx context.Context, vmName, name string) error { + return d.e.WorkspaceDiskDetach(ctx, vmName, name) +} diff --git a/sandboxd/pool/archive_test.go b/sandboxd/pool/archive_test.go index d26957e..f40734e 100644 --- a/sandboxd/pool/archive_test.go +++ b/sandboxd/pool/archive_test.go @@ -306,7 +306,7 @@ func TestArchiveKeepsForeverWhenDeleteZero(t *testing.T) { func TestArchiveRetentionPurge(t *testing.T) { eng := newFakeEngine() m := newTestManager(t, eng, archivePool(3600)) - sb, err := m.ClaimProvision(t.Context(), testKey, time.Hour, "acme", "", nil) + sb, err := m.ClaimProvision(t.Context(), testKey, time.Hour, "acme", "", "", nil) if err != nil { t.Fatalf("claim: %v", err) } diff --git a/sandboxd/pool/checkpoint_test.go b/sandboxd/pool/checkpoint_test.go index a924463..80b35fe 100644 --- a/sandboxd/pool/checkpoint_test.go +++ b/sandboxd/pool/checkpoint_test.go @@ -120,11 +120,11 @@ func TestRetentionSweepsExpiredCheckpoints(t *testing.T) { func TestCheckpointTenantIsolation(t *testing.T) { eng := newFakeEngine() m := newTestManager(t, eng) - srcA, err := m.ClaimProvision(t.Context(), testKey, time.Hour, "acme", "", nil) + srcA, err := m.ClaimProvision(t.Context(), testKey, time.Hour, "acme", "", "", nil) if err != nil { t.Fatalf("acme claim: %v", err) } - srcB, err := m.ClaimProvision(t.Context(), testKey, time.Hour, "beta", "", nil) + srcB, err := m.ClaimProvision(t.Context(), testKey, time.Hour, "beta", "", "", nil) if err != nil { t.Fatalf("beta claim: %v", err) } diff --git a/sandboxd/pool/claim.go b/sandboxd/pool/claim.go index d4b9b14..f77b694 100644 --- a/sandboxd/pool/claim.go +++ b/sandboxd/pool/claim.go @@ -9,6 +9,7 @@ import ( "github.com/projecteru2/core/log" + "github.com/cocoonstack/sandbox/sandboxd/filecache" "github.com/cocoonstack/sandbox/sandboxd/types" ) @@ -25,7 +26,7 @@ const ( // ErrNoWarm means the pool is empty (the caller may redirect or provision). // tenant attributes the claim; empty means the operator (root). claimRef is an // opaque caller reference recorded on the claim; empty means none. -func (m *Manager) ClaimWarm(ctx context.Context, key types.PoolKey, ttl time.Duration, tenant, claimRef string, volumes []types.Volume) (*types.Sandbox, error) { +func (m *Manager) ClaimWarm(ctx context.Context, key types.PoolKey, ttl time.Duration, tenant, claimRef, workspace string, volumes []types.Volume) (*types.Sandbox, error) { start := time.Now() if err := m.validate(key); err != nil { return nil, err @@ -57,6 +58,7 @@ func (m *Manager) ClaimWarm(ctx context.Context, key types.PoolKey, ttl time.Dur } sb.Tenant = tenant sb.ClaimRef = claimRef + sb.Workspace = workspace out, err := m.finalize(ctx, sb, ttl) if err == nil { m.counters.claimsWarm.Add(1) @@ -67,14 +69,14 @@ func (m *Manager) ClaimWarm(ctx context.Context, key types.PoolKey, ttl time.Dur // ClaimProvision creates a claim-ready sandbox (golden clone or cold boot). // claimRef is an opaque caller reference recorded on the claim; empty means none. -func (m *Manager) ClaimProvision(ctx context.Context, key types.PoolKey, ttl time.Duration, tenant, claimRef string, volumes []types.Volume) (*types.Sandbox, error) { - return m.claimProvision(ctx, key, ttl, tenant, claimRef, volumes, false) +func (m *Manager) ClaimProvision(ctx context.Context, key types.PoolKey, ttl time.Duration, tenant, claimRef, workspace string, volumes []types.Volume) (*types.Sandbox, error) { + return m.claimProvision(ctx, key, ttl, tenant, claimRef, workspace, volumes, false) } // ClaimProvisionPromoted requires key to resolve from a promoted template and // never falls through to a cold image boot. -func (m *Manager) ClaimProvisionPromoted(ctx context.Context, key types.PoolKey, ttl time.Duration, tenant, claimRef string, volumes []types.Volume) (*types.Sandbox, error) { - return m.claimProvision(ctx, key, ttl, tenant, claimRef, volumes, true) +func (m *Manager) ClaimProvisionPromoted(ctx context.Context, key types.PoolKey, ttl time.Duration, tenant, claimRef, workspace string, volumes []types.Volume) (*types.Sandbox, error) { + return m.claimProvision(ctx, key, ttl, tenant, claimRef, workspace, volumes, true) } // Release destroys a claimed sandbox after authorizing cred. @@ -175,6 +177,9 @@ func (m *Manager) releaseResolved(ctx context.Context, id string, sb *types.Sand } // Cleanup must survive the caller hanging up; the claim is already dropped. ctx = context.WithoutCancel(ctx) + // Barrier before the VM is torn down: publish the workspace one last time + // so every local change is on the NAS and visible to other clients. + m.barrierWorkspace(ctx, id) if ck != "" { m.purgeArchiveCk(ctx, id, ck, sb.Tenant) // archived: no local VM m.untrack(m.pendingCks, ck) @@ -280,9 +285,59 @@ func (m *Manager) finalizeBatch(ctx context.Context, sbs []*types.Sandbox, ttl t KeyHash: sb.Key.Hash(), Tenant: sb.Tenant, Volumes: types.VolumeNames(sb.Volumes), }) } + // Workspace sync is a best-effort enhancement: a failure to bind the NAS + // workspace must not fail an otherwise-good claim, so it arms after the + // batch is durable and armed, and only logs on error. + for _, sb := range sbs { + m.armWorkspace(ctx, sb) + } return nil } +// armWorkspace binds a claimed sandbox to its shared workspace and starts the +// sync loops. No-op when workspace sync is disabled or the sandbox carries no +// workspace. Best-effort: an error is logged, not returned. +func (m *Manager) armWorkspace(ctx context.Context, sb *types.Sandbox) { + if m.wsMgr == nil || sb.Workspace == "" || sb.VsockSocket == "" { + return + } + ws := filecache.WorkspaceDir(m.wsRoot, sb.Workspace) + // One writer per sandbox: the sandbox id is globally unique, so it is a + // safe, collision-free writer id across nodes. + cfg := filecache.Config{VMName: sb.VMName, DedicatedDisk: m.wsDedicatedDisk} + if err := m.wsMgr.Arm(ctx, sb.ID, sb.VsockSocket, ws, sb.ID, cfg); err != nil { + log.WithFunc("pool.armWorkspace").Errorf(ctx, err, "arm workspace %s for %s", sb.Workspace, sb.ID) + } +} + +// barrierWorkspace runs the final workspace sync for id (publish local changes +// to the NAS) and stops its loops. No-op when sync is disabled or id has none. +func (m *Manager) barrierWorkspace(ctx context.Context, id string) { + if m.wsMgr == nil { + return + } + m.wsMgr.Barrier(ctx, id) +} + +// EnableWorkspaceSync turns on the workspace filecache, driving guests through +// g and resolving workspace tokens under root. Call once before serving. +func (m *Manager) EnableWorkspaceSync(g filecache.Guest, root string) { + m.wsMgr = filecache.NewManager(g) + m.wsRoot = root +} + +// EnableWorkspaceDisk turns on the dedicated-workspace-disk mode for every +// workspace claim: a fresh ext4 virtio-blk disk (image under imageRoot, sized +// sizeMB) is attached read-write and mounted before hydration, isolating the +// workspace from the guest rootfs layer. Requires EnableWorkspaceSync first. +func (m *Manager) EnableWorkspaceDisk(d filecache.Disk, imageRoot string, sizeMB int) { + if m.wsMgr == nil { + return + } + m.wsMgr.EnableDedicatedDisk(d, imageRoot, sizeMB) + m.wsDedicatedDisk = true +} + // rollbackClaim unwinds a claim batch after a persist or egress-arm failure: // drop the claims, reconverge the journal, destroy the VMs — a NIC that // cannot be locked must never be handed out. @@ -408,6 +463,7 @@ func (m *Manager) reapOnce(ctx context.Context) { case reapArchive: logSweepResult(ctx, logger, m.archive(ctx, v.sb), "archived expired sandbox "+v.id, "archive expired sandbox "+v.id) default: + m.barrierWorkspace(ctx, v.id) m.disarmEgress(v.id, m.removeOrRetry(ctx, v.vmName, v.id, "")) m.dropSnap(ctx, v.snap) m.counters.reaps.Add(1) @@ -461,7 +517,7 @@ func (m *Manager) authed(id, token string) (*types.Sandbox, bool) { return sb, true } -func (m *Manager) claimProvision(ctx context.Context, key types.PoolKey, ttl time.Duration, tenant, claimRef string, volumes []types.Volume, requirePromoted bool) (*types.Sandbox, error) { +func (m *Manager) claimProvision(ctx context.Context, key types.PoolKey, ttl time.Duration, tenant, claimRef, workspace string, volumes []types.Volume, requirePromoted bool) (*types.Sandbox, error) { start := time.Now() if err := m.validate(key); err != nil { return nil, err @@ -493,6 +549,7 @@ func (m *Manager) claimProvision(ctx context.Context, key types.PoolKey, ttl tim sb.TemplateDigest = golden.templateDigest sb.Tenant = tenant sb.ClaimRef = claimRef + sb.Workspace = workspace out, err := m.finalize(ctx, sb, ttl) if err == nil { if golden.dir != "" { diff --git a/sandboxd/pool/drain_test.go b/sandboxd/pool/drain_test.go index 2332633..a03d2e5 100644 --- a/sandboxd/pool/drain_test.go +++ b/sandboxd/pool/drain_test.go @@ -26,10 +26,10 @@ func TestDrainRefusesClaimsTrimsWarmAndUncordonRefills(t *testing.T) { m.Drain(t.Context()) - if _, err := m.ClaimWarm(t.Context(), testKey, 0, "", "", nil); !errors.Is(err, ErrQuota) { + if _, err := m.ClaimWarm(t.Context(), testKey, 0, "", "", "", nil); !errors.Is(err, ErrQuota) { t.Fatalf("ClaimWarm during drain: %v, want ErrQuota", err) } - if _, err := m.ClaimProvision(t.Context(), testKey, 0, "", "", nil); !errors.Is(err, ErrQuota) { + if _, err := m.ClaimProvision(t.Context(), testKey, 0, "", "", "", nil); !errors.Is(err, ErrQuota) { t.Fatalf("ClaimProvision during drain: %v, want ErrQuota", err) } infos, g := m.Info() diff --git a/sandboxd/pool/fork_test.go b/sandboxd/pool/fork_test.go index 49d8412..2ef7391 100644 --- a/sandboxd/pool/fork_test.go +++ b/sandboxd/pool/fork_test.go @@ -230,7 +230,7 @@ func TestForkChildrenInheritTenantAndQuota(t *testing.T) { eng := newFakeEngine() m := newTestManager(t, eng) m.tenantMax = map[string]int{"acme": 3} - parent, err := m.ClaimProvision(t.Context(), testKey, time.Hour, "acme", "", nil) + parent, err := m.ClaimProvision(t.Context(), testKey, time.Hour, "acme", "", "", nil) if err != nil { t.Fatalf("claim: %v", err) } diff --git a/sandboxd/pool/idle_test.go b/sandboxd/pool/idle_test.go index 71f5919..f06c759 100644 --- a/sandboxd/pool/idle_test.go +++ b/sandboxd/pool/idle_test.go @@ -46,7 +46,7 @@ func TestIdleOncePolicyScope(t *testing.T) { // An unpooled key (template claim shape) takes the node default. unpooled := types.PoolKey{Template: "tpl:v1", Net: types.NetNone, Size: types.SizeSmall} - sb2, err := m.ClaimProvision(t.Context(), unpooled, time.Hour, "", "", nil) + sb2, err := m.ClaimProvision(t.Context(), unpooled, time.Hour, "", "", "", nil) if err != nil { t.Fatalf("provision: %v", err) } diff --git a/sandboxd/pool/pool.go b/sandboxd/pool/pool.go index 07525ae..d38896f 100644 --- a/sandboxd/pool/pool.go +++ b/sandboxd/pool/pool.go @@ -27,6 +27,7 @@ import ( "github.com/cocoonstack/sandbox/sandboxd/config" "github.com/cocoonstack/sandbox/sandboxd/egress" "github.com/cocoonstack/sandbox/sandboxd/engine" + "github.com/cocoonstack/sandbox/sandboxd/filecache" "github.com/cocoonstack/sandbox/sandboxd/netfilter" "github.com/cocoonstack/sandbox/sandboxd/store" "github.com/cocoonstack/sandbox/sandboxd/store/dir" @@ -244,7 +245,15 @@ type Manager struct { egress bool maxFork int store *claimStore - volumes map[string]catalogVolume + + // wsMgr syncs claimed sandboxes' workspaces with a shared NAS root; nil + // (the default) disables the feature so arm/barrier are no-ops. wsRoot is + // the host mount a claim's workspace token resolves under. Set once before + // serving via EnableWorkspaceSync/EnableWorkspaceDisk. + wsMgr *filecache.Manager + wsRoot string + wsDedicatedDisk bool + volumes map[string]catalogVolume poolStore *poolStore configSeedHash string // config pools' hash, to warn when a file edit is overridden diff --git a/sandboxd/pool/pool_test.go b/sandboxd/pool/pool_test.go index 9247688..652a230 100644 --- a/sandboxd/pool/pool_test.go +++ b/sandboxd/pool/pool_test.go @@ -705,9 +705,9 @@ func newTestManager(t *testing.T, eng *fakeEngine, pools ...config.PoolSpec) *Ma // claimAny composes warm-then-provision the way the server does around the // redirect decision; production has no single-call form. func claimAny(ctx context.Context, m *Manager, key types.PoolKey, ttl time.Duration) (*types.Sandbox, error) { - sb, err := m.ClaimWarm(ctx, key, ttl, "", "", nil) + sb, err := m.ClaimWarm(ctx, key, ttl, "", "", "", nil) if errors.Is(err, ErrNoWarm) { - return m.ClaimProvision(ctx, key, ttl, "", "", nil) + return m.ClaimProvision(ctx, key, ttl, "", "", "", nil) } return sb, err } diff --git a/sandboxd/pool/promote_test.go b/sandboxd/pool/promote_test.go index 916454e..a6fa142 100644 --- a/sandboxd/pool/promote_test.go +++ b/sandboxd/pool/promote_test.go @@ -213,7 +213,7 @@ func TestResolveGoldenSkipsPromotedEgressTemplate(t *testing.T) { func TestTemplateTenantScopedDelete(t *testing.T) { eng := newFakeEngine() m := newTestManager(t, eng) - parent, err := m.ClaimProvision(t.Context(), testKey, time.Hour, "acme", "", nil) + parent, err := m.ClaimProvision(t.Context(), testKey, time.Hour, "acme", "", "", nil) if err != nil { t.Fatalf("claim: %v", err) } @@ -269,7 +269,7 @@ func TestPromoteFailsClosedOnMetaError(t *testing.T) { } eng := newFakeEngine() m := newTestManager(t, eng) - a, err := m.ClaimProvision(t.Context(), testKey, time.Hour, "acme", "", nil) + a, err := m.ClaimProvision(t.Context(), testKey, time.Hour, "acme", "", "", nil) if err != nil { t.Fatalf("claim: %v", err) } @@ -292,7 +292,7 @@ func TestPromoteRefusesCrossTenantOverwrite(t *testing.T) { m := newTestManager(t, eng) claim := func(tenant string) *types.Sandbox { t.Helper() - sb, err := m.ClaimProvision(t.Context(), testKey, time.Hour, tenant, "", nil) + sb, err := m.ClaimProvision(t.Context(), testKey, time.Hour, tenant, "", "", nil) if err != nil { t.Fatalf("claim %q: %v", tenant, err) } diff --git a/sandboxd/pool/telemetry_test.go b/sandboxd/pool/telemetry_test.go index d573cf3..f2c738d 100644 --- a/sandboxd/pool/telemetry_test.go +++ b/sandboxd/pool/telemetry_test.go @@ -99,22 +99,22 @@ func TestTenantQuotaBindsPerTenant(t *testing.T) { m := newTestManager(t, eng) m.tenantMax = map[string]int{"acme": 1, "beta": 2} - first, err := m.ClaimProvision(t.Context(), testKey, time.Hour, "acme", "", nil) + first, err := m.ClaimProvision(t.Context(), testKey, time.Hour, "acme", "", "", nil) if err != nil { t.Fatalf("acme claim: %v", err) } if first.Tenant != "acme" { t.Errorf("tenant %q, want acme", first.Tenant) } - if _, err := m.ClaimProvision(t.Context(), testKey, time.Hour, "acme", "", nil); !errors.Is(err, ErrQuota) { + if _, err := m.ClaimProvision(t.Context(), testKey, time.Hour, "acme", "", "", nil); !errors.Is(err, ErrQuota) { t.Fatalf("acme past its cap: %v, want ErrQuota", err) } // The node-wide cap (unset here) stays untouched: other tenants and root // keep claiming while acme is full. - if _, err := m.ClaimProvision(t.Context(), testKey, time.Hour, "beta", "", nil); err != nil { + if _, err := m.ClaimProvision(t.Context(), testKey, time.Hour, "beta", "", "", nil); err != nil { t.Errorf("beta claim while acme is at cap: %v", err) } - if _, err := m.ClaimProvision(t.Context(), testKey, time.Hour, "", "", nil); err != nil { + if _, err := m.ClaimProvision(t.Context(), testKey, time.Hour, "", "", "", nil); err != nil { t.Errorf("root claim while acme is at cap: %v", err) } counts := m.TenantClaims() @@ -124,7 +124,7 @@ func TestTenantQuotaBindsPerTenant(t *testing.T) { if err := m.Release(t.Context(), first.ID, Cred{Token: first.Token}); err != nil { t.Fatalf("Release: %v", err) } - if _, err := m.ClaimProvision(t.Context(), testKey, time.Hour, "acme", "", nil); err != nil { + if _, err := m.ClaimProvision(t.Context(), testKey, time.Hour, "acme", "", "", nil); err != nil { t.Errorf("acme claim after release: %v", err) } } @@ -132,7 +132,7 @@ func TestTenantQuotaBindsPerTenant(t *testing.T) { func TestTenantStampedInJournals(t *testing.T) { eng := newFakeEngine() m := newTestManager(t, eng) - sb, err := m.ClaimProvision(t.Context(), testKey, time.Hour, "acme", "", nil) + sb, err := m.ClaimProvision(t.Context(), testKey, time.Hour, "acme", "", "", nil) if err != nil { t.Fatalf("claim: %v", err) } diff --git a/sandboxd/pool/volume_capture_test.go b/sandboxd/pool/volume_capture_test.go index aaf29f3..7af10d3 100644 --- a/sandboxd/pool/volume_capture_test.go +++ b/sandboxd/pool/volume_capture_test.go @@ -75,7 +75,7 @@ func TestReconcileRetainsVolumeCaptureGateWithoutCatalog(t *testing.T) { if err != nil { t.Fatalf("setup manager: %v", err) } - sb, err := m.ClaimProvision(t.Context(), testKey, 0, "", "", []types.Volume{{Name: "dataset", Mount: "/datasets"}}) + sb, err := m.ClaimProvision(t.Context(), testKey, 0, "", "", "", []types.Volume{{Name: "dataset", Mount: "/datasets"}}) if err != nil { t.Fatalf("ClaimProvision: %v", err) } diff --git a/sandboxd/pool/volume_test.go b/sandboxd/pool/volume_test.go index 1b9d761..a4aa59b 100644 --- a/sandboxd/pool/volume_test.go +++ b/sandboxd/pool/volume_test.go @@ -28,7 +28,7 @@ func TestClaimProvisionAppliesVolumesInOrder(t *testing.T) { {Name: "imagenet", Mount: "/volumes/imagenet"}, } - sb, err := m.ClaimProvision(t.Context(), testKey, 0, "", "", requested) + sb, err := m.ClaimProvision(t.Context(), testKey, 0, "", "", "", requested) if err != nil { t.Fatalf("ClaimProvision: %v", err) } @@ -87,7 +87,7 @@ func TestClaimWarmAppliesVolumesAndRefillsAfterFailure(t *testing.T) { warm := &types.Sandbox{VMName: "sbx-warm", Key: testKey, VsockSocket: "/vsock/warm"} m.pools[testKey].warm = append(m.pools[testKey].warm, warm) - sb, err := m.ClaimWarm(t.Context(), testKey, 0, "", "", []types.Volume{{Name: "data"}}) + sb, err := m.ClaimWarm(t.Context(), testKey, 0, "", "", "", []types.Volume{{Name: "data"}}) if tt.wantClaim && err != nil { t.Fatalf("ClaimWarm: %v", err) } @@ -143,7 +143,7 @@ func TestClaimProvisionVolumeFailureDestroysVM(t *testing.T) { eng.diskAttachCancel = cancel } - _, err := m.ClaimProvision(ctx, testKey, 0, "", "", []types.Volume{{Name: "data"}}) + _, err := m.ClaimProvision(ctx, testKey, 0, "", "", "", []types.Volume{{Name: "data"}}) if err == nil { t.Fatal("ClaimProvision succeeded") } @@ -186,7 +186,7 @@ func TestClaimProvisionRejectsInvalidVolumesBeforeProvision(t *testing.T) { t.Run(tt.name, func(t *testing.T) { eng := newFakeEngine() m := newVolumeManager(t, eng, tt.catalog) - if _, err := m.ClaimProvision(t.Context(), tt.key, 0, "", "", tt.volumes); !errors.Is(err, ErrBadVolume) { + if _, err := m.ClaimProvision(t.Context(), tt.key, 0, "", "", "", tt.volumes); !errors.Is(err, ErrBadVolume) { t.Errorf("got %v, want ErrBadVolume", err) } if len(eng.colds)+len(eng.clones) != 0 { @@ -201,7 +201,7 @@ func TestClaimProvisionPromotedRejectsMissingTemplateBeforeProvision(t *testing. eng := newFakeEngine() m := newVolumeManager(t, eng, []config.VolumeSpec{{Name: "data", Path: path}}) - _, err := m.ClaimProvisionPromoted(t.Context(), testKey, 0, "", "", []types.Volume{{Name: "data"}}) + _, err := m.ClaimProvisionPromoted(t.Context(), testKey, 0, "", "", "", []types.Volume{{Name: "data"}}) if !errors.Is(err, ErrVolumeUnavailable) { t.Errorf("error=%v, want ErrVolumeUnavailable", err) } @@ -222,7 +222,7 @@ func TestClaimProvisionPromotedAppliesVolumesFromTemplate(t *testing.T) { beforeColds, beforeClones := len(eng.colds), len(eng.clones) eng.volumeOps = nil - sb, err := m.ClaimProvisionPromoted(t.Context(), key, 0, "", "", []types.Volume{{Name: "data"}}) + sb, err := m.ClaimProvisionPromoted(t.Context(), key, 0, "", "", "", []types.Volume{{Name: "data"}}) if err != nil { t.Fatalf("ClaimProvisionPromoted: %v", err) } @@ -246,8 +246,8 @@ func TestClaimProvisionVolumeACL(t *testing.T) { {Name: "public", Path: path}, }) - _, forbiddenErr := m.ClaimProvision(t.Context(), testKey, 0, "beta", "", []types.Volume{{Name: "private"}}) - _, unknownErr := m.ClaimProvision(t.Context(), testKey, 0, "beta", "", []types.Volume{{Name: "unknown"}}) + _, forbiddenErr := m.ClaimProvision(t.Context(), testKey, 0, "beta", "", "", []types.Volume{{Name: "private"}}) + _, unknownErr := m.ClaimProvision(t.Context(), testKey, 0, "beta", "", "", []types.Volume{{Name: "unknown"}}) if forbiddenErr == nil || unknownErr == nil || forbiddenErr.Error() != unknownErr.Error() { t.Fatalf("forbidden=%q unknown=%q, want byte-identical errors", forbiddenErr, unknownErr) } @@ -265,7 +265,7 @@ func TestClaimProvisionVolumeACL(t *testing.T) { {"public", "beta", "public"}, } { t.Run(tt.name, func(t *testing.T) { - if _, err := m.ClaimProvision(t.Context(), testKey, 0, tt.tenant, "", []types.Volume{{Name: tt.volume}}); err != nil { + if _, err := m.ClaimProvision(t.Context(), testKey, 0, tt.tenant, "", "", []types.Volume{{Name: tt.volume}}); err != nil { t.Errorf("ClaimProvision: %v", err) } }) @@ -349,11 +349,11 @@ func TestVolumeClaimUsageAndScopedSummaries(t *testing.T) { m := newVolumeManager(t, newFakeEngine(), []config.VolumeSpec{{Name: "data", Path: path}}) request := []types.Volume{{Name: "data", Mount: "/datasets"}} wantApplied := []types.Volume{{Name: "data", Mount: "/datasets"}} - acme, err := m.ClaimProvision(t.Context(), testKey, 0, "acme", "", request) + acme, err := m.ClaimProvision(t.Context(), testKey, 0, "acme", "", "", request) if err != nil { t.Fatalf("acme claim: %v", err) } - _, betaErr := m.ClaimProvision(t.Context(), testKey, 0, "beta", "", request) + _, betaErr := m.ClaimProvision(t.Context(), testKey, 0, "beta", "", "", request) if betaErr != nil { t.Fatalf("beta claim: %v", betaErr) } @@ -391,7 +391,7 @@ func TestClaimProvisionRejectsMissingVolumePathBeforeProvision(t *testing.T) { eng := newFakeEngine() m := newVolumeManager(t, eng, []config.VolumeSpec{{Name: "data", Path: filepath.Join(t.TempDir(), "missing.img")}}) - _, err := m.ClaimProvision(t.Context(), testKey, 0, "", "", []types.Volume{{Name: "data"}}) + _, err := m.ClaimProvision(t.Context(), testKey, 0, "", "", "", []types.Volume{{Name: "data"}}) if err == nil || !strings.Contains(err.Error(), "missing.img") { t.Errorf("got %v, want missing-path error", err) } diff --git a/sandboxd/server/server.go b/sandboxd/server/server.go index 673ae5e..5b60e51 100644 --- a/sandboxd/server/server.go +++ b/sandboxd/server/server.go @@ -61,9 +61,9 @@ var poolErrHTTP = []struct { // parameters attribute created resources and scope listings/deletes; empty // means the operator (root) — unquotaed, unfiltered. type Manager interface { - ClaimWarm(ctx context.Context, key types.PoolKey, ttl time.Duration, tenant, claimRef string, volumes []types.Volume) (*types.Sandbox, error) - ClaimProvision(ctx context.Context, key types.PoolKey, ttl time.Duration, tenant, claimRef string, volumes []types.Volume) (*types.Sandbox, error) - ClaimProvisionPromoted(ctx context.Context, key types.PoolKey, ttl time.Duration, tenant, claimRef string, volumes []types.Volume) (*types.Sandbox, error) + ClaimWarm(ctx context.Context, key types.PoolKey, ttl time.Duration, tenant, claimRef, workspace string, volumes []types.Volume) (*types.Sandbox, error) + ClaimProvision(ctx context.Context, key types.PoolKey, ttl time.Duration, tenant, claimRef, workspace string, volumes []types.Volume) (*types.Sandbox, error) + ClaimProvisionPromoted(ctx context.Context, key types.PoolKey, ttl time.Duration, tenant, claimRef, workspace string, volumes []types.Volume) (*types.Sandbox, error) Release(ctx context.Context, id string, cred pool.Cred) error Hibernate(ctx context.Context, id string, cred pool.Cred) error Wake(ctx context.Context, id string, cred pool.Cred) error @@ -244,12 +244,12 @@ func (s *Server) handleClaim(w http.ResponseWriter, r *http.Request) { // peer that reports a warm sandbox gets the claim via redirect (data plane // must be direct, so redirect beats proxy); only if no peer has one does // this node provision (golden clone or cold boot). - sb, err := s.mgr.ClaimWarm(r.Context(), key, req.TTL(), tenant, req.ClaimRef, nil) + sb, err := s.mgr.ClaimWarm(r.Context(), key, req.TTL(), tenant, req.ClaimRef, req.Workspace, nil) if errors.Is(err, pool.ErrNoWarm) { if s.redirectClaim(r.Context(), w, req, key, hash) { return } - sb, err = s.mgr.ClaimProvision(r.Context(), key, req.TTL(), tenant, req.ClaimRef, nil) + sb, err = s.mgr.ClaimProvision(r.Context(), key, req.TTL(), tenant, req.ClaimRef, req.Workspace, nil) } // A full node bounces the claim to a warm peer before answering 429 — // quota is per node, and a peer with capacity is a better answer. @@ -282,14 +282,14 @@ func (s *Server) handleVolumeClaim(w http.ResponseWriter, r *http.Request, req t } var sb *types.Sandbox if req.RequirePromoted { - sb, err = s.mgr.ClaimProvisionPromoted(r.Context(), key, req.TTL(), tenant, req.ClaimRef, req.Volumes) + sb, err = s.mgr.ClaimProvisionPromoted(r.Context(), key, req.TTL(), tenant, req.ClaimRef, req.Workspace, req.Volumes) } else { - sb, err = s.mgr.ClaimWarm(r.Context(), key, req.TTL(), tenant, req.ClaimRef, req.Volumes) + sb, err = s.mgr.ClaimWarm(r.Context(), key, req.TTL(), tenant, req.ClaimRef, req.Workspace, req.Volumes) if errors.Is(err, pool.ErrNoWarm) { if s.placer != nil && !req.NoRedirect && writeRedirect(w, s.placer.VolumeCandidates(hash, types.VolumeNames(req.Volumes))) { return } - sb, err = s.mgr.ClaimProvision(r.Context(), key, req.TTL(), tenant, req.ClaimRef, req.Volumes) + sb, err = s.mgr.ClaimProvision(r.Context(), key, req.TTL(), tenant, req.ClaimRef, req.Workspace, req.Volumes) } } writeResult(w, r, "claim", hash, "provisioning failed", err, func() { diff --git a/sandboxd/server/server_test.go b/sandboxd/server/server_test.go index 920fc28..595d1b7 100644 --- a/sandboxd/server/server_test.go +++ b/sandboxd/server/server_test.go @@ -1860,7 +1860,7 @@ type fakeManager struct { draining bool } -func (f *fakeManager) ClaimWarm(ctx context.Context, key types.PoolKey, ttl time.Duration, tenant, claimRef string, volumes []types.Volume) (*types.Sandbox, error) { +func (f *fakeManager) ClaimWarm(ctx context.Context, key types.PoolKey, ttl time.Duration, tenant, claimRef, workspace string, volumes []types.Volume) (*types.Sandbox, error) { f.warmCalls++ f.gotTenant = tenant f.gotClaimRef = claimRef @@ -1871,11 +1871,11 @@ func (f *fakeManager) ClaimWarm(ctx context.Context, key types.PoolKey, ttl time return f.warmClaim(ctx, key, ttl) } -func (f *fakeManager) ClaimProvision(ctx context.Context, key types.PoolKey, ttl time.Duration, tenant, claimRef string, volumes []types.Volume) (*types.Sandbox, error) { +func (f *fakeManager) ClaimProvision(ctx context.Context, key types.PoolKey, ttl time.Duration, tenant, claimRef, workspace string, volumes []types.Volume) (*types.Sandbox, error) { return fakeClaimProvision(f, ctx, key, ttl, tenant, claimRef, volumes, false) } -func (f *fakeManager) ClaimProvisionPromoted(ctx context.Context, key types.PoolKey, ttl time.Duration, tenant, claimRef string, volumes []types.Volume) (*types.Sandbox, error) { +func (f *fakeManager) ClaimProvisionPromoted(ctx context.Context, key types.PoolKey, ttl time.Duration, tenant, claimRef, workspace string, volumes []types.Volume) (*types.Sandbox, error) { return fakeClaimProvision(f, ctx, key, ttl, tenant, claimRef, volumes, true) } diff --git a/sandboxd/types/api.go b/sandboxd/types/api.go index bd3bf11..cecb1d6 100644 --- a/sandboxd/types/api.go +++ b/sandboxd/types/api.go @@ -33,6 +33,11 @@ type ClaimRequest struct { // the k8s "/") recorded on the claim so the read path can // map a listed sandbox back to the name it was claimed under. ClaimRef string `json:"claim_ref,omitempty"` + // Workspace names a shared workspace to bind this sandbox to: the node keeps + // / synced with the guest workspace dir under a + // multi-writer, close-to-open contract. Ignored when the node has no + // workspace root configured. Optional; a newer field older nodes ignore. + Workspace string `json:"workspace,omitempty"` } // Key resolves the requested pool key with the wire defaults filled. diff --git a/sandboxd/types/types.go b/sandboxd/types/types.go index c7183ea..cbb8f33 100644 --- a/sandboxd/types/types.go +++ b/sandboxd/types/types.go @@ -182,6 +182,11 @@ type Sandbox struct { // Volumes records the read-only volumes successfully applied to this claim. Volumes []Volume `json:"volumes,omitempty"` + // Workspace names the shared workspace this sandbox is bound to (the + // filecache syncs / with the guest); empty means none. + // Persisted so a reconciled/adopted claim re-arms its sync after a restart. + Workspace string `json:"workspace,omitempty"` + VsockSocket string `json:"vsock_socket,omitempty"` // TAP is the egress-lane NIC's host tap, captured at provision; empty on // the none lane and on claims adopted from pre-tap journals. diff --git a/sdk/go/client.go b/sdk/go/client.go index 104c091..62bfcb6 100644 --- a/sdk/go/client.go +++ b/sdk/go/client.go @@ -430,6 +430,7 @@ type claimRequest struct { TTLSeconds int `json:"ttl_seconds,omitempty"` NoRedirect bool `json:"no_redirect,omitempty"` RequirePromoted bool `json:"require_promoted,omitempty"` + Workspace string `json:"workspace,omitempty"` } // rejectPinnedAxes fails a snapshot claim (checkpoint, template) that passed diff --git a/sdk/go/options.go b/sdk/go/options.go index faeb49f..4de3c65 100644 --- a/sdk/go/options.go +++ b/sdk/go/options.go @@ -70,3 +70,17 @@ func WithTimeout(d time.Duration) Option { func ttlSeconds(d time.Duration) int { return int((d + time.Second - 1) / time.Second) } + +// WithWorkspace binds the sandbox to a shared workspace: the owning node keeps +// / synced with the guest workspace dir under a +// multi-writer, close-to-open contract. Ignored by nodes with no workspace +// root configured. +func WithWorkspace(name string) Option { + return func(r *claimRequest) { r.Workspace = name } +} + +// WithNoRedirect pins the claim to the addressed node instead of following a +// warm-pool redirect to a peer — useful for driving a specific node. +func WithNoRedirect() Option { + return func(r *claimRequest) { r.NoRedirect = true } +}