Expose promoted template content digests

docs/deploy.md

@@ -82,8 +82,8 @@ sandboxd reads one JSON file (`-config`, default
 | `preview_listen` | (off) | address for a preview HTTP server that serves guest ports under signed URLs; needs `preview_secret` |
 | `preview_secret` | — | cluster-shared HMAC secret signing preview tokens (all nodes share one) |
 | `preview_advertise` | = `preview_listen` | the base URL a browser/proxy reaches this node's preview server at |
-| `checkpoint_dir` | `<data_dir>/checkpoints` | where checkpoints and promoted templates live. Point it at a shared FUSE mount (JuiceFS over object storage, NFS) and every node sharing the mount can branch every checkpoint — the path's filesystem is the operator's choice. One contract on any shared root (mount or bucket): a template key has a single writer — promotes go to the sandbox's owner node, and operators must not race promotes of one name from different nodes (checkpoint ids are node-generated and never collide). A checkpoint deleted on one node while another is mid-branch from it fails that branch visibly |
-| `checkpoint_store` | dir | checkpoint AND promoted-template backend (both live in one store root, id-namespaced ck_/tp_): `{"kind": "s3", "s3": {"bucket": "…", "prefix": "ck/", "endpoint": "…", "region": "…", "force_path_style": true}}` stores checkpoints in object storage (any node claims any checkpoint, no shared mount needed). Credentials come from the standard AWS chain (env/IAM role), never this file. A crash between upload and the meta.json commit marker leaves orphan objects invisible to listings — add an S3 lifecycle rule to reclaim them. Absent = the dir backend at `checkpoint_dir` |
+| `checkpoint_dir` | `<data_dir>/checkpoints` | where checkpoints and promoted templates live. Point it at a shared FUSE mount (JuiceFS over object storage, NFS) and every node sharing the mount can branch every checkpoint — the filesystem must provide working cross-node POSIX advisory `flock`; local-only or ignored locks are unsupported. A fixed, bounded hash-striped lock set keeps a fetched directory generation read-locked through clone, so replace/delete waits for that reader. One contract on any shared root (mount or bucket): a template key has a single writer — promotes go to the sandbox's owner node, and operators must not race promotes of one name from different nodes (checkpoint ids are node-generated and never collide) |
+| `checkpoint_store` | dir | checkpoint AND promoted-template backend (both live in one store root, id-namespaced ck_/tp_): `{"kind": "s3", "s3": {"bucket": "…", "prefix": "ck/", "endpoint": "…", "region": "…", "force_path_style": true}}` stores checkpoints in object storage (any node claims any checkpoint, no shared mount needed). Credentials come from the standard AWS chain (env/IAM role), never this file. Re-publish retains prior export generations until Delete so an in-flight fetch that selected old metadata can finish; budget storage for those generations. An explicit S3 Delete can still make a concurrent fetch that has not finished materializing fail visibly. A crash between upload and the meta.json commit marker leaves orphan objects invisible to listings — add an S3 lifecycle rule to reclaim them. Absent = the dir backend at `checkpoint_dir` |
 | `checkpoint_ttl_hours` | 0 (keep forever) | ages out checkpoints older than this; the sweep runs hourly and at startup. Explicit deletes never wait for it. Must be nonzero and match fleet-wide when `checkpoint_peer_heal` is on — it is the expiry eligibility point for a healed replica a delete broadcast missed, after which its next successful hourly sweep removes it; persistent sweep failure extends retention until one succeeds, so it is not a hard ceiling |
 | `checkpoint_peer_heal` | false | on a cluster, lets a node pull a checkpoint it lacks from a peer — found via a live probe, not gossip — rather than failing the branch; see [placement lifecycle](cluster.md#checkpoints-on-a-cluster). Three requirements, all enforced at config load: a nonempty `api_token` (the blob transfer between peers authenticates with it; without one the raw record stream would be open), `mesh.cluster_key` set (the pull presents the fleet `api_token` to an address learned from the peer probe, so the gossip layer carrying that address must itself be authenticated), and `checkpoint_ttl_hours` nonzero (a replica a delete broadcast missed becomes eligible for expiry after it, and its next successful hourly sweep removes it — so it is the finite eligibility point, not an exact ceiling). A shared checkpoint store (`checkpoint_store` kind `s3`) ignores this setting — every node already resolves every checkpoint directly, so there is nothing to heal |
 | `warm_max` (pool entry) | 0 (static) | turns on the demand-adaptive watermark for that pool: the warm target rises from `warm` toward `warm_max` while claims arrive faster than the measured provision lead covers, and decays back over ~a minute of silence |

docs/sandboxd-api.md

@@ -39,9 +39,14 @@ Success:
 
 ```json
 {"id": "sb_…", "token": "…", "deadline": "2026-07-06T00:05:00Z",
- "owner_addr": "10.0.0.5:7777"}
+ "owner_addr": "10.0.0.5:7777", "template_digest": "sha256:…"}
 ```
 
+A claim cloned from a promoted template carries `template_digest`, the exact
+export generation fetched for that clone. It is absent for configured pools,
+cold image boots, forks, checkpoints, and templates published by an older
+sandboxd until they are re-promoted.
+
 A claim branched from a checkpoint (fork children included) additionally
 carries `"from_checkpoint": "ck_…"` — the lineage edge for reconstructing
 the checkpoint tree.
@@ -136,15 +141,24 @@ Publishes the sandbox's current state as a node-local template under
 from it, provision-on-demand — no warm pool unless the node config adds one.
 Re-promoting to the same name replaces the template. A hibernated sandbox is
 promoted from its memory image without waking. 200 returns the template's
-full key. On the default local-disk backend a template is node-local, so a
-cluster client claims from and deletes on this node (name-based calls route
-via gossip); a shared checkpoint store makes every node resolve it. Under
-exactly this key:
+full key and immutable content identity. On the default local-disk backend a
+template is node-local, so a cluster client claims from and deletes on this
+node (name-based calls route via gossip); a shared checkpoint store makes
+every node resolve it. Under exactly this key:
 
 ```json
-{"key": {"template": "myproj:v1", "net": "none", "size": "small"}}
+{"key": {"template": "myproj:v1", "net": "none", "size": "small"},
+ "content_digest": "sha256:…"}
 ```
 
+`content_digest` is SHA-256 over a versioned canonical stream of the published
+export's regular files: slash-relative path, byte length, and bytes, ordered
+lexically. Directory entries, modes, mtimes, and the template's ownership/
+creation metadata do not affect it. The digest is computed once while
+promoting, stored in `meta.json`, and therefore has identical semantics on the
+directory and S3 backends. Re-promoting unchanged export bytes keeps the
+digest; changing any exported path or bytes changes it.
+
 400 invalid name, 401 bad api token, 409 when the name collides with a
 configured pool, the template is owned by another tenant, or the sandbox is
 on the egress lane (see [egress](egress.md)), 404 unknown id or wrong

docs/sdk-python.md

@@ -98,9 +98,10 @@ sb = client.new("ghcr.io/cocoonstack/sandbox/rt:24.04",
 `new` returns when the sandbox's silkd answers: a warm hit is milliseconds,
 a cold key can take the full boot. The handle exposes `sb.id`, `sb.token`,
 `sb.owner`, `sb.deadline`, and `sb.from_checkpoint` (the lineage edge when
-branched). `Sandbox` is a context manager; `sb.close()` releases it
-(releasing one already gone is not an error — double-release and reap races
-stay silent).
+branched). `sb.template_digest` is the exact content identity when the claim
+cloned a promoted template; it is empty for other sources. `Sandbox` is a
+context manager; `sb.close()` releases it (releasing one already gone is not
+an error — double-release and reap races stay silent).
 
 ## Hibernating
 
@@ -138,6 +139,7 @@ All-or-nothing: on error no child survived. Count is capped at the node's
 ```python
 tpl = sb.promote("myproj:v1")     # publish current state
 child = tpl.new()                 # clones the promoted state
+assert tpl.content_digest and tpl.content_digest == child.template_digest
 tpl.delete()                      # caller owns the lifecycle
 ```
 
@@ -149,6 +151,11 @@ bound there, so its `new`/`delete` always reach it. The name-based calls
 cluster-wide via template gossip and lag a promote/delete by about a gossip
 tick — prefer the handle right after promoting (see
 [Templates on a cluster](cluster.md#templates-on-a-cluster)).
+`tpl.content_digest` identifies the published export bytes. A caller pinning
+the mutable name can compare a claim's `template_digest` with its expected
+value and close/refuse a mismatch.
+Templates published by an older sandboxd have empty digests until they are
+re-promoted after the node is upgraded.
 
 ## Checkpoints — branching and time travel
 

docs/sdk.md

@@ -143,11 +143,12 @@ defer sb.Close()
 
 `New` returns when the sandbox's silkd answers: a warm hit is milliseconds,
 a cold key can take the full boot. `Sandbox.ID`, `Sandbox.Deadline`, and
-`Sandbox.FromCheckpoint` (the lineage edge when branched) are exported,
-`Owner()` names the owning node, and `Token()` returns the per-sandbox
-bearer to persist with `ID` for a later `Lookup`; `Close()` releases the
-sandbox (releasing one already gone is not an error, and `Close` is bounded
-internally so it stays defer-friendly).
+`Sandbox.FromCheckpoint` (the lineage edge when branched) are exported.
+`Sandbox.TemplateDigest` is the exact content identity when the claim cloned a
+promoted template; it is empty for other sources. `Owner()` names the owning
+node, and `Token()` returns the per-sandbox bearer to persist with `ID` for a
+later `Lookup`; `Close()` releases the sandbox (releasing one already gone is
+not an error, and `Close` is bounded internally so it stays defer-friendly).
 
 ## Hibernating
 
@@ -190,14 +191,20 @@ client needs `WithAPIToken` — a sandbox handle alone cannot amplify.
 ```go
 tpl, err := sb.Promote(ctx, "myproj:v1")  // publish current state
 child, err := tpl.New(ctx)                // clones the promoted state
+fmt.Println(tpl.ContentDigest != "" && tpl.ContentDigest == child.TemplateDigest) // true
 err = tpl.Delete(ctx)                     // caller owns the lifecycle
 ```
 
 `Promote` publishes the sandbox's state as a template on its owning node,
 keyed by (name, the sandbox's network lane, its size). Claims clone on
 demand (~a golden-clone's latency); there is no warm pool for promoted
 templates unless the node's config adds one. Re-promoting to the same name
-replaces the template.
+replaces the template. `Template.ContentDigest` identifies the published
+export bytes; a claim from that exact generation carries the same value in
+`Sandbox.TemplateDigest`. A caller pinning a mutable template name can compare
+the claim's value with its expected digest and close/refuse a mismatch.
+Templates published by an older sandboxd have empty digests until they are
+re-promoted after the node is upgraded.
 
 **On the default local-disk backend templates live on one node**, and on a
 cluster the parent claim may have been redirected — the returned `Template`

e2e/e2e_test.go

@@ -121,12 +121,18 @@ func TestPromoteEndToEnd(t *testing.T) {
 	if err != nil {
 		t.Fatalf("Promote: %v", err)
 	}
+	if tpl.ContentDigest == "" {
+		t.Fatal("Promote returned an empty content digest")
+	}
 	// Both claim surfaces must work: the owner-bound handle and the
 	// name-based Client call on the (single) node that holds the template.
 	child, err := tpl.New(t.Context())
 	if err != nil {
 		t.Fatalf("claim via template handle: %v", err)
 	}
+	if child.TemplateDigest != tpl.ContentDigest {
+		t.Errorf("claim template digest %q, want %q", child.TemplateDigest, tpl.ContentDigest)
+	}
 	if out, err := child.Exec(t.Context(), "echo", "tpl"); err != nil || out != "tpl\n" {
 		t.Errorf("exec on promoted claim: %q, %v", out, err)
 	}
@@ -135,6 +141,9 @@ func TestPromoteEndToEnd(t *testing.T) {
 	if nameErr != nil {
 		t.Fatalf("claim promoted template by name: %v", nameErr)
 	}
+	if byName.TemplateDigest != tpl.ContentDigest {
+		t.Errorf("name claim template digest %q, want %q", byName.TemplateDigest, tpl.ContentDigest)
+	}
 	_ = byName.Close()
 
 	if err := tpl.Delete(t.Context()); err != nil {

sandboxd/pool/archive.go

@@ -4,6 +4,9 @@ import (
 	"context"
 	"errors"
 	"fmt"
+	"io/fs"
+	"os"
+	"path/filepath"
 	"time"
 
 	"github.com/projecteru2/core/log"
@@ -12,6 +15,8 @@ import (
 	"github.com/cocoonstack/sandbox/sandboxd/types"
 )
 
+const archiveDeleteDir = "archive-deletes"
+
 // archive*For resolve a key's thresholds (pool's, else node default); callers hold m.mu.
 func (m *Manager) archiveAfterFor(key types.PoolKey) time.Duration {
 	if p, ok := m.activePool(key); ok {
@@ -92,6 +97,9 @@ func (m *Manager) archive(ctx context.Context, sb *types.Sandbox) error {
 	m.pendingCks[ckID] = struct{}{}
 	m.mu.Unlock()
 	defer m.untrack(m.pendingCks, ckID)
+	if err := m.markArchiveCk(ckID); err != nil {
+		return fmt.Errorf("track archive checkpoint: %w", err)
+	}
 	// A hibernated source is copied from its wake image, so no VM starts.
 	ck, srcSnap, err := m.publishCheckpoint(ctx, sb, ckID, "archive", sb.Tenant, true)
 	if err != nil {
@@ -174,8 +182,8 @@ func (m *Manager) wakeArchived(ctx context.Context, sb *types.Sandbox) (string,
 	if err != nil {
 		return "", fmt.Errorf("wake %s: fetch archive: %w", sb.ID, err)
 	}
-	defer release()
 	built, err := m.provision(ctx, sb.Key, dir)
+	release()
 	if err != nil {
 		return "", fmt.Errorf("wake %s: %w", sb.ID, err)
 	}
@@ -192,6 +200,9 @@ func (m *Manager) wakeArchived(ctx context.Context, sb *types.Sandbox) (string,
 		log.WithFunc("pool.wakeArchived").Warnf(ctx, "delete consumed archive ck %s: %v", ck, delErr)
 	} else {
 		consumed = true
+		if clearErr := m.clearArchiveCk(ck); clearErr != nil {
+			log.WithFunc("pool.wakeArchived").Warnf(ctx, "clear archive ck %s: %v", ck, clearErr)
+		}
 	}
 	// Only the none lane reaches here (the egress guard above fails closed), so
 	// this rebinds the none-lane proxy; there is no NIC to re-lock.
@@ -236,7 +247,105 @@ func (m *Manager) commitWake(ctx context.Context, sb *types.Sandbox, vmName, soc
 
 // deleteOrphanArchiveCk drops the published ck when archive() aborts pre-commit.
 func (m *Manager) deleteOrphanArchiveCk(ctx context.Context, ckID string) {
-	if err := m.ckpts.Delete(ctx, ckID); err != nil {
+	if err := m.deleteArchiveCk(ctx, ckID); err != nil {
 		log.WithFunc("pool.deleteOrphanArchiveCk").Warnf(ctx, "delete orphaned archive ck %s: %v", ckID, err)
 	}
 }
+
+func (m *Manager) markArchiveCk(ckID string) error {
+	if !store.CheckpointIDRe.MatchString(ckID) {
+		return fmt.Errorf("invalid checkpoint id %q", ckID)
+	}
+	dir := filepath.Join(m.dataDir, archiveDeleteDir)
+	if err := os.MkdirAll(dir, 0o750); err != nil {
+		return err
+	}
+	return os.WriteFile(filepath.Join(dir, ckID), nil, 0o600)
+}
+
+func (m *Manager) clearArchiveCk(ckID string) error {
+	err := os.Remove(filepath.Join(m.dataDir, archiveDeleteDir, ckID))
+	if errors.Is(err, fs.ErrNotExist) {
+		return nil
+	}
+	return err
+}
+
+func (m *Manager) deleteArchiveCk(ctx context.Context, ckID string) error {
+	if err := m.markArchiveCk(ckID); err != nil {
+		return fmt.Errorf("track: %w", err)
+	}
+	if err := m.deleteCkLocked(ctx, ckID); err != nil {
+		return err
+	}
+	if err := m.clearArchiveCk(ckID); err != nil {
+		return fmt.Errorf("clear: %w", err)
+	}
+	return nil
+}
+
+func (m *Manager) retryArchiveDeletes(ctx context.Context) {
+	if !m.archiveDeleteSweep.CompareAndSwap(false, true) {
+		return
+	}
+	defer m.archiveDeleteSweep.Store(false)
+	entries, err := os.ReadDir(filepath.Join(m.dataDir, archiveDeleteDir))
+	if errors.Is(err, fs.ErrNotExist) {
+		return
+	}
+	if err != nil {
+		log.WithFunc("pool.retryArchiveDeletes").Warnf(ctx, "list: %v", err)
+		return
+	}
+	if len(entries) == 0 {
+		return
+	}
+	pinned := m.pinnedArchiveCks()
+	ids := make([]string, 0, len(entries))
+	for _, entry := range entries {
+		if !entry.Type().IsRegular() || !store.CheckpointIDRe.MatchString(entry.Name()) {
+			continue
+		}
+		if _, ok := pinned[entry.Name()]; !ok {
+			ids = append(ids, entry.Name())
+		}
+	}
+	m.runBounded(ctx, len(ids), func(ctx context.Context, i int) {
+		m.retryArchiveDelete(ctx, ids[i])
+	}).Wait()
+}
+
+func (m *Manager) retryArchiveDelete(ctx context.Context, ckID string) {
+	l := m.recLock(ckID)
+	l.Lock()
+	if m.archiveCkPinned(ckID) {
+		l.Unlock()
+		m.recDone(ckID)
+		return
+	}
+	err := m.ckpts.Delete(ctx, ckID)
+	l.Unlock()
+	if err != nil {
+		m.recDone(ckID)
+		log.WithFunc("pool.retryArchiveDeletes").Warnf(ctx, "delete %s: %v", ckID, err)
+		return
+	}
+	m.recDoneEvict(ckID)
+	if err := m.clearArchiveCk(ckID); err != nil {
+		log.WithFunc("pool.retryArchiveDeletes").Warnf(ctx, "clear %s: %v", ckID, err)
+	}
+}
+
+func (m *Manager) archiveCkPinned(ckID string) bool {
+	m.mu.Lock()
+	defer m.mu.Unlock()
+	if _, ok := m.pendingCks[ckID]; ok {
+		return true
+	}
+	for _, sb := range m.claimed {
+		if sb.ArchiveCk == ckID {
+			return true
+		}
+	}
+	return false
+}