From 95d32ae9fb4dd786e9d3d6961c1ee64812c40b1c Mon Sep 17 00:00:00 2001 From: Benjamin Elder Date: Mon, 10 Aug 2026 21:33:51 -0700 Subject: [PATCH] api: move the pause image from ActorTemplate to SandboxConfig The pause image holds the sandbox's namespaces and runs no workload code. It is an implementation detail of the sandbox, not something actor authors pick, so it belongs with the sandbox binaries that already moved off the ActorTemplate onto the cluster-scoped SandboxConfig. It now travels with those binaries end to end: resolved from the pool's SandboxConfig, carried on ateletpb.SandboxAssets rather than WorkloadSpec, and recorded in the per-actor sandbox record so Checkpoint pins it into the snapshot manifest and Restore rebuilds the sandbox from the image the snapshot was taken with (the golden's on a DATA_ON_GOLDEN restore). A record without one is rejected outright rather than pulling an empty image. --- .../manifests/full_workloads.yaml.tmpl | 4 - .../workloads/manifests/workloads.yaml.tmpl | 2 - .../internal/controlapi/create_actor_test.go | 1 - .../internal/controlapi/functional_test.go | 7 +- .../internal/controlapi/sandbox_assets.go | 10 +- .../controlapi/sandbox_assets_test.go | 103 ++++++++++++++++++ .../internal/controlapi/workload_spec.go | 4 +- .../internal/controlapi/workload_spec_test.go | 4 - cmd/atelet/main.go | 24 ++-- cmd/atelet/main_test.go | 15 ++- cmd/atelet/sandbox_assets.go | 21 +++- .../autoscaled-workerpool.yaml.tmpl | 1 - .../claude-code-multiplex.yaml.tmpl | 3 - demos/counter/counter-microvm.yaml.tmpl | 1 - demos/counter/counter.yaml.tmpl | 1 - demos/multi-template/multi-template.yaml.tmpl | 2 - demos/parking/parking.yaml.tmpl | 1 - demos/sandbox/manual-test-multi.yaml | 1 - demos/sandbox/sandbox.yaml.tmpl | 1 - docs/api-guide.md | 17 ++- docs/architecture.md | 2 +- docs/glossary.md | 7 +- internal/e2e/fixtures/probe/probe.yaml.tmpl | 1 - internal/e2e/suites/demo/demo_test.go | 1 - .../networkpolicy/networkpolicy_test.go | 1 - internal/e2e/suites/parking/parking_test.go | 1 - internal/proto/ateletpb/atelet.pb.go | 48 ++++---- internal/proto/ateletpb/atelet.proto | 18 ++- .../generated/ate.dev_actortemplates.yaml | 14 --- .../generated/ate.dev_sandboxconfigs.yaml | 19 ++++ .../ate-install/sandboxconfig-gvisor.yaml | 3 + .../microvm/sandboxconfig-microvm.yaml.tmpl | 3 + pkg/api/v1alpha1/actortemplate_types.go | 11 -- .../v1alpha1/actortemplate_validation_test.go | 20 +--- pkg/api/v1alpha1/sandboxconfig_types.go | 16 +++ .../v1alpha1/sandboxconfig_validation_test.go | 19 ++++ 36 files changed, 274 insertions(+), 133 deletions(-) create mode 100644 cmd/ateapi/internal/controlapi/sandbox_assets_test.go diff --git a/benchmarking/workloads/manifests/full_workloads.yaml.tmpl b/benchmarking/workloads/manifests/full_workloads.yaml.tmpl index e5b3429c1..fd0736569 100644 --- a/benchmarking/workloads/manifests/full_workloads.yaml.tmpl +++ b/benchmarking/workloads/manifests/full_workloads.yaml.tmpl @@ -46,7 +46,6 @@ metadata: name: sleep namespace: benchmark-workloads spec: - pauseImage: "gcr.io/gke-release/pause@sha256:bcbd57ba5653580ec647b16d8163cdd1112df3609129b01f912a8032e48265da" containers: - name: sleep image: busybox@sha256:1487d0af5f52b4ba31c7e465126ee2123fe3f2305d638e7827681e7cf6c83d5e @@ -65,7 +64,6 @@ metadata: name: usermem namespace: benchmark-workloads spec: - pauseImage: "gcr.io/gke-release/pause@sha256:bcbd57ba5653580ec647b16d8163cdd1112df3609129b01f912a8032e48265da" containers: - name: usermem image: containerstack/alpine-stress:latest @@ -84,7 +82,6 @@ metadata: name: kernelmem namespace: benchmark-workloads spec: - pauseImage: "gcr.io/gke-release/pause@sha256:bcbd57ba5653580ec647b16d8163cdd1112df3609129b01f912a8032e48265da" containers: - name: kernelmem image: containerstack/alpine-stress:latest @@ -103,7 +100,6 @@ metadata: name: glutton namespace: benchmark-workloads spec: - pauseImage: "gcr.io/gke-release/pause@sha256:bcbd57ba5653580ec647b16d8163cdd1112df3609129b01f912a8032e48265da" containers: - name: glutton image: ko://github.com/agent-substrate/substrate/cmd/benchmarking/glutton diff --git a/benchmarking/workloads/manifests/workloads.yaml.tmpl b/benchmarking/workloads/manifests/workloads.yaml.tmpl index 939fb8f37..ca9761500 100644 --- a/benchmarking/workloads/manifests/workloads.yaml.tmpl +++ b/benchmarking/workloads/manifests/workloads.yaml.tmpl @@ -42,7 +42,6 @@ metadata: spec: # Must match the WorkerPool's sandboxClass so snapshots stay within a class. sandboxClass: ${SANDBOX_CLASS} - pauseImage: "gcr.io/gke-release/pause@sha256:bcbd57ba5653580ec647b16d8163cdd1112df3609129b01f912a8032e48265da" containers: - name: sleep image: busybox@sha256:1487d0af5f52b4ba31c7e465126ee2123fe3f2305d638e7827681e7cf6c83d5e @@ -62,7 +61,6 @@ metadata: namespace: benchmark-workloads spec: sandboxClass: ${SANDBOX_CLASS} - pauseImage: "gcr.io/gke-release/pause@sha256:bcbd57ba5653580ec647b16d8163cdd1112df3609129b01f912a8032e48265da" containers: - name: glutton image: ko://github.com/agent-substrate/substrate/cmd/benchmarking/glutton diff --git a/cmd/ateapi/internal/controlapi/create_actor_test.go b/cmd/ateapi/internal/controlapi/create_actor_test.go index d138a7de7..e39111eed 100644 --- a/cmd/ateapi/internal/controlapi/create_actor_test.go +++ b/cmd/ateapi/internal/controlapi/create_actor_test.go @@ -213,7 +213,6 @@ func TestCreateActor_RejectsSnapshotWithExternalVolumes(t *testing.T) { template, err := tc.substrateClient.ApiV1alpha1().ActorTemplates(ns).Create(context.Background(), &atev1alpha1.ActorTemplate{ ObjectMeta: metav1.ObjectMeta{Name: "tmpl1", Namespace: ns}, Spec: atev1alpha1.ActorTemplateSpec{ - PauseImage: "pause@sha256:abc", SnapshotsConfig: atev1alpha1.SnapshotsConfig{Location: "gs://snapshots"}, Containers: []atev1alpha1.Container{{ Name: "main", Image: "main@sha256:abc", VolumeMounts: []atev1alpha1.VolumeMount{{Name: "data", MountPath: "/data"}}, diff --git a/cmd/ateapi/internal/controlapi/functional_test.go b/cmd/ateapi/internal/controlapi/functional_test.go index 33b9e1551..a9afe432c 100644 --- a/cmd/ateapi/internal/controlapi/functional_test.go +++ b/cmd/ateapi/internal/controlapi/functional_test.go @@ -525,7 +525,6 @@ func createTemplateWithContainersAndVolumes(t *testing.T, tc *testContext, ns st Namespace: ns, }, Spec: atev1alpha1.ActorTemplateSpec{ - PauseImage: "pause@sha256:abc", SnapshotsConfig: atev1alpha1.SnapshotsConfig{ Location: "gs://fake-fake-fake", }, @@ -574,6 +573,10 @@ func createTemplateWithContainersAndVolumes(t *testing.T, tc *testContext, ns st } } +// testPauseImage is the pause image the default test SandboxConfig carries; +// it is what a resolved WorkloadSpec's sandbox assets should name. +const testPauseImage = "pause@sha256:abc" + // ensureDefaultGvisorSandboxConfig creates the cluster-scoped default gvisor // SandboxConfig (idempotently) and waits for it to appear in the lister. func ensureDefaultGvisorSandboxConfig(t *testing.T, tc *testContext) { @@ -584,6 +587,7 @@ func ensureDefaultGvisorSandboxConfig(t *testing.T, tc *testContext) { Spec: atev1alpha1.SandboxConfigSpec{ SandboxClass: atev1alpha1.SandboxClassGvisor, Default: true, + PauseImage: testPauseImage, Assets: map[string]map[string]atev1alpha1.AssetFile{ "amd64": {"runsc": { URL: "gs://gvisor/releases/nightly/2026-05-19/x86_64/runsc", @@ -643,7 +647,6 @@ func createTemplateWithSelector(t *testing.T, tc *testContext, ns string, name s Namespace: ns, }, Spec: atev1alpha1.ActorTemplateSpec{ - PauseImage: "pause@sha256:abc", SnapshotsConfig: atev1alpha1.SnapshotsConfig{ Location: "gs://fake-fake-fake", }, diff --git a/cmd/ateapi/internal/controlapi/sandbox_assets.go b/cmd/ateapi/internal/controlapi/sandbox_assets.go index 0cf4e5d06..a56458259 100644 --- a/cmd/ateapi/internal/controlapi/sandbox_assets.go +++ b/cmd/ateapi/internal/controlapi/sandbox_assets.go @@ -23,10 +23,11 @@ import ( "k8s.io/apimachinery/pkg/labels" ) -// resolveSandboxAssets determines the sandbox binaries an actor should boot with -// and projects them onto the ateletpb.SandboxAssets atelet fetches. It takes the -// SandboxClass (default gvisor) of a given worker pool, then picks the SandboxConfig -// named by the pool — or, if none is named, the cluster default SandboxConfig for that class. +// resolveSandboxAssets determines the sandbox binaries and pause image an actor +// should boot with and projects them onto the ateletpb.SandboxAssets atelet +// fetches. It takes the SandboxClass (default gvisor) of a given worker pool, +// then picks the SandboxConfig named by the pool — or, if none is named, the +// cluster default SandboxConfig for that class. func resolveSandboxAssets( workerPoolLister listersv1alpha1.WorkerPoolLister, sandboxConfigLister listersv1alpha1.SandboxConfigLister, @@ -89,6 +90,7 @@ func defaultSandboxConfig(lister listersv1alpha1.SandboxConfigLister, class atev func sandboxAssetsProto(class atev1alpha1.SandboxClass, sc *atev1alpha1.SandboxConfig) *ateletpb.SandboxAssets { out := &ateletpb.SandboxAssets{ SandboxClass: string(class), + PauseImage: sc.Spec.PauseImage, Assets: make(map[string]*ateletpb.ArchAssets, len(sc.Spec.Assets)), } for arch, files := range sc.Spec.Assets { diff --git a/cmd/ateapi/internal/controlapi/sandbox_assets_test.go b/cmd/ateapi/internal/controlapi/sandbox_assets_test.go new file mode 100644 index 000000000..a20bd35a1 --- /dev/null +++ b/cmd/ateapi/internal/controlapi/sandbox_assets_test.go @@ -0,0 +1,103 @@ +// Copyright 2026 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package controlapi + +import ( + "testing" + + atev1alpha1 "github.com/agent-substrate/substrate/pkg/api/v1alpha1" + listersv1alpha1 "github.com/agent-substrate/substrate/pkg/client/listers/api/v1alpha1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/client-go/tools/cache" +) + +// listerFor builds listers over the given objects, using the same key +// functions the informers use. +func listersFor(t *testing.T, pools []*atev1alpha1.WorkerPool, configs []*atev1alpha1.SandboxConfig) (listersv1alpha1.WorkerPoolLister, listersv1alpha1.SandboxConfigLister) { + t.Helper() + poolIdx := cache.NewIndexer(cache.MetaNamespaceKeyFunc, cache.Indexers{cache.NamespaceIndex: cache.MetaNamespaceIndexFunc}) + for _, p := range pools { + if err := poolIdx.Add(p); err != nil { + t.Fatalf("adding WorkerPool: %v", err) + } + } + configIdx := cache.NewIndexer(cache.MetaNamespaceKeyFunc, cache.Indexers{}) + for _, c := range configs { + if err := configIdx.Add(c); err != nil { + t.Fatalf("adding SandboxConfig: %v", err) + } + } + return listersv1alpha1.NewWorkerPoolLister(poolIdx), listersv1alpha1.NewSandboxConfigLister(configIdx) +} + +func testAssets() map[string]map[string]atev1alpha1.AssetFile { + return map[string]map[string]atev1alpha1.AssetFile{ + "amd64": {"gvisor": {URL: "gs://bucket/gvisor.tar.bz2", SHA256: "abc"}}, + } +} + +// TestResolveSandboxAssetsCarriesPauseImage pins that the pause image travels +// with the sandbox binaries — it is resolved from the pool's SandboxConfig, not +// from the ActorTemplate — for both the named and the class-default config. +func TestResolveSandboxAssetsCarriesPauseImage(t *testing.T) { + const ( + defaultPause = "registry.k8s.io/pause@sha256:default" + namedPause = "gcr.io/gke-release/pause@sha256:named" + ) + defaultConfig := &atev1alpha1.SandboxConfig{ + ObjectMeta: metav1.ObjectMeta{Name: "gvisor-default"}, + Spec: atev1alpha1.SandboxConfigSpec{ + SandboxClass: atev1alpha1.SandboxClassGvisor, + Default: true, + PauseImage: defaultPause, + Assets: testAssets(), + }, + } + namedConfig := &atev1alpha1.SandboxConfig{ + ObjectMeta: metav1.ObjectMeta{Name: "gvisor-custom"}, + Spec: atev1alpha1.SandboxConfigSpec{ + SandboxClass: atev1alpha1.SandboxClassGvisor, + PauseImage: namedPause, + Assets: testAssets(), + }, + } + + tests := []struct { + name string + configName string + wantPauseImage string + }{ + {name: "class default", wantPauseImage: defaultPause}, + {name: "named config", configName: "gvisor-custom", wantPauseImage: namedPause}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + pool := &atev1alpha1.WorkerPool{ + ObjectMeta: metav1.ObjectMeta{Name: "pool1", Namespace: "worker-ns"}, + Spec: atev1alpha1.WorkerPoolSpec{SandboxConfigName: tt.configName}, + } + poolLister, configLister := listersFor(t, []*atev1alpha1.WorkerPool{pool}, + []*atev1alpha1.SandboxConfig{defaultConfig, namedConfig}) + + got, err := resolveSandboxAssets(poolLister, configLister, "worker-ns", "pool1") + if err != nil { + t.Fatalf("resolveSandboxAssets() error: %v", err) + } + if got.GetPauseImage() != tt.wantPauseImage { + t.Errorf("pause image = %q, want %q", got.GetPauseImage(), tt.wantPauseImage) + } + }) + } +} diff --git a/cmd/ateapi/internal/controlapi/workload_spec.go b/cmd/ateapi/internal/controlapi/workload_spec.go index 6e1b12d0d..15eae5ebe 100644 --- a/cmd/ateapi/internal/controlapi/workload_spec.go +++ b/cmd/ateapi/internal/controlapi/workload_spec.go @@ -37,9 +37,7 @@ const envSecretCacheTTL = 30 * time.Second // container env vars. Use this when downstream consumers (e.g. checkpoint // requests) don't need env entries materialized. func workloadSpecFromActorTemplate(actorTemplate *atev1alpha1.ActorTemplate, actor *ateapipb.Actor) (*ateletpb.WorkloadSpec, error) { - workloadSpec := &ateletpb.WorkloadSpec{ - PauseImage: actorTemplate.Spec.PauseImage, - } + workloadSpec := &ateletpb.WorkloadSpec{} // add volumes for _, vol := range actorTemplate.Spec.Volumes { diff --git a/cmd/ateapi/internal/controlapi/workload_spec_test.go b/cmd/ateapi/internal/controlapi/workload_spec_test.go index 0de964056..ddd85d859 100644 --- a/cmd/ateapi/internal/controlapi/workload_spec_test.go +++ b/cmd/ateapi/internal/controlapi/workload_spec_test.go @@ -43,7 +43,6 @@ func TestWorkloadSpecFromActorTemplate(t *testing.T) { template: &atev1alpha1.ActorTemplate{ ObjectMeta: metav1.ObjectMeta{Name: "tmpl1", Namespace: "agent-ns"}, Spec: atev1alpha1.ActorTemplateSpec{ - PauseImage: "pause", Volumes: []atev1alpha1.Volume{ {Name: "home", VolumeSource: atev1alpha1.VolumeSource{DurableDir: &atev1alpha1.DurableDirVolumeSource{}}}, }, @@ -60,7 +59,6 @@ func TestWorkloadSpecFromActorTemplate(t *testing.T) { }, }, want: &ateletpb.WorkloadSpec{ - PauseImage: "pause", Volumes: []*ateletpb.Volume{ { Name: "home", @@ -227,7 +225,6 @@ func TestWorkloadSpecFromActorTemplateWithEnv(t *testing.T) { template: &atev1alpha1.ActorTemplate{ ObjectMeta: metav1.ObjectMeta{Name: "tmpl1", Namespace: "agent-ns"}, Spec: atev1alpha1.ActorTemplateSpec{ - PauseImage: "pause", Containers: []atev1alpha1.Container{ { Name: "main", @@ -247,7 +244,6 @@ func TestWorkloadSpecFromActorTemplateWithEnv(t *testing.T) { }, }, want: &ateletpb.WorkloadSpec{ - PauseImage: "pause", Containers: []*ateletpb.Container{ { Name: "main", diff --git a/cmd/atelet/main.go b/cmd/atelet/main.go index 99844673b..213a011f3 100644 --- a/cmd/atelet/main.go +++ b/cmd/atelet/main.go @@ -420,7 +420,7 @@ func (s *AteomHerder) Run(ctx context.Context, req *ateletpb.RunRequest) (resp * } if err := s.prepareOCIBundles(ctx, actorUID, actorRef.Name, - req.GetSpec(), req.GetTargetAteomUid(), + req.GetSpec(), sandboxRec.PauseImage, req.GetTargetAteomUid(), ); err != nil { return nil, ateerrors.CrashIfReason(ctx, err, ateerrors.ReasonInvalidContainerConfig) } @@ -825,12 +825,12 @@ func (s *AteomHerder) Restore(ctx context.Context, req *ateletpb.RestoreRequest) op.kind = restoreSnapshotKind(req, sandboxRec) op.sandboxClass = sandboxRec.SandboxClass - // The record whose pinned binaries run the restored workload: the golden's - // for a DATA_ON_GOLDEN restore, the snapshot's own otherwise. The golden's - // set wins because the guest state being resumed is the golden snapshot's - // memory image, and a memory image must be resumed by the exact binary - // versions that produced it; the actor's snapshot contributes only durable - // data (a plain tar), which no binary version reads back. + // The record whose pinned sandbox (binaries + pause image) runs the restored + // workload: the golden's for a DATA_ON_GOLDEN restore, the snapshot's own + // otherwise. The golden's set wins because the guest state being resumed is + // the golden snapshot's memory image, and a memory image must be resumed by + // the exact sandbox that produced it; the actor's snapshot contributes only + // durable data (a plain tar), which no sandbox version reads back. runtimeRec := sandboxRec if goldenRec != nil { runtimeRec = goldenRec @@ -905,7 +905,7 @@ func (s *AteomHerder) Restore(ctx context.Context, req *ateletpb.RestoreRequest) return ateerrors.CrashIfReason(ctx, err, ateerrors.ReasonFailedGetExternalObject, ateerrors.ReasonInvalidObjectURL, ateerrors.ReasonTerminalFileSystemError, ateerrors.ReasonInvalidSandboxAsset) } t := time.Now() - err = s.prepareOCIBundles(gctx, actorUID, actorRef.Name, req.GetSpec(), req.GetTargetAteomUid()) + err = s.prepareOCIBundles(gctx, actorUID, actorRef.Name, req.GetSpec(), runtimeRec.PauseImage, req.GetTargetAteomUid()) dBundles = time.Since(t) if err != nil { prepFailedPhase = ateattr.SnapshotPhaseOCIUnpack @@ -1201,12 +1201,16 @@ func (s *AteomHerder) downloadExternalCheckpoint(ctx context.Context, snapshotUR } // prepareOCIBundles pulls images and assembles OCI bundles for the pause -// container and every application container in spec, in parallel. +// container and every application container in spec, in parallel. pauseImage +// comes from the sandbox record, not the workload spec: it is sandbox +// configuration, and on a restore it must be the image the snapshot was taken +// with. func (s *AteomHerder) prepareOCIBundles( ctx context.Context, actorUID string, actorName string, spec *ateletpb.WorkloadSpec, + pauseImage string, targetAteomUid string, ) error { // Populate the per-actor identity directory that gets bind-mounted into @@ -1252,7 +1256,7 @@ func (s *AteomHerder) prepareOCIBundles( s.imageCache, actorUID, "pause", - spec.GetPauseImage(), + pauseImage, []string{"/pause"}, nil, nil, diff --git a/cmd/atelet/main_test.go b/cmd/atelet/main_test.go index ffe0f9d06..f786a2033 100644 --- a/cmd/atelet/main_test.go +++ b/cmd/atelet/main_test.go @@ -70,7 +70,7 @@ func TestSnapshotManifestActorMetadata(t *testing.T) { // written before the scope field existed must still parse, reporting an empty // scope, and a scope-less record must not serialize a scope key at all. func TestSnapshotManifestScopeAbsent(t *testing.T) { - legacy := []byte(`{"sandboxClass":"gvisor","snapshotFiles":["checkpoint.img"]}`) + legacy := []byte(`{"sandboxClass":"gvisor","pauseImage":"pause@sha256:abc","snapshotFiles":["checkpoint.img"]}`) rec, err := unmarshalSandboxRecord(legacy) if err != nil { t.Fatalf("unmarshalSandboxRecord(legacy manifest): %v", err) @@ -88,6 +88,18 @@ func TestSnapshotManifestScopeAbsent(t *testing.T) { } } +// TestSnapshotManifestRequiresPauseImage pins that a manifest without a pause +// image is rejected outright rather than yielding an empty image that would +// fail later, deep in the image pull. +func TestSnapshotManifestRequiresPauseImage(t *testing.T) { + noPause := []byte(`{"sandboxClass":"gvisor","snapshotFiles":["checkpoint.img"]}`) + if _, err := unmarshalSandboxRecord(noPause); err == nil { + t.Fatal("unmarshalSandboxRecord accepted a manifest with no pauseImage") + } else if !strings.Contains(err.Error(), "pauseImage") { + t.Errorf("error = %v, want it to name pauseImage", err) + } +} + func TestWriteFileAtomic(t *testing.T) { dir := t.TempDir() target := filepath.Join(dir, "actor-id") @@ -636,7 +648,6 @@ func TestRPCBoundariesReject(t *testing.T) { func TestBuildAteomWorkloadSpecForwardsReadyz(t *testing.T) { in := &ateletpb.WorkloadSpec{ - PauseImage: "pause", Containers: []*ateletpb.Container{ { Name: "with-probe", diff --git a/cmd/atelet/sandbox_assets.go b/cmd/atelet/sandbox_assets.go index ba62d6b0c..808e3af21 100644 --- a/cmd/atelet/sandbox_assets.go +++ b/cmd/atelet/sandbox_assets.go @@ -67,15 +67,20 @@ type assetEntry struct { } // sandboxAssetsRecord is the sandbox runtime an actor is running, projected onto -// the local node's architecture: the sandbox class plus the asset set keyed by -// asset name (gVisor uses a single "gvisor" release-tarball asset; records -// written before the tarball release mechanism use a bare "runsc" asset). +// the local node's architecture: the sandbox class and pause image plus the +// asset set keyed by asset name (gVisor uses a single "gvisor" release-tarball +// asset; records written before the tarball release mechanism use a bare +// "runsc" asset). // It is both the per-actor on-node record (written at Run/Restore, read at // Checkpoint) and the snapshot manifest (written at Checkpoint, read at // Restore). type sandboxAssetsRecord struct { SandboxClass string `json:"sandboxClass"` Assets map[string]assetEntry `json:"assets"` + // PauseImage is the root sandbox container's image. It is recorded here + // rather than taken from the request at Restore so a snapshot is rebuilt + // with the same sandbox it was captured from. + PauseImage string `json:"pauseImage"` // Actor identity makes a flat snapshot self-identifying if control-plane // persistence is unavailable. Atespace string `json:"atespace,omitempty"` @@ -107,8 +112,12 @@ func recordFromRequest(sa *ateletpb.SandboxAssets) (*sandboxAssetsRecord, error) if archAssets == nil || len(archAssets.GetFiles()) == 0 { return nil, fmt.Errorf("sandbox_assets has no assets for architecture %q", arch) } + if sa.GetPauseImage() == "" { + return nil, fmt.Errorf("sandbox_assets has no pause_image") + } rec := &sandboxAssetsRecord{ SandboxClass: sa.GetSandboxClass(), + PauseImage: sa.GetPauseImage(), Assets: make(map[string]assetEntry, len(archAssets.GetFiles())), } for name, f := range archAssets.GetFiles() { @@ -416,6 +425,12 @@ func unmarshalSandboxRecord(data []byte) (*sandboxAssetsRecord, error) { if err := json.Unmarshal(data, rec); err != nil { return nil, fmt.Errorf("%w: while parsing sandbox record/manifest: %w", ateerrors.ReasonInvalidSandboxAsset, err) } + // Fail loudly rather than let an empty image reach the image pull: a record + // without one predates the pause image moving into the sandbox config, and + // its snapshot cannot be rebuilt with a known-matching sandbox. + if rec.PauseImage == "" { + return nil, fmt.Errorf("%w: sandbox record/manifest has no pauseImage", ateerrors.ReasonInvalidSandboxAsset) + } return rec, nil } diff --git a/demos/autoscaled-workerpool/autoscaled-workerpool.yaml.tmpl b/demos/autoscaled-workerpool/autoscaled-workerpool.yaml.tmpl index 4880383c8..74e54b8f7 100644 --- a/demos/autoscaled-workerpool/autoscaled-workerpool.yaml.tmpl +++ b/demos/autoscaled-workerpool/autoscaled-workerpool.yaml.tmpl @@ -45,7 +45,6 @@ metadata: name: counter namespace: ate-demo-autoscaled-workerpool spec: - pauseImage: "registry.k8s.io/pause:3.10.2@sha256:f548e0e8e3dc1896ca956272154dde3314e8cc4fde0a57577ee9fa1c63f5baf4" containers: - name: counter image: ko://github.com/agent-substrate/substrate/demos/counter diff --git a/demos/claude-code-multiplex/claude-code-multiplex.yaml.tmpl b/demos/claude-code-multiplex/claude-code-multiplex.yaml.tmpl index eb06f284a..e48268992 100644 --- a/demos/claude-code-multiplex/claude-code-multiplex.yaml.tmpl +++ b/demos/claude-code-multiplex/claude-code-multiplex.yaml.tmpl @@ -65,7 +65,6 @@ metadata: name: agent-luna namespace: claude-multiplex-demo spec: - pauseImage: "registry.k8s.io/pause:3.10.2@sha256:f548e0e8e3dc1896ca956272154dde3314e8cc4fde0a57577ee9fa1c63f5baf4" containers: - name: claude image: ${WORKLOAD_IMAGE} @@ -95,7 +94,6 @@ metadata: name: agent-mars namespace: claude-multiplex-demo spec: - pauseImage: "registry.k8s.io/pause:3.10.2@sha256:f548e0e8e3dc1896ca956272154dde3314e8cc4fde0a57577ee9fa1c63f5baf4" containers: - name: claude image: ${WORKLOAD_IMAGE} @@ -125,7 +123,6 @@ metadata: name: agent-orion namespace: claude-multiplex-demo spec: - pauseImage: "registry.k8s.io/pause:3.10.2@sha256:f548e0e8e3dc1896ca956272154dde3314e8cc4fde0a57577ee9fa1c63f5baf4" containers: - name: claude image: ${WORKLOAD_IMAGE} diff --git a/demos/counter/counter-microvm.yaml.tmpl b/demos/counter/counter-microvm.yaml.tmpl index e661c5865..960bd9018 100644 --- a/demos/counter/counter-microvm.yaml.tmpl +++ b/demos/counter/counter-microvm.yaml.tmpl @@ -51,7 +51,6 @@ spec: # sandbox classes, so only pools of the same class are eligible to run this # template's actors. sandboxClass: microvm - pauseImage: "registry.k8s.io/pause:3.10.2@sha256:f548e0e8e3dc1896ca956272154dde3314e8cc4fde0a57577ee9fa1c63f5baf4" containers: - name: counter image: ko://github.com/agent-substrate/substrate/demos/counter diff --git a/demos/counter/counter.yaml.tmpl b/demos/counter/counter.yaml.tmpl index fa8801842..892a21f99 100644 --- a/demos/counter/counter.yaml.tmpl +++ b/demos/counter/counter.yaml.tmpl @@ -38,7 +38,6 @@ metadata: name: counter namespace: ate-demo-counter spec: - pauseImage: "registry.k8s.io/pause:3.10.2@sha256:f548e0e8e3dc1896ca956272154dde3314e8cc4fde0a57577ee9fa1c63f5baf4" containers: - name: counter image: ko://github.com/agent-substrate/substrate/demos/counter diff --git a/demos/multi-template/multi-template.yaml.tmpl b/demos/multi-template/multi-template.yaml.tmpl index b4d0fc4e4..c19ab8724 100644 --- a/demos/multi-template/multi-template.yaml.tmpl +++ b/demos/multi-template/multi-template.yaml.tmpl @@ -54,7 +54,6 @@ metadata: name: counter namespace: ate-demo-multi-template-counter spec: - pauseImage: "registry.k8s.io/pause:3.10.2@sha256:f548e0e8e3dc1896ca956272154dde3314e8cc4fde0a57577ee9fa1c63f5baf4" containers: - name: counter image: ko://github.com/agent-substrate/substrate/demos/counter @@ -70,7 +69,6 @@ metadata: name: fspersist namespace: ate-demo-multi-template-fspersist spec: - pauseImage: "registry.k8s.io/pause:3.10.2@sha256:f548e0e8e3dc1896ca956272154dde3314e8cc4fde0a57577ee9fa1c63f5baf4" containers: - name: fspersist image: ko://github.com/agent-substrate/substrate/demos/multi-template/fspersist diff --git a/demos/parking/parking.yaml.tmpl b/demos/parking/parking.yaml.tmpl index a4406a8ec..3ede10394 100644 --- a/demos/parking/parking.yaml.tmpl +++ b/demos/parking/parking.yaml.tmpl @@ -50,7 +50,6 @@ metadata: name: parking namespace: ate-demo-parking spec: - pauseImage: "registry.k8s.io/pause:3.10.2@sha256:f548e0e8e3dc1896ca956272154dde3314e8cc4fde0a57577ee9fa1c63f5baf4" containers: - name: counter image: ko://github.com/agent-substrate/substrate/demos/counter diff --git a/demos/sandbox/manual-test-multi.yaml b/demos/sandbox/manual-test-multi.yaml index 909fc8c83..4c456afd5 100644 --- a/demos/sandbox/manual-test-multi.yaml +++ b/demos/sandbox/manual-test-multi.yaml @@ -37,7 +37,6 @@ spec: workerSelector: matchLabels: workload: sandbox - pauseImage: "registry.k8s.io/pause:3.10.2@sha256:f548e0e8e3dc1896ca956272154dde3314e8cc4fde0a57577ee9fa1c63f5baf4" containers: - name: sandbox image: ko://github.com/agent-substrate/substrate/demos/sandbox diff --git a/demos/sandbox/sandbox.yaml.tmpl b/demos/sandbox/sandbox.yaml.tmpl index 0265eea62..b9a868f04 100644 --- a/demos/sandbox/sandbox.yaml.tmpl +++ b/demos/sandbox/sandbox.yaml.tmpl @@ -37,7 +37,6 @@ spec: workerSelector: matchLabels: workload: sandbox - pauseImage: "registry.k8s.io/pause:3.10.2@sha256:f548e0e8e3dc1896ca956272154dde3314e8cc4fde0a57577ee9fa1c63f5baf4" containers: - name: sandbox image: ko://github.com/agent-substrate/substrate/demos/sandbox diff --git a/docs/api-guide.md b/docs/api-guide.md index 2ef434710..02bb563e6 100644 --- a/docs/api-guide.md +++ b/docs/api-guide.md @@ -13,7 +13,7 @@ The `WorkerPool` defines the pool of physical "warm" compute capacity. It manage | `replicas` | `int32` | **Required.** Number of physical standby pods to maintain in the cluster. | | `ateomImage` | `string` | **Required.** The container image for the `ateom` herder process (e.g. `ko://github.com/agent-substrate/substrate/cmd/ateom-gvisor`). | | `sandboxClass` | `string` | Optional. The sandbox runtime family for the pool: `gvisor` (default) or `microvm`. Drives the worker pod shape (e.g. KVM device mounts, node placement) and which `SandboxConfig`s are eligible. | -| `sandboxConfigName` | `string` | Optional. Name of a cluster-scoped [`SandboxConfig`](#3-sandboxconfig-sandbox-binaries) providing the sandbox binaries. If empty, the cluster default `SandboxConfig` for the pool's `sandboxClass` is used. | +| `sandboxConfigName` | `string` | Optional. Name of a cluster-scoped [`SandboxConfig`](#3-sandboxconfig-the-sandbox-itself) providing the sandbox binaries and pause image. If empty, the cluster default `SandboxConfig` for the pool's `sandboxClass` is used. | | `template` | `WorkerPoolPodTemplate` | **Optional.** Pod scheduling and resource settings for worker pods. | #### `WorkerPoolPodTemplate` (`spec.template`) @@ -144,10 +144,9 @@ The `ActorTemplate` defines the code, environment, and state-management policies | `sandboxClass` | `string` | Optional. The sandbox runtime family this template's actors require: `gvisor` (default) or `microvm`. Only `WorkerPool`s whose `sandboxClass` matches are eligible. | | `workerSelector` | `*LabelSelector` | Optional. Gates which `WorkerPool`s actors from this template may use, by matching against each pool's labels. If unset, all pools are eligible (subject to the actor's own `worker_selector`). | | `snapshotsConfig` | `SnapshotsConfig` | **Required.** The base object-storage location snapshots are written under, plus the pause/commit/resume scopes. See [Snapshot Storage Layout](#snapshot-storage-layout). | -| `pauseImage` | `string` | **Required.** The image used for the sandbox root (e.g. `gcr.io/gke-release/pause`). | | `volumes` | `[]Volume` | Optional. Volumes the containers may mount, each either a `durableDir` or an `externalVolumeTemplate`. Every declared volume must be mounted by at least one container. A `microvm` template may declare several `durableDir` volumes; a `gvisor` template is limited to one, and `externalVolumeTemplate` is `gvisor`-only. | -The sandbox binaries (e.g. the gVisor `runsc` binary) are **no longer configured on the `ActorTemplate`**. They are resolved from the referenced `WorkerPool`'s [`SandboxConfig`](#3-sandboxconfig-sandbox-binaries) — by name (`workerPool.spec.sandboxConfigName`) or, by default, the cluster default `SandboxConfig` for the pool's `sandboxClass`. +The sandbox itself — the binaries (e.g. the gVisor `runsc` binary) and the `pauseImage` holding the sandbox's namespaces — is **not configured on the `ActorTemplate`**. It is resolved from the referenced `WorkerPool`'s [`SandboxConfig`](#3-sandboxconfig-the-sandbox-itself) — by name (`workerPool.spec.sandboxConfigName`) or, by default, the cluster default `SandboxConfig` for the pool's `sandboxClass`. Because a snapshot is not restorable across sandbox runtimes, `sandboxClass` is a **hard scheduling gate**: an actor is only ever placed on a `WorkerPool` of the matching class. It is AND'd with `workerSelector` (and the actor's `worker_selector`), which can only narrow the eligible pools further. It defaults to `gvisor` and, like the rest of the spec, is immutable, so each template's class is fixed at creation. @@ -207,10 +206,8 @@ metadata: name: secret-agent namespace: ate-demo spec: - # No sandbox/runsc config here — the binaries come from the WorkerPool's - # SandboxConfig (see section 3). - # GKE clusters: use gcr.io/gke-release/pause. Other clusters: registry.k8s.io/pause:3.9 - pauseImage: "gcr.io/gke-release/pause@sha256:bcbd57ba5653580ec647b16d8163cdd1112df3609129b01f912a8032e48265da" + # No sandbox config here — the binaries and pause image come from the + # WorkerPool's SandboxConfig (see section 3). containers: - name: agent image: gcr.io/my-project/my-agent:latest @@ -261,9 +258,9 @@ Two consequences worth planning for: --- -## 3. SandboxConfig: Sandbox Binaries +## 3. SandboxConfig: The Sandbox Itself -`SandboxConfig` is a **cluster-scoped** resource that decouples the sandbox binaries (the gVisor `runsc` binary, or a micro-VM kernel/firmware/config) from the `ActorTemplate`. A `WorkerPool` resolves its binaries from a `SandboxConfig` — either the one named by `spec.sandboxConfigName`, or the cluster default for the pool's `sandboxClass`. +`SandboxConfig` is a **cluster-scoped** resource that decouples the sandbox — its binaries (the gVisor `runsc` binary, or a micro-VM kernel/firmware/config) and the `pauseImage` that holds the sandbox's namespaces — from the `ActorTemplate`. A `WorkerPool` resolves its sandbox from a `SandboxConfig` — either the one named by `spec.sandboxConfigName`, or the cluster default for the pool's `sandboxClass`. This means a single, cluster-managed config pins the sandbox runtime version for many templates: snapshots stay restorable because the version is recorded in each snapshot's manifest, and operators upgrade the runtime in one place. @@ -272,6 +269,7 @@ This means a single, cluster-managed config pins the sandbox runtime version for | Field | Type | Description | | :--- | :--- | :--- | | `sandboxClass` | `string` | **Required.** Runtime family this config applies to: `gvisor` (default) or `microvm`. A `WorkerPool` only uses `SandboxConfig`s whose `sandboxClass` matches its own. | +| `pauseImage` | `string` | **Required.** The image for the sandbox's root container (e.g. `registry.k8s.io/pause`, or `gcr.io/gke-release/pause` on GKE). Must be pinned by digest (`...@sha256:...`) — it is recorded in each snapshot's manifest so a restore rebuilds the sandbox from the same image. | | `default` | `bool` | Optional. Marks this as the cluster default for its `sandboxClass`. A `WorkerPool` with no `sandboxConfigName` resolves to the default for its class. At most one default per class. | | `assets` | `map[arch]map[name]AssetFile` | Optional. Content-addressed files atelet fetches, keyed by architecture (`amd64`, `arm64`) then asset name. gVisor expects a `gvisor` asset (the release's `gvisor.tar.bz2`), which atelet auto-extracts. A micro-VM backend expects several. Each `AssetFile` is a `{ url, sha256 }` pair. | @@ -287,6 +285,7 @@ metadata: spec: sandboxClass: gvisor default: true + pauseImage: "registry.k8s.io/pause:3.10.2@sha256:f548e0e8e3dc1896ca956272154dde3314e8cc4fde0a57577ee9fa1c63f5baf4" assets: amd64: gvisor: diff --git a/docs/architecture.md b/docs/architecture.md index c4b1a2fee..38b14fe9c 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -324,7 +324,7 @@ The node-level subsystem manages the physical execution of sandboxes and the mov ### Sandbox Classes -A `WorkerPool` selects a **sandbox class** (`spec.sandboxClass`), and each class has a matching `ateom` herder image. The sandbox binaries themselves are not baked into the worker image — they are fetched at runtime from a cluster-scoped [`SandboxConfig`](api-guide.md#3-sandboxconfig-sandbox-binaries) and pinned into each snapshot's manifest so restores stay reproducible across runtime upgrades. +A `WorkerPool` selects a **sandbox class** (`spec.sandboxClass`), and each class has a matching `ateom` herder image. The sandbox binaries themselves are not baked into the worker image — they, and the pause image holding the sandbox's namespaces, come at runtime from a cluster-scoped [`SandboxConfig`](api-guide.md#3-sandboxconfig-the-sandbox-itself) and are pinned into each snapshot's manifest so restores stay reproducible across runtime upgrades. * **gVisor** (`ateom-gvisor`, the default): Runs the workload under `runsc` for kernel-level sandboxing. Suspend and resume leverage gVisor's native checkpoint/restore of the sandboxed process tree. diff --git a/docs/glossary.md b/docs/glossary.md index 73534ac0e..e6e651e3e 100644 --- a/docs/glossary.md +++ b/docs/glossary.md @@ -19,9 +19,10 @@ For how the pieces fit together, see the [Architecture](architecture.md) and - **SandboxConfig**: a cluster-scoped resource holding the sandbox binaries for one runtime family (the gVisor `runsc` binary, or a micro-VM - kernel/firmware/config). A `WorkerPool` resolves its binaries from the config - it names, or from the cluster default for its class, so one config pins the - runtime version for many templates. + kernel/firmware/config), plus the pause image for the sandbox's root + container. A `WorkerPool` resolves its sandbox from the config it names, or + from the cluster default for its class, so one config pins the runtime version + for many templates. ## Records (dynamic state, in the control-plane store) diff --git a/internal/e2e/fixtures/probe/probe.yaml.tmpl b/internal/e2e/fixtures/probe/probe.yaml.tmpl index f79e0cdfe..85ea73fd7 100644 --- a/internal/e2e/fixtures/probe/probe.yaml.tmpl +++ b/internal/e2e/fixtures/probe/probe.yaml.tmpl @@ -38,7 +38,6 @@ metadata: name: probe namespace: ate-e2e-probe spec: - pauseImage: "registry.k8s.io/pause:3.10.2@sha256:f548e0e8e3dc1896ca956272154dde3314e8cc4fde0a57577ee9fa1c63f5baf4" containers: - name: probe image: ko://github.com/agent-substrate/substrate/internal/e2e/fixtures/probe diff --git a/internal/e2e/suites/demo/demo_test.go b/internal/e2e/suites/demo/demo_test.go index da94d7aad..7b3f8a690 100644 --- a/internal/e2e/suites/demo/demo_test.go +++ b/internal/e2e/suites/demo/demo_test.go @@ -860,7 +860,6 @@ func createActorTemplateInternal(ctx context.Context, t *testing.T, clients *e2e // ActorTemplate↔WorkerPool match succeeds. The micro-VM source sets // "microvm"; the gVisor source leaves it "" — copying keeps both correct. SandboxClass: existingAt.Spec.SandboxClass, - PauseImage: existingAt.Spec.PauseImage, Containers: existingAt.Spec.Containers, SnapshotsConfig: v1alpha1.SnapshotsConfig{ Location: "gs://" + env["BUCKET_NAME"] + "/ate-demo-" + name, diff --git a/internal/e2e/suites/networkpolicy/networkpolicy_test.go b/internal/e2e/suites/networkpolicy/networkpolicy_test.go index 8b441a0ec..033f08485 100644 --- a/internal/e2e/suites/networkpolicy/networkpolicy_test.go +++ b/internal/e2e/suites/networkpolicy/networkpolicy_test.go @@ -326,7 +326,6 @@ func setupDemoCounterTemplate(ctx context.Context, t *testing.T, clients *e2e.Cl MatchLabels: map[string]string{"netpol-test": ns}, }, SandboxClass: existingAt.Spec.SandboxClass, - PauseImage: existingAt.Spec.PauseImage, Containers: existingAt.Spec.Containers, SnapshotsConfig: existingAt.Spec.SnapshotsConfig, Volumes: existingAt.Spec.Volumes, diff --git a/internal/e2e/suites/parking/parking_test.go b/internal/e2e/suites/parking/parking_test.go index 5cfe14fbc..233b95fec 100644 --- a/internal/e2e/suites/parking/parking_test.go +++ b/internal/e2e/suites/parking/parking_test.go @@ -210,7 +210,6 @@ func createParkingFixture(ctx context.Context, t *testing.T, clients *e2e.Client MatchLabels: map[string]string{"demo": nsObj.Name}, }, SandboxClass: existingAt.Spec.SandboxClass, - PauseImage: existingAt.Spec.PauseImage, Containers: existingAt.Spec.Containers, SnapshotsConfig: v1alpha1.SnapshotsConfig{ Location: "gs://" + env["BUCKET_NAME"] + "/e2e-parking-" + nsObj.Name, diff --git a/internal/proto/ateletpb/atelet.pb.go b/internal/proto/ateletpb/atelet.pb.go index cc5a3f693..834d61728 100644 --- a/internal/proto/ateletpb/atelet.pb.go +++ b/internal/proto/ateletpb/atelet.pb.go @@ -563,14 +563,19 @@ func (x *ArchAssets) GetFiles() map[string]*AssetFile { } // SandboxAssets is the generic, backend-agnostic description of the sandbox -// binaries for an actor: a sandbox class plus assets keyed first by -// architecture (GOARCH) and then by asset name. atelet's backend code -// interprets the asset names (gVisor expects "gvisor", the release tarball; -// legacy "runsc", a bare binary, is still accepted). +// an actor runs in: a sandbox class, the pause image holding its namespaces, +// and binaries keyed first by architecture (GOARCH) and then by asset name. +// atelet's backend code interprets the asset names (gVisor expects "gvisor", +// the release tarball; legacy "runsc", a bare binary, is still accepted). type SandboxAssets struct { - state protoimpl.MessageState `protogen:"open.v1"` - SandboxClass string `protobuf:"bytes,1,opt,name=sandbox_class,json=sandboxClass,proto3" json:"sandbox_class,omitempty"` // e.g. "gvisor" - Assets map[string]*ArchAssets `protobuf:"bytes,2,rep,name=assets,proto3" json:"assets,omitempty" protobuf_key:"bytes,1,opt,name=key" protobuf_val:"bytes,2,opt,name=value"` // arch -> {name -> file} + state protoimpl.MessageState `protogen:"open.v1"` + SandboxClass string `protobuf:"bytes,1,opt,name=sandbox_class,json=sandboxClass,proto3" json:"sandbox_class,omitempty"` // e.g. "gvisor" + Assets map[string]*ArchAssets `protobuf:"bytes,2,rep,name=assets,proto3" json:"assets,omitempty" protobuf_key:"bytes,1,opt,name=key" protobuf_val:"bytes,2,opt,name=value"` // arch -> {name -> file} + // pause_image is the image for the sandbox's root container. Like the + // binaries above it is sandbox configuration, not workload configuration, + // and atelet pins it into the snapshot manifest so a restore rebuilds the + // sandbox from the same image. + PauseImage string `protobuf:"bytes,3,opt,name=pause_image,json=pauseImage,proto3" json:"pause_image,omitempty"` unknownFields protoimpl.UnknownFields sizeCache protoimpl.SizeCache } @@ -619,11 +624,17 @@ func (x *SandboxAssets) GetAssets() map[string]*ArchAssets { return nil } +func (x *SandboxAssets) GetPauseImage() string { + if x != nil { + return x.PauseImage + } + return "" +} + // WorkloadSpec parallels Pod, but with far fewer configurable fields. type WorkloadSpec struct { state protoimpl.MessageState `protogen:"open.v1"` Containers []*Container `protobuf:"bytes,1,rep,name=containers,proto3" json:"containers,omitempty"` - PauseImage string `protobuf:"bytes,2,opt,name=pause_image,json=pauseImage,proto3" json:"pause_image,omitempty"` Volumes []*Volume `protobuf:"bytes,3,rep,name=volumes,proto3" json:"volumes,omitempty"` unknownFields protoimpl.UnknownFields sizeCache protoimpl.SizeCache @@ -666,13 +677,6 @@ func (x *WorkloadSpec) GetContainers() []*Container { return nil } -func (x *WorkloadSpec) GetPauseImage() string { - if x != nil { - return x.PauseImage - } - return "" -} - func (x *WorkloadSpec) GetVolumes() []*Volume { if x != nil { return x.Volumes @@ -1760,20 +1764,20 @@ const file_atelet_proto_rawDesc = "" + "\n" + "FilesEntry\x12\x10\n" + "\x03key\x18\x01 \x01(\tR\x03key\x12'\n" + - "\x05value\x18\x02 \x01(\v2\x11.atelet.AssetFileR\x05value:\x028\x01\"\xbe\x01\n" + + "\x05value\x18\x02 \x01(\v2\x11.atelet.AssetFileR\x05value:\x028\x01\"\xdf\x01\n" + "\rSandboxAssets\x12#\n" + "\rsandbox_class\x18\x01 \x01(\tR\fsandboxClass\x129\n" + - "\x06assets\x18\x02 \x03(\v2!.atelet.SandboxAssets.AssetsEntryR\x06assets\x1aM\n" + + "\x06assets\x18\x02 \x03(\v2!.atelet.SandboxAssets.AssetsEntryR\x06assets\x12\x1f\n" + + "\vpause_image\x18\x03 \x01(\tR\n" + + "pauseImage\x1aM\n" + "\vAssetsEntry\x12\x10\n" + "\x03key\x18\x01 \x01(\tR\x03key\x12(\n" + - "\x05value\x18\x02 \x01(\v2\x12.atelet.ArchAssetsR\x05value:\x028\x01\"\x8c\x01\n" + + "\x05value\x18\x02 \x01(\v2\x12.atelet.ArchAssetsR\x05value:\x028\x01\"~\n" + "\fWorkloadSpec\x121\n" + "\n" + "containers\x18\x01 \x03(\v2\x11.atelet.ContainerR\n" + - "containers\x12\x1f\n" + - "\vpause_image\x18\x02 \x01(\tR\n" + - "pauseImage\x12(\n" + - "\avolumes\x18\x03 \x03(\v2\x0e.atelet.VolumeR\avolumes\"\x12\n" + + "containers\x12(\n" + + "\avolumes\x18\x03 \x03(\v2\x0e.atelet.VolumeR\avolumesJ\x04\b\x02\x10\x03R\vpause_image\"\x12\n" + "\x10DurableDirVolume\"\xfd\x01\n" + "\x14ExternalVolumeSource\x12*\n" + "\x11storage_volume_id\x18\x01 \x01(\tR\x0fstorageVolumeId\x12\x1f\n" + diff --git a/internal/proto/ateletpb/atelet.proto b/internal/proto/ateletpb/atelet.proto index 771db8e10..857bbf4cd 100644 --- a/internal/proto/ateletpb/atelet.proto +++ b/internal/proto/ateletpb/atelet.proto @@ -96,20 +96,28 @@ message ArchAssets { } // SandboxAssets is the generic, backend-agnostic description of the sandbox -// binaries for an actor: a sandbox class plus assets keyed first by -// architecture (GOARCH) and then by asset name. atelet's backend code -// interprets the asset names (gVisor expects "gvisor", the release tarball; -// legacy "runsc", a bare binary, is still accepted). +// an actor runs in: a sandbox class, the pause image holding its namespaces, +// and binaries keyed first by architecture (GOARCH) and then by asset name. +// atelet's backend code interprets the asset names (gVisor expects "gvisor", +// the release tarball; legacy "runsc", a bare binary, is still accepted). message SandboxAssets { string sandbox_class = 1; // e.g. "gvisor" map assets = 2; // arch -> {name -> file} + + // pause_image is the image for the sandbox's root container. Like the + // binaries above it is sandbox configuration, not workload configuration, + // and atelet pins it into the snapshot manifest so a restore rebuilds the + // sandbox from the same image. + string pause_image = 3; } // WorkloadSpec parallels Pod, but with far fewer configurable fields. message WorkloadSpec { repeated Container containers = 1; - string pause_image = 2; repeated Volume volumes = 3; + + reserved 2; + reserved "pause_image"; // moved to SandboxAssets } enum VolumeType { diff --git a/manifests/ate-install/generated/ate.dev_actortemplates.yaml b/manifests/ate-install/generated/ate.dev_actortemplates.yaml index 1dff97416..f5da2c721 100644 --- a/manifests/ate-install/generated/ate.dev_actortemplates.yaml +++ b/manifests/ate-install/generated/ate.dev_actortemplates.yaml @@ -264,19 +264,6 @@ spec: type: object maxItems: 10 type: array - pauseImage: - description: |- - PauseImage is the container to use as the root sandbox container. - - Typically, set it to [1] for on-gcp, and [2] for off-gcp - - - [1] gcr.io/gke-release/pause@sha256:bcbd57ba5653580ec647b16d8163cdd1112df3609129b01f912a8032e48265da - - [2] registry.k8s.io/pause:3.10.2@sha256:f548e0e8e3dc1896ca956272154dde3314e8cc4fde0a57577ee9fa1c63f5baf4 - type: string - x-kubernetes-validations: - - message: All images must be pinned (changing the image invalidates - snapshots) - rule: self.contains('@') sandboxClass: default: gvisor description: |- @@ -456,7 +443,6 @@ spec: type: object x-kubernetes-map-type: atomic required: - - pauseImage - snapshotsConfig type: object x-kubernetes-validations: diff --git a/manifests/ate-install/generated/ate.dev_sandboxconfigs.yaml b/manifests/ate-install/generated/ate.dev_sandboxconfigs.yaml index 71ecf752a..a282c5f9b 100644 --- a/manifests/ate-install/generated/ate.dev_sandboxconfigs.yaml +++ b/manifests/ate-install/generated/ate.dev_sandboxconfigs.yaml @@ -111,6 +111,24 @@ spec: the default config for its SandboxClass. At most one default is expected per SandboxClass. type: boolean + pauseImage: + description: |- + PauseImage is the container image used as the root sandbox container. + It holds the sandbox's namespaces and runs no workload code, so it is an + implementation detail of the sandbox rather than something actor authors + choose. It is captured in the snapshot manifest alongside the sandbox + binaries, so a restore always re-creates the sandbox from the same image + the snapshot was taken with. + + Typically, set it to [1] for on-gcp, and [2] for off-gcp + + - [1] gcr.io/gke-release/pause@sha256:bcbd57ba5653580ec647b16d8163cdd1112df3609129b01f912a8032e48265da + - [2] registry.k8s.io/pause:3.10.2@sha256:f548e0e8e3dc1896ca956272154dde3314e8cc4fde0a57577ee9fa1c63f5baf4 + type: string + x-kubernetes-validations: + - message: All images must be pinned (changing the image invalidates + snapshots) + rule: self.contains('@') sandboxClass: default: gvisor description: |- @@ -121,6 +139,7 @@ spec: - microvm type: string required: + - pauseImage - sandboxClass type: object required: diff --git a/manifests/ate-install/sandboxconfig-gvisor.yaml b/manifests/ate-install/sandboxconfig-gvisor.yaml index 1a3c9d6f2..13884cde4 100644 --- a/manifests/ate-install/sandboxconfig-gvisor.yaml +++ b/manifests/ate-install/sandboxconfig-gvisor.yaml @@ -26,6 +26,9 @@ metadata: spec: sandboxClass: gvisor default: true + # The root sandbox container's image. On GCP, prefer the in-project mirror + # gcr.io/gke-release/pause@sha256:bcbd57ba5653580ec647b16d8163cdd1112df3609129b01f912a8032e48265da. + pauseImage: "registry.k8s.io/pause:3.10.2@sha256:f548e0e8e3dc1896ca956272154dde3314e8cc4fde0a57577ee9fa1c63f5baf4" assets: amd64: gvisor: diff --git a/manifests/microvm/sandboxconfig-microvm.yaml.tmpl b/manifests/microvm/sandboxconfig-microvm.yaml.tmpl index c41094011..6285a7535 100644 --- a/manifests/microvm/sandboxconfig-microvm.yaml.tmpl +++ b/manifests/microvm/sandboxconfig-microvm.yaml.tmpl @@ -45,6 +45,9 @@ metadata: name: microvm spec: sandboxClass: microvm + # The root sandbox container's image. On GCP, prefer the in-project mirror + # gcr.io/gke-release/pause@sha256:bcbd57ba5653580ec647b16d8163cdd1112df3609129b01f912a8032e48265da. + pauseImage: "registry.k8s.io/pause:3.10.2@sha256:f548e0e8e3dc1896ca956272154dde3314e8cc4fde0a57577ee9fa1c63f5baf4" assets: arm64: cloud-hypervisor: diff --git a/pkg/api/v1alpha1/actortemplate_types.go b/pkg/api/v1alpha1/actortemplate_types.go index 1a2480a33..358e24cc9 100644 --- a/pkg/api/v1alpha1/actortemplate_types.go +++ b/pkg/api/v1alpha1/actortemplate_types.go @@ -358,17 +358,6 @@ type SnapshotsConfig struct { // +kubebuilder:validation:XValidation:rule="!has(self.sandboxClass) || self.sandboxClass != 'microvm' || !has(self.volumes) || !self.volumes.exists(v, has(v.externalVolumeTemplate))",message="ExternalVolumes are not supported when sandboxClass is 'microvm'" // +kubebuilder:validation:XValidation:rule="(has(self.sandboxClass) && self.sandboxClass == 'microvm') || !has(self.snapshotsConfig.onResume) || (has(self.snapshotsConfig.onResume.fromData) ? self.snapshotsConfig.onResume.fromData : 'ColdBoot') != 'Golden'",message="onResume.fromData: Golden is not supported when sandboxClass is 'gvisor'" type ActorTemplateSpec struct { - // PauseImage is the container to use as the root sandbox container. - // - // Typically, set it to [1] for on-gcp, and [2] for off-gcp - // - // - [1] gcr.io/gke-release/pause@sha256:bcbd57ba5653580ec647b16d8163cdd1112df3609129b01f912a8032e48265da - // - [2] registry.k8s.io/pause:3.10.2@sha256:f548e0e8e3dc1896ca956272154dde3314e8cc4fde0a57577ee9fa1c63f5baf4 - // - // +required - // +kubebuilder:validation:XValidation:rule="self.contains('@')",message="All images must be pinned (changing the image invalidates snapshots)" - PauseImage string `json:"pauseImage,omitempty"` - // Containers is the workload definition. // // +optional diff --git a/pkg/api/v1alpha1/actortemplate_validation_test.go b/pkg/api/v1alpha1/actortemplate_validation_test.go index 4b0a5d54c..89edc6c27 100644 --- a/pkg/api/v1alpha1/actortemplate_validation_test.go +++ b/pkg/api/v1alpha1/actortemplate_validation_test.go @@ -70,7 +70,6 @@ func TestActorTemplateValidation(t *testing.T) { Namespace: "default", }, Spec: ActorTemplateSpec{ - PauseImage: "gcr.io/gke-release/pause@sha256:bcbd57ba5653580ec647b16d8163cdd1112df3609129b01f912a8032e48265da", Containers: []Container{ { Name: "main", @@ -98,20 +97,6 @@ func TestActorTemplateValidation(t *testing.T) { name: "base template", mutate: func(at *ActorTemplate) {}, wantErr: false, - }, { - name: "missing PauseImage", - mutate: func(at *ActorTemplate) { - at.Spec.PauseImage = "" - }, - wantErr: true, - errMsg: "Required value", - }, { - name: "unpinned PauseImage", - mutate: func(at *ActorTemplate) { - at.Spec.PauseImage = "pause" - }, - wantErr: true, - errMsg: "All images must be pinned", }, { name: "missing SnapshotsConfig.Location", mutate: func(at *ActorTemplate) { @@ -1269,7 +1254,6 @@ func TestActorTemplateSpecImmutability(t *testing.T) { baseTemplate := &ActorTemplate{ Spec: ActorTemplateSpec{ - PauseImage: "pause@hash", Containers: []Container{ { Name: "main", @@ -1290,9 +1274,9 @@ func TestActorTemplateSpecImmutability(t *testing.T) { mutate func(*ActorTemplate) }{ { - name: "update-pause-image", + name: "update-container-image", mutate: func(at *ActorTemplate) { - at.Spec.PauseImage = "pause@new" + at.Spec.Containers[0].Image = "busybox@new" }, }, { diff --git a/pkg/api/v1alpha1/sandboxconfig_types.go b/pkg/api/v1alpha1/sandboxconfig_types.go index e7a9c98cb..80153d53b 100644 --- a/pkg/api/v1alpha1/sandboxconfig_types.go +++ b/pkg/api/v1alpha1/sandboxconfig_types.go @@ -67,6 +67,22 @@ type SandboxConfigSpec struct { // +optional Default bool `json:"default,omitempty"` + // PauseImage is the container image used as the root sandbox container. + // It holds the sandbox's namespaces and runs no workload code, so it is an + // implementation detail of the sandbox rather than something actor authors + // choose. It is captured in the snapshot manifest alongside the sandbox + // binaries, so a restore always re-creates the sandbox from the same image + // the snapshot was taken with. + // + // Typically, set it to [1] for on-gcp, and [2] for off-gcp + // + // - [1] gcr.io/gke-release/pause@sha256:bcbd57ba5653580ec647b16d8163cdd1112df3609129b01f912a8032e48265da + // - [2] registry.k8s.io/pause:3.10.2@sha256:f548e0e8e3dc1896ca956272154dde3314e8cc4fde0a57577ee9fa1c63f5baf4 + // + // +required + // +kubebuilder:validation:XValidation:rule="self.contains('@')",message="All images must be pinned (changing the image invalidates snapshots)" + PauseImage string `json:"pauseImage"` + // Assets is the set of files atelet fetches for this runtime, keyed first by // architecture (GOARCH, e.g. "amd64", "arm64") and then by asset name. The // asset names are interpreted by the sandbox backend: gVisor expects a diff --git a/pkg/api/v1alpha1/sandboxconfig_validation_test.go b/pkg/api/v1alpha1/sandboxconfig_validation_test.go index 6dc1a2dbb..374edf966 100644 --- a/pkg/api/v1alpha1/sandboxconfig_validation_test.go +++ b/pkg/api/v1alpha1/sandboxconfig_validation_test.go @@ -35,16 +35,25 @@ const validSHA256 = "a397be1abc2420d26bce6c70e6e2ff96c73aaaab929756c56f5e2089ea8 // the install — loaded here so the test guards the policy we actually ship. const vapManifestPath = "../../../manifests/ate-install/sandboxconfig-validation.yaml" +const validPauseImage = "registry.k8s.io/pause:3.10.2@sha256:f548e0e8e3dc1896ca956272154dde3314e8cc4fde0a57577ee9fa1c63f5baf4" + func sandboxConfig(name string, class SandboxClass, assets map[string]map[string]AssetFile) *SandboxConfig { return &SandboxConfig{ ObjectMeta: metav1.ObjectMeta{Name: name}, Spec: SandboxConfigSpec{ SandboxClass: class, + PauseImage: validPauseImage, Assets: assets, }, } } +// withPauseImage overrides the pause image on an otherwise-valid config. +func withPauseImage(sc *SandboxConfig, image string) *SandboxConfig { + sc.Spec.PauseImage = image + return sc +} + func runscAsset() AssetFile { return AssetFile{URL: "gs://bucket/runsc", SHA256: validSHA256} } func gvisorAsset() AssetFile { @@ -184,6 +193,16 @@ func TestSandboxConfigValidation(t *testing.T) { sc: sandboxConfig("bad-sha", SandboxClassGvisor, map[string]map[string]AssetFile{"amd64": {"runsc": {URL: "gs://bucket/runsc", SHA256: "deadbeef"}}}), wantErr: true, errMsg: "sha256", + }, { + name: "missing pauseImage", + sc: withPauseImage(sandboxConfig("bad-no-pause", SandboxClassGvisor, map[string]map[string]AssetFile{"amd64": {"gvisor": gvisorAsset()}}), ""), + wantErr: true, + errMsg: "pauseImage", + }, { + name: "unpinned pauseImage", + sc: withPauseImage(sandboxConfig("bad-unpinned-pause", SandboxClassGvisor, map[string]map[string]AssetFile{"amd64": {"gvisor": gvisorAsset()}}), "registry.k8s.io/pause:3.10.2"), + wantErr: true, + errMsg: "All images must be pinned", }} for _, tt := range tests {