diff --git a/benchmarking/workloads/manifests/full_workloads.yaml.tmpl b/benchmarking/workloads/manifests/full_workloads.yaml.tmpl index e5b3429c1..fd0736569 100644 --- a/benchmarking/workloads/manifests/full_workloads.yaml.tmpl +++ b/benchmarking/workloads/manifests/full_workloads.yaml.tmpl @@ -46,7 +46,6 @@ metadata: name: sleep namespace: benchmark-workloads spec: - pauseImage: "gcr.io/gke-release/pause@sha256:bcbd57ba5653580ec647b16d8163cdd1112df3609129b01f912a8032e48265da" containers: - name: sleep image: busybox@sha256:1487d0af5f52b4ba31c7e465126ee2123fe3f2305d638e7827681e7cf6c83d5e @@ -65,7 +64,6 @@ metadata: name: usermem namespace: benchmark-workloads spec: - pauseImage: "gcr.io/gke-release/pause@sha256:bcbd57ba5653580ec647b16d8163cdd1112df3609129b01f912a8032e48265da" containers: - name: usermem image: containerstack/alpine-stress:latest @@ -84,7 +82,6 @@ metadata: name: kernelmem namespace: benchmark-workloads spec: - pauseImage: "gcr.io/gke-release/pause@sha256:bcbd57ba5653580ec647b16d8163cdd1112df3609129b01f912a8032e48265da" containers: - name: kernelmem image: containerstack/alpine-stress:latest @@ -103,7 +100,6 @@ metadata: name: glutton namespace: benchmark-workloads spec: - pauseImage: "gcr.io/gke-release/pause@sha256:bcbd57ba5653580ec647b16d8163cdd1112df3609129b01f912a8032e48265da" containers: - name: glutton image: ko://github.com/agent-substrate/substrate/cmd/benchmarking/glutton diff --git a/benchmarking/workloads/manifests/workloads.yaml.tmpl b/benchmarking/workloads/manifests/workloads.yaml.tmpl index 939fb8f37..ca9761500 100644 --- a/benchmarking/workloads/manifests/workloads.yaml.tmpl +++ b/benchmarking/workloads/manifests/workloads.yaml.tmpl @@ -42,7 +42,6 @@ metadata: spec: # Must match the WorkerPool's sandboxClass so snapshots stay within a class. sandboxClass: ${SANDBOX_CLASS} - pauseImage: "gcr.io/gke-release/pause@sha256:bcbd57ba5653580ec647b16d8163cdd1112df3609129b01f912a8032e48265da" containers: - name: sleep image: busybox@sha256:1487d0af5f52b4ba31c7e465126ee2123fe3f2305d638e7827681e7cf6c83d5e @@ -62,7 +61,6 @@ metadata: namespace: benchmark-workloads spec: sandboxClass: ${SANDBOX_CLASS} - pauseImage: "gcr.io/gke-release/pause@sha256:bcbd57ba5653580ec647b16d8163cdd1112df3609129b01f912a8032e48265da" containers: - name: glutton image: ko://github.com/agent-substrate/substrate/cmd/benchmarking/glutton diff --git a/cmd/ateapi/internal/controlapi/create_actor_test.go b/cmd/ateapi/internal/controlapi/create_actor_test.go index d138a7de7..e39111eed 100644 --- a/cmd/ateapi/internal/controlapi/create_actor_test.go +++ b/cmd/ateapi/internal/controlapi/create_actor_test.go @@ -213,7 +213,6 @@ func TestCreateActor_RejectsSnapshotWithExternalVolumes(t *testing.T) { template, err := tc.substrateClient.ApiV1alpha1().ActorTemplates(ns).Create(context.Background(), &atev1alpha1.ActorTemplate{ ObjectMeta: metav1.ObjectMeta{Name: "tmpl1", Namespace: ns}, Spec: atev1alpha1.ActorTemplateSpec{ - PauseImage: "pause@sha256:abc", SnapshotsConfig: atev1alpha1.SnapshotsConfig{Location: "gs://snapshots"}, Containers: []atev1alpha1.Container{{ Name: "main", Image: "main@sha256:abc", VolumeMounts: []atev1alpha1.VolumeMount{{Name: "data", MountPath: "/data"}}, diff --git a/cmd/ateapi/internal/controlapi/functional_test.go b/cmd/ateapi/internal/controlapi/functional_test.go index 33b9e1551..a9afe432c 100644 --- a/cmd/ateapi/internal/controlapi/functional_test.go +++ b/cmd/ateapi/internal/controlapi/functional_test.go @@ -525,7 +525,6 @@ func createTemplateWithContainersAndVolumes(t *testing.T, tc *testContext, ns st Namespace: ns, }, Spec: atev1alpha1.ActorTemplateSpec{ - PauseImage: "pause@sha256:abc", SnapshotsConfig: atev1alpha1.SnapshotsConfig{ Location: "gs://fake-fake-fake", }, @@ -574,6 +573,10 @@ func createTemplateWithContainersAndVolumes(t *testing.T, tc *testContext, ns st } } +// testPauseImage is the pause image the default test SandboxConfig carries; +// it is what a resolved WorkloadSpec's sandbox assets should name. +const testPauseImage = "pause@sha256:abc" + // ensureDefaultGvisorSandboxConfig creates the cluster-scoped default gvisor // SandboxConfig (idempotently) and waits for it to appear in the lister. func ensureDefaultGvisorSandboxConfig(t *testing.T, tc *testContext) { @@ -584,6 +587,7 @@ func ensureDefaultGvisorSandboxConfig(t *testing.T, tc *testContext) { Spec: atev1alpha1.SandboxConfigSpec{ SandboxClass: atev1alpha1.SandboxClassGvisor, Default: true, + PauseImage: testPauseImage, Assets: map[string]map[string]atev1alpha1.AssetFile{ "amd64": {"runsc": { URL: "gs://gvisor/releases/nightly/2026-05-19/x86_64/runsc", @@ -643,7 +647,6 @@ func createTemplateWithSelector(t *testing.T, tc *testContext, ns string, name s Namespace: ns, }, Spec: atev1alpha1.ActorTemplateSpec{ - PauseImage: "pause@sha256:abc", SnapshotsConfig: atev1alpha1.SnapshotsConfig{ Location: "gs://fake-fake-fake", }, diff --git a/cmd/ateapi/internal/controlapi/sandbox_assets.go b/cmd/ateapi/internal/controlapi/sandbox_assets.go index 0cf4e5d06..a56458259 100644 --- a/cmd/ateapi/internal/controlapi/sandbox_assets.go +++ b/cmd/ateapi/internal/controlapi/sandbox_assets.go @@ -23,10 +23,11 @@ import ( "k8s.io/apimachinery/pkg/labels" ) -// resolveSandboxAssets determines the sandbox binaries an actor should boot with -// and projects them onto the ateletpb.SandboxAssets atelet fetches. It takes the -// SandboxClass (default gvisor) of a given worker pool, then picks the SandboxConfig -// named by the pool — or, if none is named, the cluster default SandboxConfig for that class. +// resolveSandboxAssets determines the sandbox binaries and pause image an actor +// should boot with and projects them onto the ateletpb.SandboxAssets atelet +// fetches. It takes the SandboxClass (default gvisor) of a given worker pool, +// then picks the SandboxConfig named by the pool — or, if none is named, the +// cluster default SandboxConfig for that class. func resolveSandboxAssets( workerPoolLister listersv1alpha1.WorkerPoolLister, sandboxConfigLister listersv1alpha1.SandboxConfigLister, @@ -89,6 +90,7 @@ func defaultSandboxConfig(lister listersv1alpha1.SandboxConfigLister, class atev func sandboxAssetsProto(class atev1alpha1.SandboxClass, sc *atev1alpha1.SandboxConfig) *ateletpb.SandboxAssets { out := &ateletpb.SandboxAssets{ SandboxClass: string(class), + PauseImage: sc.Spec.PauseImage, Assets: make(map[string]*ateletpb.ArchAssets, len(sc.Spec.Assets)), } for arch, files := range sc.Spec.Assets { diff --git a/cmd/ateapi/internal/controlapi/sandbox_assets_test.go b/cmd/ateapi/internal/controlapi/sandbox_assets_test.go new file mode 100644 index 000000000..a20bd35a1 --- /dev/null +++ b/cmd/ateapi/internal/controlapi/sandbox_assets_test.go @@ -0,0 +1,103 @@ +// Copyright 2026 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package controlapi + +import ( + "testing" + + atev1alpha1 "github.com/agent-substrate/substrate/pkg/api/v1alpha1" + listersv1alpha1 "github.com/agent-substrate/substrate/pkg/client/listers/api/v1alpha1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/client-go/tools/cache" +) + +// listerFor builds listers over the given objects, using the same key +// functions the informers use. +func listersFor(t *testing.T, pools []*atev1alpha1.WorkerPool, configs []*atev1alpha1.SandboxConfig) (listersv1alpha1.WorkerPoolLister, listersv1alpha1.SandboxConfigLister) { + t.Helper() + poolIdx := cache.NewIndexer(cache.MetaNamespaceKeyFunc, cache.Indexers{cache.NamespaceIndex: cache.MetaNamespaceIndexFunc}) + for _, p := range pools { + if err := poolIdx.Add(p); err != nil { + t.Fatalf("adding WorkerPool: %v", err) + } + } + configIdx := cache.NewIndexer(cache.MetaNamespaceKeyFunc, cache.Indexers{}) + for _, c := range configs { + if err := configIdx.Add(c); err != nil { + t.Fatalf("adding SandboxConfig: %v", err) + } + } + return listersv1alpha1.NewWorkerPoolLister(poolIdx), listersv1alpha1.NewSandboxConfigLister(configIdx) +} + +func testAssets() map[string]map[string]atev1alpha1.AssetFile { + return map[string]map[string]atev1alpha1.AssetFile{ + "amd64": {"gvisor": {URL: "gs://bucket/gvisor.tar.bz2", SHA256: "abc"}}, + } +} + +// TestResolveSandboxAssetsCarriesPauseImage pins that the pause image travels +// with the sandbox binaries — it is resolved from the pool's SandboxConfig, not +// from the ActorTemplate — for both the named and the class-default config. +func TestResolveSandboxAssetsCarriesPauseImage(t *testing.T) { + const ( + defaultPause = "registry.k8s.io/pause@sha256:default" + namedPause = "gcr.io/gke-release/pause@sha256:named" + ) + defaultConfig := &atev1alpha1.SandboxConfig{ + ObjectMeta: metav1.ObjectMeta{Name: "gvisor-default"}, + Spec: atev1alpha1.SandboxConfigSpec{ + SandboxClass: atev1alpha1.SandboxClassGvisor, + Default: true, + PauseImage: defaultPause, + Assets: testAssets(), + }, + } + namedConfig := &atev1alpha1.SandboxConfig{ + ObjectMeta: metav1.ObjectMeta{Name: "gvisor-custom"}, + Spec: atev1alpha1.SandboxConfigSpec{ + SandboxClass: atev1alpha1.SandboxClassGvisor, + PauseImage: namedPause, + Assets: testAssets(), + }, + } + + tests := []struct { + name string + configName string + wantPauseImage string + }{ + {name: "class default", wantPauseImage: defaultPause}, + {name: "named config", configName: "gvisor-custom", wantPauseImage: namedPause}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + pool := &atev1alpha1.WorkerPool{ + ObjectMeta: metav1.ObjectMeta{Name: "pool1", Namespace: "worker-ns"}, + Spec: atev1alpha1.WorkerPoolSpec{SandboxConfigName: tt.configName}, + } + poolLister, configLister := listersFor(t, []*atev1alpha1.WorkerPool{pool}, + []*atev1alpha1.SandboxConfig{defaultConfig, namedConfig}) + + got, err := resolveSandboxAssets(poolLister, configLister, "worker-ns", "pool1") + if err != nil { + t.Fatalf("resolveSandboxAssets() error: %v", err) + } + if got.GetPauseImage() != tt.wantPauseImage { + t.Errorf("pause image = %q, want %q", got.GetPauseImage(), tt.wantPauseImage) + } + }) + } +} diff --git a/cmd/ateapi/internal/controlapi/workload_spec.go b/cmd/ateapi/internal/controlapi/workload_spec.go index 6e1b12d0d..15eae5ebe 100644 --- a/cmd/ateapi/internal/controlapi/workload_spec.go +++ b/cmd/ateapi/internal/controlapi/workload_spec.go @@ -37,9 +37,7 @@ const envSecretCacheTTL = 30 * time.Second // container env vars. Use this when downstream consumers (e.g. checkpoint // requests) don't need env entries materialized. func workloadSpecFromActorTemplate(actorTemplate *atev1alpha1.ActorTemplate, actor *ateapipb.Actor) (*ateletpb.WorkloadSpec, error) { - workloadSpec := &ateletpb.WorkloadSpec{ - PauseImage: actorTemplate.Spec.PauseImage, - } + workloadSpec := &ateletpb.WorkloadSpec{} // add volumes for _, vol := range actorTemplate.Spec.Volumes { diff --git a/cmd/ateapi/internal/controlapi/workload_spec_test.go b/cmd/ateapi/internal/controlapi/workload_spec_test.go index 0de964056..ddd85d859 100644 --- a/cmd/ateapi/internal/controlapi/workload_spec_test.go +++ b/cmd/ateapi/internal/controlapi/workload_spec_test.go @@ -43,7 +43,6 @@ func TestWorkloadSpecFromActorTemplate(t *testing.T) { template: &atev1alpha1.ActorTemplate{ ObjectMeta: metav1.ObjectMeta{Name: "tmpl1", Namespace: "agent-ns"}, Spec: atev1alpha1.ActorTemplateSpec{ - PauseImage: "pause", Volumes: []atev1alpha1.Volume{ {Name: "home", VolumeSource: atev1alpha1.VolumeSource{DurableDir: &atev1alpha1.DurableDirVolumeSource{}}}, }, @@ -60,7 +59,6 @@ func TestWorkloadSpecFromActorTemplate(t *testing.T) { }, }, want: &ateletpb.WorkloadSpec{ - PauseImage: "pause", Volumes: []*ateletpb.Volume{ { Name: "home", @@ -227,7 +225,6 @@ func TestWorkloadSpecFromActorTemplateWithEnv(t *testing.T) { template: &atev1alpha1.ActorTemplate{ ObjectMeta: metav1.ObjectMeta{Name: "tmpl1", Namespace: "agent-ns"}, Spec: atev1alpha1.ActorTemplateSpec{ - PauseImage: "pause", Containers: []atev1alpha1.Container{ { Name: "main", @@ -247,7 +244,6 @@ func TestWorkloadSpecFromActorTemplateWithEnv(t *testing.T) { }, }, want: &ateletpb.WorkloadSpec{ - PauseImage: "pause", Containers: []*ateletpb.Container{ { Name: "main", diff --git a/cmd/atelet/main.go b/cmd/atelet/main.go index 99844673b..213a011f3 100644 --- a/cmd/atelet/main.go +++ b/cmd/atelet/main.go @@ -420,7 +420,7 @@ func (s *AteomHerder) Run(ctx context.Context, req *ateletpb.RunRequest) (resp * } if err := s.prepareOCIBundles(ctx, actorUID, actorRef.Name, - req.GetSpec(), req.GetTargetAteomUid(), + req.GetSpec(), sandboxRec.PauseImage, req.GetTargetAteomUid(), ); err != nil { return nil, ateerrors.CrashIfReason(ctx, err, ateerrors.ReasonInvalidContainerConfig) } @@ -825,12 +825,12 @@ func (s *AteomHerder) Restore(ctx context.Context, req *ateletpb.RestoreRequest) op.kind = restoreSnapshotKind(req, sandboxRec) op.sandboxClass = sandboxRec.SandboxClass - // The record whose pinned binaries run the restored workload: the golden's - // for a DATA_ON_GOLDEN restore, the snapshot's own otherwise. The golden's - // set wins because the guest state being resumed is the golden snapshot's - // memory image, and a memory image must be resumed by the exact binary - // versions that produced it; the actor's snapshot contributes only durable - // data (a plain tar), which no binary version reads back. + // The record whose pinned sandbox (binaries + pause image) runs the restored + // workload: the golden's for a DATA_ON_GOLDEN restore, the snapshot's own + // otherwise. The golden's set wins because the guest state being resumed is + // the golden snapshot's memory image, and a memory image must be resumed by + // the exact sandbox that produced it; the actor's snapshot contributes only + // durable data (a plain tar), which no sandbox version reads back. runtimeRec := sandboxRec if goldenRec != nil { runtimeRec = goldenRec @@ -905,7 +905,7 @@ func (s *AteomHerder) Restore(ctx context.Context, req *ateletpb.RestoreRequest) return ateerrors.CrashIfReason(ctx, err, ateerrors.ReasonFailedGetExternalObject, ateerrors.ReasonInvalidObjectURL, ateerrors.ReasonTerminalFileSystemError, ateerrors.ReasonInvalidSandboxAsset) } t := time.Now() - err = s.prepareOCIBundles(gctx, actorUID, actorRef.Name, req.GetSpec(), req.GetTargetAteomUid()) + err = s.prepareOCIBundles(gctx, actorUID, actorRef.Name, req.GetSpec(), runtimeRec.PauseImage, req.GetTargetAteomUid()) dBundles = time.Since(t) if err != nil { prepFailedPhase = ateattr.SnapshotPhaseOCIUnpack @@ -1201,12 +1201,16 @@ func (s *AteomHerder) downloadExternalCheckpoint(ctx context.Context, snapshotUR } // prepareOCIBundles pulls images and assembles OCI bundles for the pause -// container and every application container in spec, in parallel. +// container and every application container in spec, in parallel. pauseImage +// comes from the sandbox record, not the workload spec: it is sandbox +// configuration, and on a restore it must be the image the snapshot was taken +// with. func (s *AteomHerder) prepareOCIBundles( ctx context.Context, actorUID string, actorName string, spec *ateletpb.WorkloadSpec, + pauseImage string, targetAteomUid string, ) error { // Populate the per-actor identity directory that gets bind-mounted into @@ -1252,7 +1256,7 @@ func (s *AteomHerder) prepareOCIBundles( s.imageCache, actorUID, "pause", - spec.GetPauseImage(), + pauseImage, []string{"/pause"}, nil, nil, diff --git a/cmd/atelet/main_test.go b/cmd/atelet/main_test.go index ffe0f9d06..f786a2033 100644 --- a/cmd/atelet/main_test.go +++ b/cmd/atelet/main_test.go @@ -70,7 +70,7 @@ func TestSnapshotManifestActorMetadata(t *testing.T) { // written before the scope field existed must still parse, reporting an empty // scope, and a scope-less record must not serialize a scope key at all. func TestSnapshotManifestScopeAbsent(t *testing.T) { - legacy := []byte(`{"sandboxClass":"gvisor","snapshotFiles":["checkpoint.img"]}`) + legacy := []byte(`{"sandboxClass":"gvisor","pauseImage":"pause@sha256:abc","snapshotFiles":["checkpoint.img"]}`) rec, err := unmarshalSandboxRecord(legacy) if err != nil { t.Fatalf("unmarshalSandboxRecord(legacy manifest): %v", err) @@ -88,6 +88,18 @@ func TestSnapshotManifestScopeAbsent(t *testing.T) { } } +// TestSnapshotManifestRequiresPauseImage pins that a manifest without a pause +// image is rejected outright rather than yielding an empty image that would +// fail later, deep in the image pull. +func TestSnapshotManifestRequiresPauseImage(t *testing.T) { + noPause := []byte(`{"sandboxClass":"gvisor","snapshotFiles":["checkpoint.img"]}`) + if _, err := unmarshalSandboxRecord(noPause); err == nil { + t.Fatal("unmarshalSandboxRecord accepted a manifest with no pauseImage") + } else if !strings.Contains(err.Error(), "pauseImage") { + t.Errorf("error = %v, want it to name pauseImage", err) + } +} + func TestWriteFileAtomic(t *testing.T) { dir := t.TempDir() target := filepath.Join(dir, "actor-id") @@ -636,7 +648,6 @@ func TestRPCBoundariesReject(t *testing.T) { func TestBuildAteomWorkloadSpecForwardsReadyz(t *testing.T) { in := &ateletpb.WorkloadSpec{ - PauseImage: "pause", Containers: []*ateletpb.Container{ { Name: "with-probe", diff --git a/cmd/atelet/sandbox_assets.go b/cmd/atelet/sandbox_assets.go index ba62d6b0c..808e3af21 100644 --- a/cmd/atelet/sandbox_assets.go +++ b/cmd/atelet/sandbox_assets.go @@ -67,15 +67,20 @@ type assetEntry struct { } // sandboxAssetsRecord is the sandbox runtime an actor is running, projected onto -// the local node's architecture: the sandbox class plus the asset set keyed by -// asset name (gVisor uses a single "gvisor" release-tarball asset; records -// written before the tarball release mechanism use a bare "runsc" asset). +// the local node's architecture: the sandbox class and pause image plus the +// asset set keyed by asset name (gVisor uses a single "gvisor" release-tarball +// asset; records written before the tarball release mechanism use a bare +// "runsc" asset). // It is both the per-actor on-node record (written at Run/Restore, read at // Checkpoint) and the snapshot manifest (written at Checkpoint, read at // Restore). type sandboxAssetsRecord struct { SandboxClass string `json:"sandboxClass"` Assets map[string]assetEntry `json:"assets"` + // PauseImage is the root sandbox container's image. It is recorded here + // rather than taken from the request at Restore so a snapshot is rebuilt + // with the same sandbox it was captured from. + PauseImage string `json:"pauseImage"` // Actor identity makes a flat snapshot self-identifying if control-plane // persistence is unavailable. Atespace string `json:"atespace,omitempty"` @@ -107,8 +112,12 @@ func recordFromRequest(sa *ateletpb.SandboxAssets) (*sandboxAssetsRecord, error) if archAssets == nil || len(archAssets.GetFiles()) == 0 { return nil, fmt.Errorf("sandbox_assets has no assets for architecture %q", arch) } + if sa.GetPauseImage() == "" { + return nil, fmt.Errorf("sandbox_assets has no pause_image") + } rec := &sandboxAssetsRecord{ SandboxClass: sa.GetSandboxClass(), + PauseImage: sa.GetPauseImage(), Assets: make(map[string]assetEntry, len(archAssets.GetFiles())), } for name, f := range archAssets.GetFiles() { @@ -416,6 +425,12 @@ func unmarshalSandboxRecord(data []byte) (*sandboxAssetsRecord, error) { if err := json.Unmarshal(data, rec); err != nil { return nil, fmt.Errorf("%w: while parsing sandbox record/manifest: %w", ateerrors.ReasonInvalidSandboxAsset, err) } + // Fail loudly rather than let an empty image reach the image pull: a record + // without one predates the pause image moving into the sandbox config, and + // its snapshot cannot be rebuilt with a known-matching sandbox. + if rec.PauseImage == "" { + return nil, fmt.Errorf("%w: sandbox record/manifest has no pauseImage", ateerrors.ReasonInvalidSandboxAsset) + } return rec, nil } diff --git a/demos/autoscaled-workerpool/autoscaled-workerpool.yaml.tmpl b/demos/autoscaled-workerpool/autoscaled-workerpool.yaml.tmpl index 4880383c8..74e54b8f7 100644 --- a/demos/autoscaled-workerpool/autoscaled-workerpool.yaml.tmpl +++ b/demos/autoscaled-workerpool/autoscaled-workerpool.yaml.tmpl @@ -45,7 +45,6 @@ metadata: name: counter namespace: ate-demo-autoscaled-workerpool spec: - pauseImage: "registry.k8s.io/pause:3.10.2@sha256:f548e0e8e3dc1896ca956272154dde3314e8cc4fde0a57577ee9fa1c63f5baf4" containers: - name: counter image: ko://github.com/agent-substrate/substrate/demos/counter diff --git a/demos/claude-code-multiplex/claude-code-multiplex.yaml.tmpl b/demos/claude-code-multiplex/claude-code-multiplex.yaml.tmpl index eb06f284a..e48268992 100644 --- a/demos/claude-code-multiplex/claude-code-multiplex.yaml.tmpl +++ b/demos/claude-code-multiplex/claude-code-multiplex.yaml.tmpl @@ -65,7 +65,6 @@ metadata: name: agent-luna namespace: claude-multiplex-demo spec: - pauseImage: "registry.k8s.io/pause:3.10.2@sha256:f548e0e8e3dc1896ca956272154dde3314e8cc4fde0a57577ee9fa1c63f5baf4" containers: - name: claude image: ${WORKLOAD_IMAGE} @@ -95,7 +94,6 @@ metadata: name: agent-mars namespace: claude-multiplex-demo spec: - pauseImage: "registry.k8s.io/pause:3.10.2@sha256:f548e0e8e3dc1896ca956272154dde3314e8cc4fde0a57577ee9fa1c63f5baf4" containers: - name: claude image: ${WORKLOAD_IMAGE} @@ -125,7 +123,6 @@ metadata: name: agent-orion namespace: claude-multiplex-demo spec: - pauseImage: "registry.k8s.io/pause:3.10.2@sha256:f548e0e8e3dc1896ca956272154dde3314e8cc4fde0a57577ee9fa1c63f5baf4" containers: - name: claude image: ${WORKLOAD_IMAGE} diff --git a/demos/counter/counter-microvm.yaml.tmpl b/demos/counter/counter-microvm.yaml.tmpl index e661c5865..960bd9018 100644 --- a/demos/counter/counter-microvm.yaml.tmpl +++ b/demos/counter/counter-microvm.yaml.tmpl @@ -51,7 +51,6 @@ spec: # sandbox classes, so only pools of the same class are eligible to run this # template's actors. sandboxClass: microvm - pauseImage: "registry.k8s.io/pause:3.10.2@sha256:f548e0e8e3dc1896ca956272154dde3314e8cc4fde0a57577ee9fa1c63f5baf4" containers: - name: counter image: ko://github.com/agent-substrate/substrate/demos/counter diff --git a/demos/counter/counter.yaml.tmpl b/demos/counter/counter.yaml.tmpl index fa8801842..892a21f99 100644 --- a/demos/counter/counter.yaml.tmpl +++ b/demos/counter/counter.yaml.tmpl @@ -38,7 +38,6 @@ metadata: name: counter namespace: ate-demo-counter spec: - pauseImage: "registry.k8s.io/pause:3.10.2@sha256:f548e0e8e3dc1896ca956272154dde3314e8cc4fde0a57577ee9fa1c63f5baf4" containers: - name: counter image: ko://github.com/agent-substrate/substrate/demos/counter diff --git a/demos/multi-template/multi-template.yaml.tmpl b/demos/multi-template/multi-template.yaml.tmpl index b4d0fc4e4..c19ab8724 100644 --- a/demos/multi-template/multi-template.yaml.tmpl +++ b/demos/multi-template/multi-template.yaml.tmpl @@ -54,7 +54,6 @@ metadata: name: counter namespace: ate-demo-multi-template-counter spec: - pauseImage: "registry.k8s.io/pause:3.10.2@sha256:f548e0e8e3dc1896ca956272154dde3314e8cc4fde0a57577ee9fa1c63f5baf4" containers: - name: counter image: ko://github.com/agent-substrate/substrate/demos/counter @@ -70,7 +69,6 @@ metadata: name: fspersist namespace: ate-demo-multi-template-fspersist spec: - pauseImage: "registry.k8s.io/pause:3.10.2@sha256:f548e0e8e3dc1896ca956272154dde3314e8cc4fde0a57577ee9fa1c63f5baf4" containers: - name: fspersist image: ko://github.com/agent-substrate/substrate/demos/multi-template/fspersist diff --git a/demos/parking/parking.yaml.tmpl b/demos/parking/parking.yaml.tmpl index a4406a8ec..3ede10394 100644 --- a/demos/parking/parking.yaml.tmpl +++ b/demos/parking/parking.yaml.tmpl @@ -50,7 +50,6 @@ metadata: name: parking namespace: ate-demo-parking spec: - pauseImage: "registry.k8s.io/pause:3.10.2@sha256:f548e0e8e3dc1896ca956272154dde3314e8cc4fde0a57577ee9fa1c63f5baf4" containers: - name: counter image: ko://github.com/agent-substrate/substrate/demos/counter diff --git a/demos/sandbox/manual-test-multi.yaml b/demos/sandbox/manual-test-multi.yaml index 909fc8c83..4c456afd5 100644 --- a/demos/sandbox/manual-test-multi.yaml +++ b/demos/sandbox/manual-test-multi.yaml @@ -37,7 +37,6 @@ spec: workerSelector: matchLabels: workload: sandbox - pauseImage: "registry.k8s.io/pause:3.10.2@sha256:f548e0e8e3dc1896ca956272154dde3314e8cc4fde0a57577ee9fa1c63f5baf4" containers: - name: sandbox image: ko://github.com/agent-substrate/substrate/demos/sandbox diff --git a/demos/sandbox/sandbox.yaml.tmpl b/demos/sandbox/sandbox.yaml.tmpl index 0265eea62..b9a868f04 100644 --- a/demos/sandbox/sandbox.yaml.tmpl +++ b/demos/sandbox/sandbox.yaml.tmpl @@ -37,7 +37,6 @@ spec: workerSelector: matchLabels: workload: sandbox - pauseImage: "registry.k8s.io/pause:3.10.2@sha256:f548e0e8e3dc1896ca956272154dde3314e8cc4fde0a57577ee9fa1c63f5baf4" containers: - name: sandbox image: ko://github.com/agent-substrate/substrate/demos/sandbox diff --git a/docs/api-guide.md b/docs/api-guide.md index 2ef434710..02bb563e6 100644 --- a/docs/api-guide.md +++ b/docs/api-guide.md @@ -13,7 +13,7 @@ The `WorkerPool` defines the pool of physical "warm" compute capacity. It manage | `replicas` | `int32` | **Required.** Number of physical standby pods to maintain in the cluster. | | `ateomImage` | `string` | **Required.** The container image for the `ateom` herder process (e.g. `ko://github.com/agent-substrate/substrate/cmd/ateom-gvisor`). | | `sandboxClass` | `string` | Optional. The sandbox runtime family for the pool: `gvisor` (default) or `microvm`. Drives the worker pod shape (e.g. KVM device mounts, node placement) and which `SandboxConfig`s are eligible. | -| `sandboxConfigName` | `string` | Optional. Name of a cluster-scoped [`SandboxConfig`](#3-sandboxconfig-sandbox-binaries) providing the sandbox binaries. If empty, the cluster default `SandboxConfig` for the pool's `sandboxClass` is used. | +| `sandboxConfigName` | `string` | Optional. Name of a cluster-scoped [`SandboxConfig`](#3-sandboxconfig-the-sandbox-itself) providing the sandbox binaries and pause image. If empty, the cluster default `SandboxConfig` for the pool's `sandboxClass` is used. | | `template` | `WorkerPoolPodTemplate` | **Optional.** Pod scheduling and resource settings for worker pods. | #### `WorkerPoolPodTemplate` (`spec.template`) @@ -144,10 +144,9 @@ The `ActorTemplate` defines the code, environment, and state-management policies | `sandboxClass` | `string` | Optional. The sandbox runtime family this template's actors require: `gvisor` (default) or `microvm`. Only `WorkerPool`s whose `sandboxClass` matches are eligible. | | `workerSelector` | `*LabelSelector` | Optional. Gates which `WorkerPool`s actors from this template may use, by matching against each pool's labels. If unset, all pools are eligible (subject to the actor's own `worker_selector`). | | `snapshotsConfig` | `SnapshotsConfig` | **Required.** The base object-storage location snapshots are written under, plus the pause/commit/resume scopes. See [Snapshot Storage Layout](#snapshot-storage-layout). | -| `pauseImage` | `string` | **Required.** The image used for the sandbox root (e.g. `gcr.io/gke-release/pause`). | | `volumes` | `[]Volume` | Optional. Volumes the containers may mount, each either a `durableDir` or an `externalVolumeTemplate`. Every declared volume must be mounted by at least one container. A `microvm` template may declare several `durableDir` volumes; a `gvisor` template is limited to one, and `externalVolumeTemplate` is `gvisor`-only. | -The sandbox binaries (e.g. the gVisor `runsc` binary) are **no longer configured on the `ActorTemplate`**. They are resolved from the referenced `WorkerPool`'s [`SandboxConfig`](#3-sandboxconfig-sandbox-binaries) — by name (`workerPool.spec.sandboxConfigName`) or, by default, the cluster default `SandboxConfig` for the pool's `sandboxClass`. +The sandbox itself — the binaries (e.g. the gVisor `runsc` binary) and the `pauseImage` holding the sandbox's namespaces — is **not configured on the `ActorTemplate`**. It is resolved from the referenced `WorkerPool`'s [`SandboxConfig`](#3-sandboxconfig-the-sandbox-itself) — by name (`workerPool.spec.sandboxConfigName`) or, by default, the cluster default `SandboxConfig` for the pool's `sandboxClass`. Because a snapshot is not restorable across sandbox runtimes, `sandboxClass` is a **hard scheduling gate**: an actor is only ever placed on a `WorkerPool` of the matching class. It is AND'd with `workerSelector` (and the actor's `worker_selector`), which can only narrow the eligible pools further. It defaults to `gvisor` and, like the rest of the spec, is immutable, so each template's class is fixed at creation. @@ -207,10 +206,8 @@ metadata: name: secret-agent namespace: ate-demo spec: - # No sandbox/runsc config here — the binaries come from the WorkerPool's - # SandboxConfig (see section 3). - # GKE clusters: use gcr.io/gke-release/pause. Other clusters: registry.k8s.io/pause:3.9 - pauseImage: "gcr.io/gke-release/pause@sha256:bcbd57ba5653580ec647b16d8163cdd1112df3609129b01f912a8032e48265da" + # No sandbox config here — the binaries and pause image come from the + # WorkerPool's SandboxConfig (see section 3). containers: - name: agent image: gcr.io/my-project/my-agent:latest @@ -261,9 +258,9 @@ Two consequences worth planning for: --- -## 3. SandboxConfig: Sandbox Binaries +## 3. SandboxConfig: The Sandbox Itself -`SandboxConfig` is a **cluster-scoped** resource that decouples the sandbox binaries (the gVisor `runsc` binary, or a micro-VM kernel/firmware/config) from the `ActorTemplate`. A `WorkerPool` resolves its binaries from a `SandboxConfig` — either the one named by `spec.sandboxConfigName`, or the cluster default for the pool's `sandboxClass`. +`SandboxConfig` is a **cluster-scoped** resource that decouples the sandbox — its binaries (the gVisor `runsc` binary, or a micro-VM kernel/firmware/config) and the `pauseImage` that holds the sandbox's namespaces — from the `ActorTemplate`. A `WorkerPool` resolves its sandbox from a `SandboxConfig` — either the one named by `spec.sandboxConfigName`, or the cluster default for the pool's `sandboxClass`. This means a single, cluster-managed config pins the sandbox runtime version for many templates: snapshots stay restorable because the version is recorded in each snapshot's manifest, and operators upgrade the runtime in one place. @@ -272,6 +269,7 @@ This means a single, cluster-managed config pins the sandbox runtime version for | Field | Type | Description | | :--- | :--- | :--- | | `sandboxClass` | `string` | **Required.** Runtime family this config applies to: `gvisor` (default) or `microvm`. A `WorkerPool` only uses `SandboxConfig`s whose `sandboxClass` matches its own. | +| `pauseImage` | `string` | **Required.** The image for the sandbox's root container (e.g. `registry.k8s.io/pause`, or `gcr.io/gke-release/pause` on GKE). Must be pinned by digest (`...@sha256:...`) — it is recorded in each snapshot's manifest so a restore rebuilds the sandbox from the same image. | | `default` | `bool` | Optional. Marks this as the cluster default for its `sandboxClass`. A `WorkerPool` with no `sandboxConfigName` resolves to the default for its class. At most one default per class. | | `assets` | `map[arch]map[name]AssetFile` | Optional. Content-addressed files atelet fetches, keyed by architecture (`amd64`, `arm64`) then asset name. gVisor expects a `gvisor` asset (the release's `gvisor.tar.bz2`), which atelet auto-extracts. A micro-VM backend expects several. Each `AssetFile` is a `{ url, sha256 }` pair. | @@ -287,6 +285,7 @@ metadata: spec: sandboxClass: gvisor default: true + pauseImage: "registry.k8s.io/pause:3.10.2@sha256:f548e0e8e3dc1896ca956272154dde3314e8cc4fde0a57577ee9fa1c63f5baf4" assets: amd64: gvisor: diff --git a/docs/architecture.md b/docs/architecture.md index c4b1a2fee..38b14fe9c 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -324,7 +324,7 @@ The node-level subsystem manages the physical execution of sandboxes and the mov ### Sandbox Classes -A `WorkerPool` selects a **sandbox class** (`spec.sandboxClass`), and each class has a matching `ateom` herder image. The sandbox binaries themselves are not baked into the worker image — they are fetched at runtime from a cluster-scoped [`SandboxConfig`](api-guide.md#3-sandboxconfig-sandbox-binaries) and pinned into each snapshot's manifest so restores stay reproducible across runtime upgrades. +A `WorkerPool` selects a **sandbox class** (`spec.sandboxClass`), and each class has a matching `ateom` herder image. The sandbox binaries themselves are not baked into the worker image — they, and the pause image holding the sandbox's namespaces, come at runtime from a cluster-scoped [`SandboxConfig`](api-guide.md#3-sandboxconfig-the-sandbox-itself) and are pinned into each snapshot's manifest so restores stay reproducible across runtime upgrades. * **gVisor** (`ateom-gvisor`, the default): Runs the workload under `runsc` for kernel-level sandboxing. Suspend and resume leverage gVisor's native checkpoint/restore of the sandboxed process tree. diff --git a/docs/glossary.md b/docs/glossary.md index 73534ac0e..e6e651e3e 100644 --- a/docs/glossary.md +++ b/docs/glossary.md @@ -19,9 +19,10 @@ For how the pieces fit together, see the [Architecture](architecture.md) and - **SandboxConfig**: a cluster-scoped resource holding the sandbox binaries for one runtime family (the gVisor `runsc` binary, or a micro-VM - kernel/firmware/config). A `WorkerPool` resolves its binaries from the config - it names, or from the cluster default for its class, so one config pins the - runtime version for many templates. + kernel/firmware/config), plus the pause image for the sandbox's root + container. A `WorkerPool` resolves its sandbox from the config it names, or + from the cluster default for its class, so one config pins the runtime version + for many templates. ## Records (dynamic state, in the control-plane store) diff --git a/internal/e2e/fixtures/probe/probe.yaml.tmpl b/internal/e2e/fixtures/probe/probe.yaml.tmpl index f79e0cdfe..85ea73fd7 100644 --- a/internal/e2e/fixtures/probe/probe.yaml.tmpl +++ b/internal/e2e/fixtures/probe/probe.yaml.tmpl @@ -38,7 +38,6 @@ metadata: name: probe namespace: ate-e2e-probe spec: - pauseImage: "registry.k8s.io/pause:3.10.2@sha256:f548e0e8e3dc1896ca956272154dde3314e8cc4fde0a57577ee9fa1c63f5baf4" containers: - name: probe image: ko://github.com/agent-substrate/substrate/internal/e2e/fixtures/probe diff --git a/internal/e2e/suites/demo/demo_test.go b/internal/e2e/suites/demo/demo_test.go index da94d7aad..7b3f8a690 100644 --- a/internal/e2e/suites/demo/demo_test.go +++ b/internal/e2e/suites/demo/demo_test.go @@ -860,7 +860,6 @@ func createActorTemplateInternal(ctx context.Context, t *testing.T, clients *e2e // ActorTemplate↔WorkerPool match succeeds. The micro-VM source sets // "microvm"; the gVisor source leaves it "" — copying keeps both correct. SandboxClass: existingAt.Spec.SandboxClass, - PauseImage: existingAt.Spec.PauseImage, Containers: existingAt.Spec.Containers, SnapshotsConfig: v1alpha1.SnapshotsConfig{ Location: "gs://" + env["BUCKET_NAME"] + "/ate-demo-" + name, diff --git a/internal/e2e/suites/networkpolicy/networkpolicy_test.go b/internal/e2e/suites/networkpolicy/networkpolicy_test.go index 8b441a0ec..033f08485 100644 --- a/internal/e2e/suites/networkpolicy/networkpolicy_test.go +++ b/internal/e2e/suites/networkpolicy/networkpolicy_test.go @@ -326,7 +326,6 @@ func setupDemoCounterTemplate(ctx context.Context, t *testing.T, clients *e2e.Cl MatchLabels: map[string]string{"netpol-test": ns}, }, SandboxClass: existingAt.Spec.SandboxClass, - PauseImage: existingAt.Spec.PauseImage, Containers: existingAt.Spec.Containers, SnapshotsConfig: existingAt.Spec.SnapshotsConfig, Volumes: existingAt.Spec.Volumes, diff --git a/internal/e2e/suites/parking/parking_test.go b/internal/e2e/suites/parking/parking_test.go index 5cfe14fbc..233b95fec 100644 --- a/internal/e2e/suites/parking/parking_test.go +++ b/internal/e2e/suites/parking/parking_test.go @@ -210,7 +210,6 @@ func createParkingFixture(ctx context.Context, t *testing.T, clients *e2e.Client MatchLabels: map[string]string{"demo": nsObj.Name}, }, SandboxClass: existingAt.Spec.SandboxClass, - PauseImage: existingAt.Spec.PauseImage, Containers: existingAt.Spec.Containers, SnapshotsConfig: v1alpha1.SnapshotsConfig{ Location: "gs://" + env["BUCKET_NAME"] + "/e2e-parking-" + nsObj.Name, diff --git a/internal/proto/ateletpb/atelet.pb.go b/internal/proto/ateletpb/atelet.pb.go index cc5a3f693..834d61728 100644 --- a/internal/proto/ateletpb/atelet.pb.go +++ b/internal/proto/ateletpb/atelet.pb.go @@ -563,14 +563,19 @@ func (x *ArchAssets) GetFiles() map[string]*AssetFile { } // SandboxAssets is the generic, backend-agnostic description of the sandbox -// binaries for an actor: a sandbox class plus assets keyed first by -// architecture (GOARCH) and then by asset name. atelet's backend code -// interprets the asset names (gVisor expects "gvisor", the release tarball; -// legacy "runsc", a bare binary, is still accepted). +// an actor runs in: a sandbox class, the pause image holding its namespaces, +// and binaries keyed first by architecture (GOARCH) and then by asset name. +// atelet's backend code interprets the asset names (gVisor expects "gvisor", +// the release tarball; legacy "runsc", a bare binary, is still accepted). type SandboxAssets struct { - state protoimpl.MessageState `protogen:"open.v1"` - SandboxClass string `protobuf:"bytes,1,opt,name=sandbox_class,json=sandboxClass,proto3" json:"sandbox_class,omitempty"` // e.g. "gvisor" - Assets map[string]*ArchAssets `protobuf:"bytes,2,rep,name=assets,proto3" json:"assets,omitempty" protobuf_key:"bytes,1,opt,name=key" protobuf_val:"bytes,2,opt,name=value"` // arch -> {name -> file} + state protoimpl.MessageState `protogen:"open.v1"` + SandboxClass string `protobuf:"bytes,1,opt,name=sandbox_class,json=sandboxClass,proto3" json:"sandbox_class,omitempty"` // e.g. "gvisor" + Assets map[string]*ArchAssets `protobuf:"bytes,2,rep,name=assets,proto3" json:"assets,omitempty" protobuf_key:"bytes,1,opt,name=key" protobuf_val:"bytes,2,opt,name=value"` // arch -> {name -> file} + // pause_image is the image for the sandbox's root container. Like the + // binaries above it is sandbox configuration, not workload configuration, + // and atelet pins it into the snapshot manifest so a restore rebuilds the + // sandbox from the same image. + PauseImage string `protobuf:"bytes,3,opt,name=pause_image,json=pauseImage,proto3" json:"pause_image,omitempty"` unknownFields protoimpl.UnknownFields sizeCache protoimpl.SizeCache } @@ -619,11 +624,17 @@ func (x *SandboxAssets) GetAssets() map[string]*ArchAssets { return nil } +func (x *SandboxAssets) GetPauseImage() string { + if x != nil { + return x.PauseImage + } + return "" +} + // WorkloadSpec parallels Pod, but with far fewer configurable fields. type WorkloadSpec struct { state protoimpl.MessageState `protogen:"open.v1"` Containers []*Container `protobuf:"bytes,1,rep,name=containers,proto3" json:"containers,omitempty"` - PauseImage string `protobuf:"bytes,2,opt,name=pause_image,json=pauseImage,proto3" json:"pause_image,omitempty"` Volumes []*Volume `protobuf:"bytes,3,rep,name=volumes,proto3" json:"volumes,omitempty"` unknownFields protoimpl.UnknownFields sizeCache protoimpl.SizeCache @@ -666,13 +677,6 @@ func (x *WorkloadSpec) GetContainers() []*Container { return nil } -func (x *WorkloadSpec) GetPauseImage() string { - if x != nil { - return x.PauseImage - } - return "" -} - func (x *WorkloadSpec) GetVolumes() []*Volume { if x != nil { return x.Volumes @@ -1760,20 +1764,20 @@ const file_atelet_proto_rawDesc = "" + "\n" + "FilesEntry\x12\x10\n" + "\x03key\x18\x01 \x01(\tR\x03key\x12'\n" + - "\x05value\x18\x02 \x01(\v2\x11.atelet.AssetFileR\x05value:\x028\x01\"\xbe\x01\n" + + "\x05value\x18\x02 \x01(\v2\x11.atelet.AssetFileR\x05value:\x028\x01\"\xdf\x01\n" + "\rSandboxAssets\x12#\n" + "\rsandbox_class\x18\x01 \x01(\tR\fsandboxClass\x129\n" + - "\x06assets\x18\x02 \x03(\v2!.atelet.SandboxAssets.AssetsEntryR\x06assets\x1aM\n" + + "\x06assets\x18\x02 \x03(\v2!.atelet.SandboxAssets.AssetsEntryR\x06assets\x12\x1f\n" + + "\vpause_image\x18\x03 \x01(\tR\n" + + "pauseImage\x1aM\n" + "\vAssetsEntry\x12\x10\n" + "\x03key\x18\x01 \x01(\tR\x03key\x12(\n" + - "\x05value\x18\x02 \x01(\v2\x12.atelet.ArchAssetsR\x05value:\x028\x01\"\x8c\x01\n" + + "\x05value\x18\x02 \x01(\v2\x12.atelet.ArchAssetsR\x05value:\x028\x01\"~\n" + "\fWorkloadSpec\x121\n" + "\n" + "containers\x18\x01 \x03(\v2\x11.atelet.ContainerR\n" + - "containers\x12\x1f\n" + - "\vpause_image\x18\x02 \x01(\tR\n" + - "pauseImage\x12(\n" + - "\avolumes\x18\x03 \x03(\v2\x0e.atelet.VolumeR\avolumes\"\x12\n" + + "containers\x12(\n" + + "\avolumes\x18\x03 \x03(\v2\x0e.atelet.VolumeR\avolumesJ\x04\b\x02\x10\x03R\vpause_image\"\x12\n" + "\x10DurableDirVolume\"\xfd\x01\n" + "\x14ExternalVolumeSource\x12*\n" + "\x11storage_volume_id\x18\x01 \x01(\tR\x0fstorageVolumeId\x12\x1f\n" + diff --git a/internal/proto/ateletpb/atelet.proto b/internal/proto/ateletpb/atelet.proto index 771db8e10..857bbf4cd 100644 --- a/internal/proto/ateletpb/atelet.proto +++ b/internal/proto/ateletpb/atelet.proto @@ -96,20 +96,28 @@ message ArchAssets { } // SandboxAssets is the generic, backend-agnostic description of the sandbox -// binaries for an actor: a sandbox class plus assets keyed first by -// architecture (GOARCH) and then by asset name. atelet's backend code -// interprets the asset names (gVisor expects "gvisor", the release tarball; -// legacy "runsc", a bare binary, is still accepted). +// an actor runs in: a sandbox class, the pause image holding its namespaces, +// and binaries keyed first by architecture (GOARCH) and then by asset name. +// atelet's backend code interprets the asset names (gVisor expects "gvisor", +// the release tarball; legacy "runsc", a bare binary, is still accepted). message SandboxAssets { string sandbox_class = 1; // e.g. "gvisor" map assets = 2; // arch -> {name -> file} + + // pause_image is the image for the sandbox's root container. Like the + // binaries above it is sandbox configuration, not workload configuration, + // and atelet pins it into the snapshot manifest so a restore rebuilds the + // sandbox from the same image. + string pause_image = 3; } // WorkloadSpec parallels Pod, but with far fewer configurable fields. message WorkloadSpec { repeated Container containers = 1; - string pause_image = 2; repeated Volume volumes = 3; + + reserved 2; + reserved "pause_image"; // moved to SandboxAssets } enum VolumeType { diff --git a/manifests/ate-install/generated/ate.dev_actortemplates.yaml b/manifests/ate-install/generated/ate.dev_actortemplates.yaml index 1dff97416..f5da2c721 100644 --- a/manifests/ate-install/generated/ate.dev_actortemplates.yaml +++ b/manifests/ate-install/generated/ate.dev_actortemplates.yaml @@ -264,19 +264,6 @@ spec: type: object maxItems: 10 type: array - pauseImage: - description: |- - PauseImage is the container to use as the root sandbox container. - - Typically, set it to [1] for on-gcp, and [2] for off-gcp - - - [1] gcr.io/gke-release/pause@sha256:bcbd57ba5653580ec647b16d8163cdd1112df3609129b01f912a8032e48265da - - [2] registry.k8s.io/pause:3.10.2@sha256:f548e0e8e3dc1896ca956272154dde3314e8cc4fde0a57577ee9fa1c63f5baf4 - type: string - x-kubernetes-validations: - - message: All images must be pinned (changing the image invalidates - snapshots) - rule: self.contains('@') sandboxClass: default: gvisor description: |- @@ -456,7 +443,6 @@ spec: type: object x-kubernetes-map-type: atomic required: - - pauseImage - snapshotsConfig type: object x-kubernetes-validations: diff --git a/manifests/ate-install/generated/ate.dev_sandboxconfigs.yaml b/manifests/ate-install/generated/ate.dev_sandboxconfigs.yaml index 71ecf752a..a282c5f9b 100644 --- a/manifests/ate-install/generated/ate.dev_sandboxconfigs.yaml +++ b/manifests/ate-install/generated/ate.dev_sandboxconfigs.yaml @@ -111,6 +111,24 @@ spec: the default config for its SandboxClass. At most one default is expected per SandboxClass. type: boolean + pauseImage: + description: |- + PauseImage is the container image used as the root sandbox container. + It holds the sandbox's namespaces and runs no workload code, so it is an + implementation detail of the sandbox rather than something actor authors + choose. It is captured in the snapshot manifest alongside the sandbox + binaries, so a restore always re-creates the sandbox from the same image + the snapshot was taken with. + + Typically, set it to [1] for on-gcp, and [2] for off-gcp + + - [1] gcr.io/gke-release/pause@sha256:bcbd57ba5653580ec647b16d8163cdd1112df3609129b01f912a8032e48265da + - [2] registry.k8s.io/pause:3.10.2@sha256:f548e0e8e3dc1896ca956272154dde3314e8cc4fde0a57577ee9fa1c63f5baf4 + type: string + x-kubernetes-validations: + - message: All images must be pinned (changing the image invalidates + snapshots) + rule: self.contains('@') sandboxClass: default: gvisor description: |- @@ -121,6 +139,7 @@ spec: - microvm type: string required: + - pauseImage - sandboxClass type: object required: diff --git a/manifests/ate-install/sandboxconfig-gvisor.yaml b/manifests/ate-install/sandboxconfig-gvisor.yaml index 1a3c9d6f2..13884cde4 100644 --- a/manifests/ate-install/sandboxconfig-gvisor.yaml +++ b/manifests/ate-install/sandboxconfig-gvisor.yaml @@ -26,6 +26,9 @@ metadata: spec: sandboxClass: gvisor default: true + # The root sandbox container's image. On GCP, prefer the in-project mirror + # gcr.io/gke-release/pause@sha256:bcbd57ba5653580ec647b16d8163cdd1112df3609129b01f912a8032e48265da. + pauseImage: "registry.k8s.io/pause:3.10.2@sha256:f548e0e8e3dc1896ca956272154dde3314e8cc4fde0a57577ee9fa1c63f5baf4" assets: amd64: gvisor: diff --git a/manifests/microvm/sandboxconfig-microvm.yaml.tmpl b/manifests/microvm/sandboxconfig-microvm.yaml.tmpl index c41094011..6285a7535 100644 --- a/manifests/microvm/sandboxconfig-microvm.yaml.tmpl +++ b/manifests/microvm/sandboxconfig-microvm.yaml.tmpl @@ -45,6 +45,9 @@ metadata: name: microvm spec: sandboxClass: microvm + # The root sandbox container's image. On GCP, prefer the in-project mirror + # gcr.io/gke-release/pause@sha256:bcbd57ba5653580ec647b16d8163cdd1112df3609129b01f912a8032e48265da. + pauseImage: "registry.k8s.io/pause:3.10.2@sha256:f548e0e8e3dc1896ca956272154dde3314e8cc4fde0a57577ee9fa1c63f5baf4" assets: arm64: cloud-hypervisor: diff --git a/pkg/api/v1alpha1/actortemplate_types.go b/pkg/api/v1alpha1/actortemplate_types.go index 1a2480a33..358e24cc9 100644 --- a/pkg/api/v1alpha1/actortemplate_types.go +++ b/pkg/api/v1alpha1/actortemplate_types.go @@ -358,17 +358,6 @@ type SnapshotsConfig struct { // +kubebuilder:validation:XValidation:rule="!has(self.sandboxClass) || self.sandboxClass != 'microvm' || !has(self.volumes) || !self.volumes.exists(v, has(v.externalVolumeTemplate))",message="ExternalVolumes are not supported when sandboxClass is 'microvm'" // +kubebuilder:validation:XValidation:rule="(has(self.sandboxClass) && self.sandboxClass == 'microvm') || !has(self.snapshotsConfig.onResume) || (has(self.snapshotsConfig.onResume.fromData) ? self.snapshotsConfig.onResume.fromData : 'ColdBoot') != 'Golden'",message="onResume.fromData: Golden is not supported when sandboxClass is 'gvisor'" type ActorTemplateSpec struct { - // PauseImage is the container to use as the root sandbox container. - // - // Typically, set it to [1] for on-gcp, and [2] for off-gcp - // - // - [1] gcr.io/gke-release/pause@sha256:bcbd57ba5653580ec647b16d8163cdd1112df3609129b01f912a8032e48265da - // - [2] registry.k8s.io/pause:3.10.2@sha256:f548e0e8e3dc1896ca956272154dde3314e8cc4fde0a57577ee9fa1c63f5baf4 - // - // +required - // +kubebuilder:validation:XValidation:rule="self.contains('@')",message="All images must be pinned (changing the image invalidates snapshots)" - PauseImage string `json:"pauseImage,omitempty"` - // Containers is the workload definition. // // +optional diff --git a/pkg/api/v1alpha1/actortemplate_validation_test.go b/pkg/api/v1alpha1/actortemplate_validation_test.go index 4b0a5d54c..89edc6c27 100644 --- a/pkg/api/v1alpha1/actortemplate_validation_test.go +++ b/pkg/api/v1alpha1/actortemplate_validation_test.go @@ -70,7 +70,6 @@ func TestActorTemplateValidation(t *testing.T) { Namespace: "default", }, Spec: ActorTemplateSpec{ - PauseImage: "gcr.io/gke-release/pause@sha256:bcbd57ba5653580ec647b16d8163cdd1112df3609129b01f912a8032e48265da", Containers: []Container{ { Name: "main", @@ -98,20 +97,6 @@ func TestActorTemplateValidation(t *testing.T) { name: "base template", mutate: func(at *ActorTemplate) {}, wantErr: false, - }, { - name: "missing PauseImage", - mutate: func(at *ActorTemplate) { - at.Spec.PauseImage = "" - }, - wantErr: true, - errMsg: "Required value", - }, { - name: "unpinned PauseImage", - mutate: func(at *ActorTemplate) { - at.Spec.PauseImage = "pause" - }, - wantErr: true, - errMsg: "All images must be pinned", }, { name: "missing SnapshotsConfig.Location", mutate: func(at *ActorTemplate) { @@ -1269,7 +1254,6 @@ func TestActorTemplateSpecImmutability(t *testing.T) { baseTemplate := &ActorTemplate{ Spec: ActorTemplateSpec{ - PauseImage: "pause@hash", Containers: []Container{ { Name: "main", @@ -1290,9 +1274,9 @@ func TestActorTemplateSpecImmutability(t *testing.T) { mutate func(*ActorTemplate) }{ { - name: "update-pause-image", + name: "update-container-image", mutate: func(at *ActorTemplate) { - at.Spec.PauseImage = "pause@new" + at.Spec.Containers[0].Image = "busybox@new" }, }, { diff --git a/pkg/api/v1alpha1/sandboxconfig_types.go b/pkg/api/v1alpha1/sandboxconfig_types.go index e7a9c98cb..80153d53b 100644 --- a/pkg/api/v1alpha1/sandboxconfig_types.go +++ b/pkg/api/v1alpha1/sandboxconfig_types.go @@ -67,6 +67,22 @@ type SandboxConfigSpec struct { // +optional Default bool `json:"default,omitempty"` + // PauseImage is the container image used as the root sandbox container. + // It holds the sandbox's namespaces and runs no workload code, so it is an + // implementation detail of the sandbox rather than something actor authors + // choose. It is captured in the snapshot manifest alongside the sandbox + // binaries, so a restore always re-creates the sandbox from the same image + // the snapshot was taken with. + // + // Typically, set it to [1] for on-gcp, and [2] for off-gcp + // + // - [1] gcr.io/gke-release/pause@sha256:bcbd57ba5653580ec647b16d8163cdd1112df3609129b01f912a8032e48265da + // - [2] registry.k8s.io/pause:3.10.2@sha256:f548e0e8e3dc1896ca956272154dde3314e8cc4fde0a57577ee9fa1c63f5baf4 + // + // +required + // +kubebuilder:validation:XValidation:rule="self.contains('@')",message="All images must be pinned (changing the image invalidates snapshots)" + PauseImage string `json:"pauseImage"` + // Assets is the set of files atelet fetches for this runtime, keyed first by // architecture (GOARCH, e.g. "amd64", "arm64") and then by asset name. The // asset names are interpreted by the sandbox backend: gVisor expects a diff --git a/pkg/api/v1alpha1/sandboxconfig_validation_test.go b/pkg/api/v1alpha1/sandboxconfig_validation_test.go index 6dc1a2dbb..374edf966 100644 --- a/pkg/api/v1alpha1/sandboxconfig_validation_test.go +++ b/pkg/api/v1alpha1/sandboxconfig_validation_test.go @@ -35,16 +35,25 @@ const validSHA256 = "a397be1abc2420d26bce6c70e6e2ff96c73aaaab929756c56f5e2089ea8 // the install — loaded here so the test guards the policy we actually ship. const vapManifestPath = "../../../manifests/ate-install/sandboxconfig-validation.yaml" +const validPauseImage = "registry.k8s.io/pause:3.10.2@sha256:f548e0e8e3dc1896ca956272154dde3314e8cc4fde0a57577ee9fa1c63f5baf4" + func sandboxConfig(name string, class SandboxClass, assets map[string]map[string]AssetFile) *SandboxConfig { return &SandboxConfig{ ObjectMeta: metav1.ObjectMeta{Name: name}, Spec: SandboxConfigSpec{ SandboxClass: class, + PauseImage: validPauseImage, Assets: assets, }, } } +// withPauseImage overrides the pause image on an otherwise-valid config. +func withPauseImage(sc *SandboxConfig, image string) *SandboxConfig { + sc.Spec.PauseImage = image + return sc +} + func runscAsset() AssetFile { return AssetFile{URL: "gs://bucket/runsc", SHA256: validSHA256} } func gvisorAsset() AssetFile { @@ -184,6 +193,16 @@ func TestSandboxConfigValidation(t *testing.T) { sc: sandboxConfig("bad-sha", SandboxClassGvisor, map[string]map[string]AssetFile{"amd64": {"runsc": {URL: "gs://bucket/runsc", SHA256: "deadbeef"}}}), wantErr: true, errMsg: "sha256", + }, { + name: "missing pauseImage", + sc: withPauseImage(sandboxConfig("bad-no-pause", SandboxClassGvisor, map[string]map[string]AssetFile{"amd64": {"gvisor": gvisorAsset()}}), ""), + wantErr: true, + errMsg: "pauseImage", + }, { + name: "unpinned pauseImage", + sc: withPauseImage(sandboxConfig("bad-unpinned-pause", SandboxClassGvisor, map[string]map[string]AssetFile{"amd64": {"gvisor": gvisorAsset()}}), "registry.k8s.io/pause:3.10.2"), + wantErr: true, + errMsg: "All images must be pinned", }} for _, tt := range tests {