From 2ef9c9eddda7bd7c3adf511a419b0e139d95ba12 Mon Sep 17 00:00:00 2001 From: Elijah Rodriguez-Beltran Date: Fri, 7 Aug 2026 21:12:56 +0000 Subject: [PATCH] Enable CSI support for micro vms. --- cmd/atelet/main.go | 68 +++++- cmd/atelet/main_test.go | 68 +++++- cmd/ateom-microvm/checkpoint.go | 8 +- cmd/ateom-microvm/csi.go | 84 +++++++ cmd/ateom-microvm/csi_test.go | 113 +++++++++ cmd/ateom-microvm/durable.go | 9 +- .../internal/kata/overlay_linux.go | 40 +++- cmd/ateom-microvm/internal/kata/restore.go | 6 + cmd/ateom-microvm/restore.go | 17 +- cmd/ateom-microvm/run.go | 52 +++- demos/counter/counter-microvm-csi-test.yaml | 67 ++++++ internal/e2e/suites/demo/demo_test.go | 3 - internal/proto/ateompb/ateom.pb.go | 222 ++++++++++++------ internal/proto/ateompb/ateom.proto | 9 + .../generated/ate.dev_actortemplates.yaml | 3 - pkg/api/v1alpha1/actortemplate_types.go | 1 - .../v1alpha1/actortemplate_validation_test.go | 5 +- 17 files changed, 654 insertions(+), 121 deletions(-) create mode 100644 cmd/ateom-microvm/csi.go create mode 100644 cmd/ateom-microvm/csi_test.go create mode 100644 demos/counter/counter-microvm-csi-test.yaml diff --git a/cmd/atelet/main.go b/cmd/atelet/main.go index c59deadd6..06c936985 100644 --- a/cmd/atelet/main.go +++ b/cmd/atelet/main.go @@ -432,6 +432,11 @@ func (s *AteomHerder) Run(ctx context.Context, req *ateletpb.RunRequest) (resp * return nil, err } + spec, err := buildAteomWorkloadSpec(req.GetSpec()) + if err != nil { + return nil, status.Errorf(codes.InvalidArgument, "invalid workload spec: %v", err) + } + // Tell ateom to start the workload. gVisor uses RunscPath; the micro-VM // runtime uses the full RuntimeAssetPaths set. if _, err := client.RunWorkload(ctx, &ateompb.RunWorkloadRequest{ @@ -441,7 +446,7 @@ func (s *AteomHerder) Run(ctx context.Context, req *ateletpb.RunRequest) (resp * ActorTemplateName: req.GetActorTemplateName(), RunscPath: runscPathFor(assetPaths), RuntimeAssetPaths: assetPaths, - Spec: buildAteomWorkloadSpec(req.GetSpec()), + Spec: spec, ActorUid: actorUID, EgressGateway: toAteomEgressGateway(req.GetEgressGateway()), }); err != nil { @@ -544,6 +549,11 @@ func (s *AteomHerder) Checkpoint(ctx context.Context, req *ateletpb.CheckpointRe // Tell ateom to take the checkpoint and delete containers. ateom reports the // exact files it wrote so we ship precisely that set (gVisor's image files, // cloud-hypervisor's snapshot set, ...) rather than a hardcoded list. + spec, err := buildAteomWorkloadSpec(req.GetSpec()) + if err != nil { + return nil, status.Errorf(codes.InvalidArgument, "invalid workload spec: %v", err) + } + tAteom := time.Now() resp, err := client.CheckpointWorkload(ctx, &ateompb.CheckpointWorkloadRequest{ Atespace: actorRef.Atespace, @@ -552,7 +562,7 @@ func (s *AteomHerder) Checkpoint(ctx context.Context, req *ateletpb.CheckpointRe ActorTemplateName: req.GetActorTemplateName(), RunscPath: runscPathFor(assetPaths), RuntimeAssetPaths: assetPaths, - Spec: buildAteomWorkloadSpec(req.GetSpec()), + Spec: spec, Scope: toAteomSnapshotScope(req.GetScope()), ActorUid: actorUID, }) @@ -566,7 +576,22 @@ func (s *AteomHerder) Checkpoint(ctx context.Context, req *ateletpb.CheckpointRe sandboxRec.SnapshotFiles = resp.GetSnapshotFiles() if len(sandboxRec.SnapshotFiles) == 0 { - return nil, ateerrors.NewGRPCError(ctx, codes.DataLoss, ateerrors.ReasonInvalidCheckpointResult, ateerrors.ActorCrashedMetadata(), errors.New("ateom reported no snapshot files for checkpoint")) + hasDurable := false + hasCsi := false + for _, vol := range req.GetSpec().GetVolumes() { + switch vol.GetType() { + case ateletpb.VolumeType_VOLUME_TYPE_DURABLE_DIR: + hasDurable = true + case ateletpb.VolumeType_VOLUME_TYPE_EXTERNAL: + hasCsi = true + } + } + isDataScope := req.GetScope() == ateletpb.SnapshotScope_SNAPSHOT_SCOPE_DATA + if isDataScope && hasCsi && !hasDurable { + // OK: CSI volumes don't produce snapshot files in DATA scope. + } else { + return nil, ateerrors.NewGRPCError(ctx, codes.DataLoss, ateerrors.ReasonInvalidCheckpointResult, ateerrors.ActorCrashedMetadata(), errors.New("ateom reported no snapshot files for checkpoint")) + } } sandboxRec.Atespace = req.GetAtespace() sandboxRec.ActorName = req.GetActorName() @@ -1069,6 +1094,11 @@ func (s *AteomHerder) Restore(ctx context.Context, req *ateletpb.RestoreRequest) // Tell ateom to do runsc create + runsc restore for pause container and // all application containers. + spec, err := buildAteomWorkloadSpec(req.GetSpec()) + if err != nil { + return nil, status.Errorf(codes.InvalidArgument, "invalid workload spec: %v", err) + } + tAteom := time.Now() _, err = client.RestoreWorkload(ctx, &ateompb.RestoreWorkloadRequest{ Atespace: actorRef.Atespace, @@ -1077,7 +1107,7 @@ func (s *AteomHerder) Restore(ctx context.Context, req *ateletpb.RestoreRequest) ActorTemplateName: req.GetActorTemplateName(), RunscPath: runscPathFor(assetPaths), RuntimeAssetPaths: assetPaths, - Spec: buildAteomWorkloadSpec(req.GetSpec()), + Spec: spec, Scope: toAteomSnapshotScope(req.GetScope()), ActorUid: req.GetActorUid(), EgressGateway: toAteomEgressGateway(req.GetEgressGateway()), @@ -1457,32 +1487,46 @@ func (s *AteomHerder) dialAteom(ctx context.Context, targetAteomUid string) (ate // buildAteomWorkloadSpec projects the atelet-facing workload spec onto // the ateom-facing one. -func buildAteomWorkloadSpec(spec *ateletpb.WorkloadSpec) *ateompb.WorkloadSpec { - ddVolumes := make(map[string]bool) +func buildAteomWorkloadSpec(spec *ateletpb.WorkloadSpec) (*ateompb.WorkloadSpec, error) { + volumes := make(map[string]ateletpb.VolumeType) for _, vol := range spec.GetVolumes() { - if vol.GetType() == ateletpb.VolumeType_VOLUME_TYPE_DURABLE_DIR { - ddVolumes[vol.GetName()] = true - } + volumes[vol.GetName()] = vol.GetType() } out := &ateompb.WorkloadSpec{} for _, ctr := range spec.GetContainers() { var ddMounts []*ateompb.DurableDirVolumeMount + var csiMounts []*ateompb.VolumeMount for _, vm := range ctr.GetVolumeMounts() { - if ddVolumes[vm.GetName()] { + volName := vm.GetName() + volType, ok := volumes[volName] + if !ok { + return nil, fmt.Errorf("container %q mounts volume %q which is not defined in workload volumes", ctr.GetName(), volName) + } + + switch volType { + case ateletpb.VolumeType_VOLUME_TYPE_DURABLE_DIR: ddMounts = append(ddMounts, &ateompb.DurableDirVolumeMount{ - VolumeName: vm.GetName(), + VolumeName: volName, + MountPath: vm.GetMountPath(), + }) + case ateletpb.VolumeType_VOLUME_TYPE_EXTERNAL: + csiMounts = append(csiMounts, &ateompb.VolumeMount{ + VolumeName: volName, MountPath: vm.GetMountPath(), }) + default: + return nil, fmt.Errorf("container %q mounts volume %q with unsupported type %v", ctr.GetName(), volName, volType) } } out.Containers = append(out.Containers, &ateompb.Container{ Name: ctr.GetName(), DurableDirVolumeMounts: ddMounts, + CsiVolumeMounts: csiMounts, Readyz: toAteomReadyz(ctr.GetReadyz()), }) } - return out + return out, nil } func toAteomEgressGateway(gateway *ateletpb.EgressGateway) *ateompb.EgressGateway { diff --git a/cmd/atelet/main_test.go b/cmd/atelet/main_test.go index b08e203d6..6c89746d9 100644 --- a/cmd/atelet/main_test.go +++ b/cmd/atelet/main_test.go @@ -679,7 +679,10 @@ func TestBuildAteomWorkloadSpecForwardsReadyz(t *testing.T) { {Name: "without-probe"}, }, } - got := buildAteomWorkloadSpec(in) + got, err := buildAteomWorkloadSpec(in) + if err != nil { + t.Fatalf("buildAteomWorkloadSpec failed: %v", err) + } if diff := cmp.Diff(want, got, protocmp.Transform()); diff != "" { t.Errorf("buildAteomWorkloadSpec mismatch (-want +got):\n%s", diff) } @@ -722,6 +725,9 @@ func TestBuildAteomWorkloadSpecForwardsDurableDirMounts(t *testing.T) { {VolumeName: "data", MountPath: "/home/counter"}, {VolumeName: "cache", MountPath: "/var/cache"}, }, + CsiVolumeMounts: []*ateompb.VolumeMount{ + {VolumeName: "scratch", MountPath: "/scratch"}, + }, }, { Name: "sidecar", @@ -732,12 +738,70 @@ func TestBuildAteomWorkloadSpecForwardsDurableDirMounts(t *testing.T) { {Name: "no-volumes"}, }, } - got := buildAteomWorkloadSpec(in) + got, err := buildAteomWorkloadSpec(in) + if err != nil { + t.Fatalf("buildAteomWorkloadSpec failed: %v", err) + } if diff := cmp.Diff(want, got, protocmp.Transform()); diff != "" { t.Errorf("buildAteomWorkloadSpec mismatch (-want +got):\n%s", diff) } } +func TestBuildAteomWorkloadSpecValidation(t *testing.T) { + tests := []struct { + name string + in *ateletpb.WorkloadSpec + wantErr string + }{ + { + name: "missing volume definition", + in: &ateletpb.WorkloadSpec{ + Volumes: []*ateletpb.Volume{ + {Name: "data", Type: ateletpb.VolumeType_VOLUME_TYPE_DURABLE_DIR}, + }, + Containers: []*ateletpb.Container{ + { + Name: "ctr", + VolumeMounts: []*ateletpb.VolumeMount{ + {Name: "missing-vol", MountPath: "/data"}, + }, + }, + }, + }, + wantErr: `container "ctr" mounts volume "missing-vol" which is not defined in workload volumes`, + }, + { + name: "unsupported volume type", + in: &ateletpb.WorkloadSpec{ + Volumes: []*ateletpb.Volume{ + {Name: "data", Type: ateletpb.VolumeType_VOLUME_TYPE_UNSPECIFIED}, + }, + Containers: []*ateletpb.Container{ + { + Name: "ctr", + VolumeMounts: []*ateletpb.VolumeMount{ + {Name: "data", MountPath: "/data"}, + }, + }, + }, + }, + wantErr: `container "ctr" mounts volume "data" with unsupported type VOLUME_TYPE_UNSPECIFIED`, + }, + } + + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + _, err := buildAteomWorkloadSpec(tc.in) + if err == nil { + t.Fatal("expected error, got nil") + } + if got, want := err.Error(), tc.wantErr; !strings.Contains(got, want) { + t.Errorf("error mismatch:\nwant: %s\ngot: %s", want, got) + } + }) + } +} + func TestToAteomEgressGateway(t *testing.T) { if got := toAteomEgressGateway(nil); got != nil { t.Fatalf("toAteomEgressGateway(nil) = %v, want nil", got) diff --git a/cmd/ateom-microvm/checkpoint.go b/cmd/ateom-microvm/checkpoint.go index f55dfe2f9..a240bdafc 100644 --- a/cmd/ateom-microvm/checkpoint.go +++ b/cmd/ateom-microvm/checkpoint.go @@ -81,13 +81,15 @@ func (s *AteomService) CheckpointWorkload(ctx context.Context, req *ateompb.Chec // captures. DATA_ON_GOLDEN is restore-only (a DataOnGolden commit arrives // here as plain DATA) and lands in the default rejection. durable := hasDurableVolumes(req.GetSpec().GetContainers()) + csi := hasCsiVolumes(req.GetSpec().GetContainers()) scope := req.GetScope() switch scope { case ateompb.SnapshotScope_SNAPSHOT_SCOPE_FULL: case ateompb.SnapshotScope_SNAPSHOT_SCOPE_DATA: - if !durable { + // TODO: Revisit handling for CSI volumes since snapshots are currently quietly ignored. + if !durable && !csi { return nil, status.Error(codes.FailedPrecondition, - "no durable-dir volumes found for a Data-scope snapshot") + "no durable-dir or CSI volumes found for a Data-scope snapshot") } default: return nil, status.Errorf(codes.InvalidArgument, "unsupported snapshot scope: %v", scope) @@ -284,7 +286,7 @@ func (s *AteomService) teardownActor(ctx context.Context, id string, ra *running } // Kill the virtiofsds (after CH, their only client): the overlay RO lower's // and, when the actor has durable-dir volumes, the writable share's. - for _, cmd := range []*exec.Cmd{ra.vfsdCmd, ra.durableVfsdCmd} { + for _, cmd := range []*exec.Cmd{ra.vfsdCmd, ra.durableVfsdCmd, ra.csiVfsdCmd} { if cmd != nil && cmd.Process != nil { _ = cmd.Process.Kill() _, _ = cmd.Process.Wait() diff --git a/cmd/ateom-microvm/csi.go b/cmd/ateom-microvm/csi.go new file mode 100644 index 000000000..49c5974ef --- /dev/null +++ b/cmd/ateom-microvm/csi.go @@ -0,0 +1,84 @@ +//go:build linux + +// Copyright 2026 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package main + +import ( + "context" + "fmt" + "os" + "os/exec" + "path/filepath" + + "github.com/agent-substrate/substrate/cmd/ateom-microvm/internal/kata" + "github.com/agent-substrate/substrate/internal/ateompath" + "github.com/agent-substrate/substrate/internal/proto/ateompb" + specs "github.com/opencontainers/runtime-spec/specs-go" +) + +// hasCsiVolumes reports whether any container mounts a CSI volume. +func hasCsiVolumes(containers []*ateompb.Container) bool { + for _, c := range containers { + if len(c.GetCsiVolumeMounts()) > 0 { + return true + } + } + return false +} + +// csiMounts returns the OCI mounts that expose a container's CSI +// volumes at the paths it declared. Each source is that volume's directory +// inside the guest's CSI share, which the agent mounts at sandbox creation. +func csiMounts(mounts []*ateompb.VolumeMount) []specs.Mount { + out := make([]specs.Mount, 0, len(mounts)) + for _, m := range mounts { + out = append(out, specs.Mount{ + Destination: m.GetMountPath(), + Source: kata.GuestCsiVolumeDir(m.GetVolumeName()), + Type: "bind", + Options: []string{"rbind", "rw"}, + }) + } + return out +} + +func csiVirtiofsdLogPath(id string) string { + return filepath.Join(kata.VMDir(id), "virtiofsd-csi.log") +} + +// stageCsiShare starts the virtiofsd serving the actor's CSI volumes. +func (s *AteomService) stageCsiShare(ctx context.Context, rr resolvedRuntime, actorUID string) (*exec.Cmd, error) { + shared := ateompath.VolumesDir(actorUID) + if _, err := os.Stat(shared); err != nil { + return nil, fmt.Errorf("while checking CSI volumes dir %q: %w", shared, err) + } + log, err := os.OpenFile(csiVirtiofsdLogPath(actorUID), os.O_CREATE|os.O_WRONLY|os.O_TRUNC, 0o600) + if err != nil { + return nil, fmt.Errorf("while opening CSI virtiofsd log file: %w", err) + } + defer log.Close() + cmd, err := kata.StartVirtiofsd(ctx, kata.VirtiofsdOptions{ + Binary: rr.virtiofsd, + SocketPath: kata.CsiVirtiofsdSocketPath(actorUID), + SharedDir: shared, + Cache: "auto", + Log: log, + }) + if err != nil { + return nil, fmt.Errorf("while starting CSI virtiofsd: %w", err) + } + return cmd, nil +} diff --git a/cmd/ateom-microvm/csi_test.go b/cmd/ateom-microvm/csi_test.go new file mode 100644 index 000000000..cce26cbb2 --- /dev/null +++ b/cmd/ateom-microvm/csi_test.go @@ -0,0 +1,113 @@ +//go:build linux + +// Copyright 2026 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package main + +import ( + "testing" + + "github.com/agent-substrate/substrate/internal/proto/ateompb" + "github.com/google/go-cmp/cmp" + specs "github.com/opencontainers/runtime-spec/specs-go" +) + +func TestHasCsiVolumes(t *testing.T) { + tests := []struct { + name string + containers []*ateompb.Container + want bool + }{ + { + name: "empty containers", + containers: nil, + want: false, + }, + { + name: "no CSI volumes", + containers: []*ateompb.Container{ + { + Name: "c1", + DurableDirVolumeMounts: []*ateompb.DurableDirVolumeMount{ + {VolumeName: "data", MountPath: "/data"}, + }, + }, + }, + want: false, + }, + { + name: "has CSI volumes", + containers: []*ateompb.Container{ + { + Name: "c1", + CsiVolumeMounts: []*ateompb.VolumeMount{ + {VolumeName: "csi-vol", MountPath: "/csi"}, + }, + }, + }, + want: true, + }, + { + name: "multiple containers, one has CSI", + containers: []*ateompb.Container{ + { + Name: "c1", + }, + { + Name: "c2", + CsiVolumeMounts: []*ateompb.VolumeMount{ + {VolumeName: "csi-vol", MountPath: "/csi"}, + }, + }, + }, + want: true, + }, + } + + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + if got := hasCsiVolumes(tc.containers); got != tc.want { + t.Errorf("hasCsiVolumes() = %v, want %v", got, tc.want) + } + }) + } +} + +func TestCsiMounts(t *testing.T) { + mounts := []*ateompb.VolumeMount{ + {VolumeName: "vol1", MountPath: "/mnt/vol1"}, + {VolumeName: "vol2", MountPath: "/mnt/vol2"}, + } + + want := []specs.Mount{ + { + Destination: "/mnt/vol1", + Source: "/run/ateom-csi/vol1", + Type: "bind", + Options: []string{"rbind", "rw"}, + }, + { + Destination: "/mnt/vol2", + Source: "/run/ateom-csi/vol2", + Type: "bind", + Options: []string{"rbind", "rw"}, + }, + } + + got := csiMounts(mounts) + if diff := cmp.Diff(want, got); diff != "" { + t.Errorf("csiMounts() mismatch (-want +got):\n%s", diff) + } +} diff --git a/cmd/ateom-microvm/durable.go b/cmd/ateom-microvm/durable.go index a28a2186a..4d8ae3dd0 100644 --- a/cmd/ateom-microvm/durable.go +++ b/cmd/ateom-microvm/durable.go @@ -93,11 +93,16 @@ func durableMounts(mounts []*ateompb.DurableDirVolumeMount) []specs.Mount { // The spec is copied rather than mutated so the bundle's on-disk config.json and // the carrier's view stay as prepared — only the workload sees the binds. func workloadSpec(c actorContainer) *specs.Spec { - if len(c.durableMounts) == 0 { + if len(c.durableMounts) == 0 && len(c.csiMounts) == 0 { return c.spec } spec := *c.spec - spec.Mounts = append(append([]specs.Mount(nil), c.spec.Mounts...), durableMounts(c.durableMounts)...) + + var mounts []specs.Mount + mounts = append(mounts, c.spec.Mounts...) + mounts = append(mounts, durableMounts(c.durableMounts)...) + mounts = append(mounts, csiMounts(c.csiMounts)...) + spec.Mounts = mounts return &spec } diff --git a/cmd/ateom-microvm/internal/kata/overlay_linux.go b/cmd/ateom-microvm/internal/kata/overlay_linux.go index 741a1a446..5ae0ed3ea 100644 --- a/cmd/ateom-microvm/internal/kata/overlay_linux.go +++ b/cmd/ateom-microvm/internal/kata/overlay_linux.go @@ -54,6 +54,14 @@ const ( // volume's contents live at / and are bind-mounted // from there into the containers that declare the volume. guestDurableDir = "/run/ateom-durable" + + // CsiFsTag is the virtio-fs tag for the actor's WRITABLE CSI volumes share, + // served by a virtiofsd. + CsiFsTag = "ateCsi" + // guestCsiDir is where the agent mounts CsiFsTag in the guest; each + // volume's contents live at / and are bind-mounted + // from there into the containers that declare the volume. + guestCsiDir = "/run/ateom-csi" ) // GuestDurableVolumeDir is the in-guest path holding one durable volume's @@ -62,6 +70,12 @@ func GuestDurableVolumeDir(volumeName string) string { return guestDurableDir + "/" + volumeName } +// GuestCsiVolumeDir is the in-guest path holding one CSI volume's +// contents, i.e. the bind source for that volume's container mount points. +func GuestCsiVolumeDir(volumeName string) string { + return guestCsiDir + "/" + volumeName +} + // SharedDir is the host directory virtiofsd serves into the guest as the RO base. // Its layout (/rootfs) is what find-paths re-opens by path on restore. func SharedDir(id string) string { @@ -213,19 +227,23 @@ func ReconstructSharedDirFromImage(ctx context.Context, bundleRootfs, restoreID, return nil } +type CreateSandboxOpts struct { + SandboxID string + Hostname string + WithDurableShare bool + WithCsiShare bool +} + // CreateSandboxForActor creates the guest sandbox with the kataShared virtio-fs mount // (the RO base backing every container's rootfs). Mirrors kata startSandbox. -// -// withDurableShare additionally mounts the writable durable-dir share, whose -// per-volume subdirectories the containers bind-mount at their declared paths. -func (a *AgentClient) CreateSandboxForActor(ctx context.Context, sandboxID, hostname string, withDurableShare bool) error { +func (a *AgentClient) CreateSandboxForActor(ctx context.Context, opts CreateSandboxOpts) error { storages := []*agentpb.Storage{{ Driver: virtioFSDriver, Source: FsTag, Fstype: typeVirtioFS, MountPoint: guestSharedDir, }} - if withDurableShare { + if opts.WithDurableShare { storages = append(storages, &agentpb.Storage{ Driver: virtioFSDriver, Source: DurableFsTag, @@ -233,9 +251,17 @@ func (a *AgentClient) CreateSandboxForActor(ctx context.Context, sandboxID, host MountPoint: guestDurableDir, }) } + if opts.WithCsiShare { + storages = append(storages, &agentpb.Storage{ + Driver: virtioFSDriver, + Source: CsiFsTag, + Fstype: typeVirtioFS, + MountPoint: guestCsiDir, + }) + } return a.CreateSandbox(ctx, &agentpb.CreateSandboxRequest{ - Hostname: hostname, - SandboxId: sandboxID, + Hostname: opts.Hostname, + SandboxId: opts.SandboxID, Storages: storages, }) } diff --git a/cmd/ateom-microvm/internal/kata/restore.go b/cmd/ateom-microvm/internal/kata/restore.go index 0b71bea9d..940bf96ce 100644 --- a/cmd/ateom-microvm/internal/kata/restore.go +++ b/cmd/ateom-microvm/internal/kata/restore.go @@ -35,3 +35,9 @@ func VsockSocketPath(id string) string { return filepath.Join(VMDir(id), "clh.so func DurableVirtiofsdSocketPath(id string) string { return filepath.Join(VMDir(id), "virtiofsd-durable.sock") } + +// CsiVirtiofsdSocketPath is the vhost-user-fs socket for the actor's writable +// CSI volumes share, served by a third virtiofsd (or second if no durableDir is present). +func CsiVirtiofsdSocketPath(id string) string { + return filepath.Join(VMDir(id), "virtiofsd-csi.sock") +} diff --git a/cmd/ateom-microvm/restore.go b/cmd/ateom-microvm/restore.go index 93d6518a9..6efab67a0 100644 --- a/cmd/ateom-microvm/restore.go +++ b/cmd/ateom-microvm/restore.go @@ -212,6 +212,19 @@ func (s *AteomService) restoreFullScope(ctx context.Context, p actorBootParams, }() } + var csiVfsdCmd *exec.Cmd + if hasCsiVolumes(containers) { + if csiVfsdCmd, err = s.stageCsiShare(ctx, rr, actorUID); err != nil { + return err + } + defer func() { + if retErr != nil && csiVfsdCmd.Process != nil { + _ = csiVfsdCmd.Process.Kill() + _, _ = csiVfsdCmd.Process.Wait() + } + }() + } + // Networking: rebuild the per-activation veth + tap; the snapshot's virtio-net // is fd-backed, so CH needs fresh tap FDs (net_fds) on restore. if err := ateomnet.SetupActorNetwork(ctx, ateomnet.NetworkConfig{ @@ -297,7 +310,7 @@ func (s *AteomService) restoreFullScope(ctx context.Context, p actorBootParams, } ra := &runningActor{ - chCmd: chCmd, vfsdCmd: vfsdCmd, durableVfsdCmd: durableVfsdCmd, + chCmd: chCmd, vfsdCmd: vfsdCmd, durableVfsdCmd: durableVfsdCmd, csiVfsdCmd: csiVfsdCmd, apiSocket: apiSocket, baseID: srcID, restoreSourceDir: restoreDir, } @@ -371,6 +384,8 @@ func rewriteSnapshotSocketPaths(snapshotDir, id string) error { fm["socket"] = kata.VirtiofsdSocketPath(id) case kata.DurableFsTag: fm["socket"] = kata.DurableVirtiofsdSocketPath(id) + case kata.CsiFsTag: + fm["socket"] = kata.CsiVirtiofsdSocketPath(id) default: return fmt.Errorf("snapshot config %q has fs device with unknown tag %q", cfgPath, tag) } diff --git a/cmd/ateom-microvm/run.go b/cmd/ateom-microvm/run.go index eb4b34740..610c76b3d 100644 --- a/cmd/ateom-microvm/run.go +++ b/cmd/ateom-microvm/run.go @@ -66,6 +66,10 @@ type runningActor struct { // durable-dir volumes. nil when the actor declares none. Owned and torn down // exactly like vfsdCmd. durableVfsdCmd *exec.Cmd + // csiVfsdCmd is a virtiofsd, serving the actor's writable + // CSI volumes. nil when the actor declares none. Owned and torn down + // exactly like vfsdCmd. + csiVfsdCmd *exec.Cmd // apiSocket is the CH api-socket for this ateom-owned VMM. apiSocket string @@ -133,6 +137,9 @@ type actorContainer struct { // durableMounts are the durable-dir volumes this container mounts, and where // (see durable.go). Empty for containers that declare none. durableMounts []*ateompb.DurableDirVolumeMount + // csiMounts are the CSI volumes this container mounts, and where (see csi.go). + // Empty for containers that declare none. + csiMounts []*ateompb.VolumeMount } // resolvedRuntime holds the concrete binary/config paths for a request, taken @@ -403,6 +410,23 @@ func (s *AteomService) coldBootActor(ctx context.Context, p actorBootParams) (re }() } + // CSI volumes (if any) share one writable virtio-fs share, served by + // a virtiofsd from the host directory atelet; each volume is a + // subdirectory of it. + csi := hasCsiVolumes(containers) + var csiVfsdCmd *exec.Cmd + if csi { + if csiVfsdCmd, err = s.stageCsiShare(ctx, rr, actorUID); err != nil { + return err + } + defer func() { + if retErr != nil && csiVfsdCmd.Process != nil { + _ = csiVfsdCmd.Process.Kill() + _, _ = csiVfsdCmd.Process.Wait() + } + }() + } + // Launch a bare VMM (CH + api-socket); ateom owns this process for teardown. apiSocket := filepath.Join(kata.VMDir(actorUID), "clh-api.sock") chCmd, client, err := ch.LaunchVMM(ctx, ch.LaunchVMMOptions{ @@ -426,7 +450,7 @@ func (s *AteomService) coldBootActor(ctx context.Context, p actorBootParams) (re // writable upper is a guest tmpfs). serialLog is also read on a failed agent dial // below, so keep it here. serialLog := filepath.Join(kata.VMDir(actorUID), "serial.log") - vmCfg := buildVMConfig(actorUID, kernel, image, kparams, serialLog, memMiB, vcpus, durable) + vmCfg := buildVMConfig(actorUID, kernel, image, kparams, serialLog, memMiB, vcpus, durable, csi) if err := client.CreateVM(ctx, vmCfg); err != nil { return fmt.Errorf("while creating VM: %w", err) } @@ -480,7 +504,7 @@ func (s *AteomService) coldBootActor(ctx context.Context, p actorBootParams) (re }() // Post-boot kata-agent setup: sandbox, guest networking, start each container. - if err := s.startActorContainers(ctx, ac, actorUID, vsockPath, ctrs, durable); err != nil { + if err := s.startActorContainers(ctx, ac, actorUID, vsockPath, ctrs, durable, csi); err != nil { return err } @@ -489,7 +513,7 @@ func (s *AteomService) coldBootActor(ctx context.Context, p actorBootParams) (re return fmt.Errorf("while waiting for container readyz: %w", err) } - ra := &runningActor{chCmd: chCmd, vfsdCmd: vfsdCmd, durableVfsdCmd: durableVfsdCmd, apiSocket: apiSocket, baseID: actorUID, logAgent: ac} + ra := &runningActor{chCmd: chCmd, vfsdCmd: vfsdCmd, durableVfsdCmd: durableVfsdCmd, csiVfsdCmd: csiVfsdCmd, apiSocket: apiSocket, baseID: actorUID, logAgent: ac} if err := s.activateActorNetworking(p.actorRef.Atespace, p.actorRef.Name, egress); err != nil { return err } @@ -544,6 +568,7 @@ func (s *AteomService) buildActorContainers(actorUID string, containers []*ateom bundleRootfs: bundleRootfs, spec: spec, durableMounts: c.GetDurableDirVolumeMounts(), + csiMounts: c.GetCsiVolumeMounts(), } } return ctrs, nil @@ -602,7 +627,7 @@ func (s *AteomService) guestConfig(rr resolvedRuntime) (memMiB, vcpus int, kpara // // withDurable adds a second virtio-fs device for the actor's writable durable-dir // volumes (see durable.go), served by its own virtiofsd on the same PCI segment. -func buildVMConfig(id, kernel, image, kparams, serialLog string, memMiB, vcpus int, withDurable bool) ch.VmConfig { +func buildVMConfig(id, kernel, image, kparams, serialLog string, memMiB, vcpus int, withDurable bool, withCsi bool) ch.VmConfig { console := "ttyS0" if runtime.GOARCH == "arm64" { console = "ttyAMA0" @@ -620,7 +645,7 @@ func buildVMConfig(id, kernel, image, kparams, serialLog string, memMiB, vcpus i Disks: []ch.DiskConfig{ {Path: image, Readonly: true, ImageType: "Raw", NumQueues: int32(vcpus), QueueSize: 1024}, }, - Fs: buildFsConfigs(id, withDurable), + Fs: buildFsConfigs(id, withDurable, withCsi), Platform: &ch.PlatformConfig{NumPciSegments: 2}, Rng: &ch.RngConfig{Src: "/dev/urandom"}, Serial: &ch.ConsoleConfig{Mode: "File", File: serialLog}, @@ -631,7 +656,7 @@ func buildVMConfig(id, kernel, image, kparams, serialLog string, memMiB, vcpus i // buildFsConfigs returns the VM's virtio-fs devices: the overlay RO lower's // share, plus the writable durable-dir share when the actor has one. Both sit on // PCI segment 1 (the segment buildVMConfig reserves for virtio-fs). -func buildFsConfigs(id string, withDurable bool) []ch.FsConfig { +func buildFsConfigs(id string, withDurable bool, withCsi bool) []ch.FsConfig { fs := []ch.FsConfig{{ Tag: kata.FsTag, Socket: kata.VirtiofsdSocketPath(id), NumQueues: 1, QueueSize: 1024, PciSegment: 1, @@ -642,6 +667,12 @@ func buildFsConfigs(id string, withDurable bool) []ch.FsConfig { NumQueues: 1, QueueSize: 1024, PciSegment: 1, }) } + if withCsi { + fs = append(fs, ch.FsConfig{ + Tag: kata.CsiFsTag, Socket: kata.CsiVirtiofsdSocketPath(id), + NumQueues: 1, QueueSize: 1024, PciSegment: 1, + }) + } return fs } @@ -652,12 +683,17 @@ func buildFsConfigs(id string, withDurable bool) []ch.FsConfig { // // durable says the actor has durable-dir volumes: the sandbox then also mounts // the writable durable share, and each container binds the volumes it declared. -func (s *AteomService) startActorContainers(ctx context.Context, ac *kata.AgentClient, id, vsockPath string, ctrs []actorContainer, durable bool) error { +func (s *AteomService) startActorContainers(ctx context.Context, ac *kata.AgentClient, id, vsockPath string, ctrs []actorContainer, durable bool, csi bool) error { // Establish the agent sandbox + the kataShared virtio-fs mount (the RO base for // every container's overlay lower). All containers share it, so use the first // container's hostname. sbCtx, sbCancel := context.WithTimeout(ctx, 20*time.Second) - err := ac.CreateSandboxForActor(sbCtx, id, ctrs[0].spec.Hostname, durable) + err := ac.CreateSandboxForActor(sbCtx, kata.CreateSandboxOpts{ + SandboxID: id, + Hostname: ctrs[0].spec.Hostname, + WithDurableShare: durable, + WithCsiShare: csi, + }) sbCancel() if err != nil { return fmt.Errorf("while creating agent sandbox: %w", err) diff --git a/demos/counter/counter-microvm-csi-test.yaml b/demos/counter/counter-microvm-csi-test.yaml new file mode 100644 index 000000000..c2ac98b34 --- /dev/null +++ b/demos/counter/counter-microvm-csi-test.yaml @@ -0,0 +1,67 @@ +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +apiVersion: v1 +kind: Namespace +metadata: + name: ate-demo-counter-microvm-csi + +--- + +apiVersion: ate.dev/v1alpha1 +kind: WorkerPool +metadata: + name: counter-microvm-csi + namespace: ate-demo-counter-microvm-csi + labels: + workload: counter-microvm-csi +spec: + replicas: 1 + sandboxClass: microvm + sandboxConfigName: microvm + ateomImage: ko://github.com/agent-substrate/substrate/cmd/ateom-microvm + +--- + +apiVersion: ate.dev/v1alpha1 +kind: ActorTemplate +metadata: + name: counter-microvm-csi + namespace: ate-demo-counter-microvm-csi +spec: + sandboxClass: microvm + containers: + - name: counter + image: ko://github.com/agent-substrate/substrate/demos/counter + readyz: + httpGet: + path: /readyz + port: 80 + volumeMounts: + - name: data + mountPath: /home/counter + workerSelector: + matchLabels: + workload: counter-microvm-csi + snapshotsConfig: + onPause: Full + onCommit: Data + onResume: + fromData: Golden + location: gs://ate-snapshots/ate-demo-counter-microvm-csi/ + volumes: + - name: data + externalVolumeTemplate: + capacity: 1Gi + storageClassName: csi-hostpath-sc diff --git a/internal/e2e/suites/demo/demo_test.go b/internal/e2e/suites/demo/demo_test.go index 8d1e1ad68..5afe2975e 100644 --- a/internal/e2e/suites/demo/demo_test.go +++ b/internal/e2e/suites/demo/demo_test.go @@ -399,9 +399,6 @@ func TestMultipleDurableDirLifecycle(t *testing.T) { } func TestExternalVolumeLifecycle(t *testing.T) { - if isMicroVMEnvironment() { - t.Skip("Skipping TestExternalVolumeLifecycle for microVM environment") - } tests := []struct { name string diff --git a/internal/proto/ateompb/ateom.pb.go b/internal/proto/ateompb/ateom.pb.go index cfe6b3ed5..231d1ad73 100644 --- a/internal/proto/ateompb/ateom.pb.go +++ b/internal/proto/ateompb/ateom.pb.go @@ -419,8 +419,10 @@ type Container struct { // durable_dir_volume_mounts are the durable-dir volumes this container // mounts, if any. DurableDirVolumeMounts []*DurableDirVolumeMount `protobuf:"bytes,4,rep,name=durable_dir_volume_mounts,json=durableDirVolumeMounts,proto3" json:"durable_dir_volume_mounts,omitempty"` - unknownFields protoimpl.UnknownFields - sizeCache protoimpl.SizeCache + // csi_volume_mounts are the CSI volumes this container mounts, if any. + CsiVolumeMounts []*VolumeMount `protobuf:"bytes,5,rep,name=csi_volume_mounts,json=csiVolumeMounts,proto3" json:"csi_volume_mounts,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache } func (x *Container) Reset() { @@ -474,6 +476,66 @@ func (x *Container) GetDurableDirVolumeMounts() []*DurableDirVolumeMount { return nil } +func (x *Container) GetCsiVolumeMounts() []*VolumeMount { + if x != nil { + return x.CsiVolumeMounts + } + return nil +} + +// VolumeMount is one volume mounted into a container. +type VolumeMount struct { + state protoimpl.MessageState `protogen:"open.v1"` + VolumeName string `protobuf:"bytes,1,opt,name=volume_name,json=volumeName,proto3" json:"volume_name,omitempty"` + MountPath string `protobuf:"bytes,2,opt,name=mount_path,json=mountPath,proto3" json:"mount_path,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *VolumeMount) Reset() { + *x = VolumeMount{} + mi := &file_ateom_proto_msgTypes[4] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *VolumeMount) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*VolumeMount) ProtoMessage() {} + +func (x *VolumeMount) ProtoReflect() protoreflect.Message { + mi := &file_ateom_proto_msgTypes[4] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use VolumeMount.ProtoReflect.Descriptor instead. +func (*VolumeMount) Descriptor() ([]byte, []int) { + return file_ateom_proto_rawDescGZIP(), []int{4} +} + +func (x *VolumeMount) GetVolumeName() string { + if x != nil { + return x.VolumeName + } + return "" +} + +func (x *VolumeMount) GetMountPath() string { + if x != nil { + return x.MountPath + } + return "" +} + // DurableDirVolumeMount is one durable-dir volume mounted into a container. type DurableDirVolumeMount struct { state protoimpl.MessageState `protogen:"open.v1"` @@ -488,7 +550,7 @@ type DurableDirVolumeMount struct { func (x *DurableDirVolumeMount) Reset() { *x = DurableDirVolumeMount{} - mi := &file_ateom_proto_msgTypes[4] + mi := &file_ateom_proto_msgTypes[5] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -500,7 +562,7 @@ func (x *DurableDirVolumeMount) String() string { func (*DurableDirVolumeMount) ProtoMessage() {} func (x *DurableDirVolumeMount) ProtoReflect() protoreflect.Message { - mi := &file_ateom_proto_msgTypes[4] + mi := &file_ateom_proto_msgTypes[5] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -513,7 +575,7 @@ func (x *DurableDirVolumeMount) ProtoReflect() protoreflect.Message { // Deprecated: Use DurableDirVolumeMount.ProtoReflect.Descriptor instead. func (*DurableDirVolumeMount) Descriptor() ([]byte, []int) { - return file_ateom_proto_rawDescGZIP(), []int{4} + return file_ateom_proto_rawDescGZIP(), []int{5} } func (x *DurableDirVolumeMount) GetVolumeName() string { @@ -544,7 +606,7 @@ type Readyz struct { func (x *Readyz) Reset() { *x = Readyz{} - mi := &file_ateom_proto_msgTypes[5] + mi := &file_ateom_proto_msgTypes[6] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -556,7 +618,7 @@ func (x *Readyz) String() string { func (*Readyz) ProtoMessage() {} func (x *Readyz) ProtoReflect() protoreflect.Message { - mi := &file_ateom_proto_msgTypes[5] + mi := &file_ateom_proto_msgTypes[6] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -569,7 +631,7 @@ func (x *Readyz) ProtoReflect() protoreflect.Message { // Deprecated: Use Readyz.ProtoReflect.Descriptor instead. func (*Readyz) Descriptor() ([]byte, []int) { - return file_ateom_proto_rawDescGZIP(), []int{5} + return file_ateom_proto_rawDescGZIP(), []int{6} } func (x *Readyz) GetHttpGet() *HTTPGetAction { @@ -599,7 +661,7 @@ type HTTPGetAction struct { func (x *HTTPGetAction) Reset() { *x = HTTPGetAction{} - mi := &file_ateom_proto_msgTypes[6] + mi := &file_ateom_proto_msgTypes[7] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -611,7 +673,7 @@ func (x *HTTPGetAction) String() string { func (*HTTPGetAction) ProtoMessage() {} func (x *HTTPGetAction) ProtoReflect() protoreflect.Message { - mi := &file_ateom_proto_msgTypes[6] + mi := &file_ateom_proto_msgTypes[7] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -624,7 +686,7 @@ func (x *HTTPGetAction) ProtoReflect() protoreflect.Message { // Deprecated: Use HTTPGetAction.ProtoReflect.Descriptor instead. func (*HTTPGetAction) Descriptor() ([]byte, []int) { - return file_ateom_proto_rawDescGZIP(), []int{6} + return file_ateom_proto_rawDescGZIP(), []int{7} } func (x *HTTPGetAction) GetPath() string { @@ -649,7 +711,7 @@ type RunWorkloadResponse struct { func (x *RunWorkloadResponse) Reset() { *x = RunWorkloadResponse{} - mi := &file_ateom_proto_msgTypes[7] + mi := &file_ateom_proto_msgTypes[8] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -661,7 +723,7 @@ func (x *RunWorkloadResponse) String() string { func (*RunWorkloadResponse) ProtoMessage() {} func (x *RunWorkloadResponse) ProtoReflect() protoreflect.Message { - mi := &file_ateom_proto_msgTypes[7] + mi := &file_ateom_proto_msgTypes[8] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -674,7 +736,7 @@ func (x *RunWorkloadResponse) ProtoReflect() protoreflect.Message { // Deprecated: Use RunWorkloadResponse.ProtoReflect.Descriptor instead. func (*RunWorkloadResponse) Descriptor() ([]byte, []int) { - return file_ateom_proto_rawDescGZIP(), []int{7} + return file_ateom_proto_rawDescGZIP(), []int{8} } type CheckpointWorkloadRequest struct { @@ -708,7 +770,7 @@ type CheckpointWorkloadRequest struct { func (x *CheckpointWorkloadRequest) Reset() { *x = CheckpointWorkloadRequest{} - mi := &file_ateom_proto_msgTypes[8] + mi := &file_ateom_proto_msgTypes[9] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -720,7 +782,7 @@ func (x *CheckpointWorkloadRequest) String() string { func (*CheckpointWorkloadRequest) ProtoMessage() {} func (x *CheckpointWorkloadRequest) ProtoReflect() protoreflect.Message { - mi := &file_ateom_proto_msgTypes[8] + mi := &file_ateom_proto_msgTypes[9] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -733,7 +795,7 @@ func (x *CheckpointWorkloadRequest) ProtoReflect() protoreflect.Message { // Deprecated: Use CheckpointWorkloadRequest.ProtoReflect.Descriptor instead. func (*CheckpointWorkloadRequest) Descriptor() ([]byte, []int) { - return file_ateom_proto_rawDescGZIP(), []int{8} + return file_ateom_proto_rawDescGZIP(), []int{9} } func (x *CheckpointWorkloadRequest) GetAtespace() string { @@ -818,7 +880,7 @@ type CheckpointWorkloadResponse struct { func (x *CheckpointWorkloadResponse) Reset() { *x = CheckpointWorkloadResponse{} - mi := &file_ateom_proto_msgTypes[9] + mi := &file_ateom_proto_msgTypes[10] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -830,7 +892,7 @@ func (x *CheckpointWorkloadResponse) String() string { func (*CheckpointWorkloadResponse) ProtoMessage() {} func (x *CheckpointWorkloadResponse) ProtoReflect() protoreflect.Message { - mi := &file_ateom_proto_msgTypes[9] + mi := &file_ateom_proto_msgTypes[10] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -843,7 +905,7 @@ func (x *CheckpointWorkloadResponse) ProtoReflect() protoreflect.Message { // Deprecated: Use CheckpointWorkloadResponse.ProtoReflect.Descriptor instead. func (*CheckpointWorkloadResponse) Descriptor() ([]byte, []int) { - return file_ateom_proto_rawDescGZIP(), []int{9} + return file_ateom_proto_rawDescGZIP(), []int{10} } func (x *CheckpointWorkloadResponse) GetSnapshotFiles() []string { @@ -882,7 +944,7 @@ type RestoreWorkloadRequest struct { func (x *RestoreWorkloadRequest) Reset() { *x = RestoreWorkloadRequest{} - mi := &file_ateom_proto_msgTypes[10] + mi := &file_ateom_proto_msgTypes[11] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -894,7 +956,7 @@ func (x *RestoreWorkloadRequest) String() string { func (*RestoreWorkloadRequest) ProtoMessage() {} func (x *RestoreWorkloadRequest) ProtoReflect() protoreflect.Message { - mi := &file_ateom_proto_msgTypes[10] + mi := &file_ateom_proto_msgTypes[11] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -907,7 +969,7 @@ func (x *RestoreWorkloadRequest) ProtoReflect() protoreflect.Message { // Deprecated: Use RestoreWorkloadRequest.ProtoReflect.Descriptor instead. func (*RestoreWorkloadRequest) Descriptor() ([]byte, []int) { - return file_ateom_proto_rawDescGZIP(), []int{10} + return file_ateom_proto_rawDescGZIP(), []int{11} } func (x *RestoreWorkloadRequest) GetAtespace() string { @@ -1002,7 +1064,7 @@ type RestoreWorkloadResponse struct { func (x *RestoreWorkloadResponse) Reset() { *x = RestoreWorkloadResponse{} - mi := &file_ateom_proto_msgTypes[11] + mi := &file_ateom_proto_msgTypes[12] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -1014,7 +1076,7 @@ func (x *RestoreWorkloadResponse) String() string { func (*RestoreWorkloadResponse) ProtoMessage() {} func (x *RestoreWorkloadResponse) ProtoReflect() protoreflect.Message { - mi := &file_ateom_proto_msgTypes[11] + mi := &file_ateom_proto_msgTypes[12] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -1027,7 +1089,7 @@ func (x *RestoreWorkloadResponse) ProtoReflect() protoreflect.Message { // Deprecated: Use RestoreWorkloadResponse.ProtoReflect.Descriptor instead. func (*RestoreWorkloadResponse) Descriptor() ([]byte, []int) { - return file_ateom_proto_rawDescGZIP(), []int{11} + return file_ateom_proto_rawDescGZIP(), []int{12} } type GetWorkloadStatsRequest struct { @@ -1043,7 +1105,7 @@ type GetWorkloadStatsRequest struct { func (x *GetWorkloadStatsRequest) Reset() { *x = GetWorkloadStatsRequest{} - mi := &file_ateom_proto_msgTypes[12] + mi := &file_ateom_proto_msgTypes[13] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -1055,7 +1117,7 @@ func (x *GetWorkloadStatsRequest) String() string { func (*GetWorkloadStatsRequest) ProtoMessage() {} func (x *GetWorkloadStatsRequest) ProtoReflect() protoreflect.Message { - mi := &file_ateom_proto_msgTypes[12] + mi := &file_ateom_proto_msgTypes[13] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -1068,7 +1130,7 @@ func (x *GetWorkloadStatsRequest) ProtoReflect() protoreflect.Message { // Deprecated: Use GetWorkloadStatsRequest.ProtoReflect.Descriptor instead. func (*GetWorkloadStatsRequest) Descriptor() ([]byte, []int) { - return file_ateom_proto_rawDescGZIP(), []int{12} + return file_ateom_proto_rawDescGZIP(), []int{13} } func (x *GetWorkloadStatsRequest) GetActorUid() string { @@ -1129,7 +1191,7 @@ type GetWorkloadStatsResponse struct { func (x *GetWorkloadStatsResponse) Reset() { *x = GetWorkloadStatsResponse{} - mi := &file_ateom_proto_msgTypes[13] + mi := &file_ateom_proto_msgTypes[14] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -1141,7 +1203,7 @@ func (x *GetWorkloadStatsResponse) String() string { func (*GetWorkloadStatsResponse) ProtoMessage() {} func (x *GetWorkloadStatsResponse) ProtoReflect() protoreflect.Message { - mi := &file_ateom_proto_msgTypes[13] + mi := &file_ateom_proto_msgTypes[14] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -1154,7 +1216,7 @@ func (x *GetWorkloadStatsResponse) ProtoReflect() protoreflect.Message { // Deprecated: Use GetWorkloadStatsResponse.ProtoReflect.Descriptor instead. func (*GetWorkloadStatsResponse) Descriptor() ([]byte, []int) { - return file_ateom_proto_rawDescGZIP(), []int{13} + return file_ateom_proto_rawDescGZIP(), []int{14} } func (x *GetWorkloadStatsResponse) GetAtespace() string { @@ -1268,11 +1330,17 @@ const file_ateom_proto_rawDesc = "" + "\fWorkloadSpec\x120\n" + "\n" + "containers\x18\x01 \x03(\v2\x10.ateom.ContainerR\n" + - "containers\"\xba\x01\n" + + "containers\"\xfa\x01\n" + "\tContainer\x12\x12\n" + "\x04name\x18\x01 \x01(\tR\x04name\x12%\n" + "\x06readyz\x18\x02 \x01(\v2\r.ateom.ReadyzR\x06readyz\x12W\n" + - "\x19durable_dir_volume_mounts\x18\x04 \x03(\v2\x1c.ateom.DurableDirVolumeMountR\x16durableDirVolumeMountsJ\x04\b\x03\x10\x04R\x13durable_dir_volumes\"W\n" + + "\x19durable_dir_volume_mounts\x18\x04 \x03(\v2\x1c.ateom.DurableDirVolumeMountR\x16durableDirVolumeMounts\x12>\n" + + "\x11csi_volume_mounts\x18\x05 \x03(\v2\x12.ateom.VolumeMountR\x0fcsiVolumeMountsJ\x04\b\x03\x10\x04R\x13durable_dir_volumes\"M\n" + + "\vVolumeMount\x12\x1f\n" + + "\vvolume_name\x18\x01 \x01(\tR\n" + + "volumeName\x12\x1d\n" + + "\n" + + "mount_path\x18\x02 \x01(\tR\tmountPath\"W\n" + "\x15DurableDirVolumeMount\x12\x1f\n" + "\vvolume_name\x18\x01 \x01(\tR\n" + "volumeName\x12\x1d\n" + @@ -1374,7 +1442,7 @@ func file_ateom_proto_rawDescGZIP() []byte { } var file_ateom_proto_enumTypes = make([]protoimpl.EnumInfo, 3) -var file_ateom_proto_msgTypes = make([]protoimpl.MessageInfo, 17) +var file_ateom_proto_msgTypes = make([]protoimpl.MessageInfo, 18) var file_ateom_proto_goTypes = []any{ (SnapshotScope)(0), // 0: ateom.SnapshotScope (SandboxClass)(0), // 1: ateom.SandboxClass @@ -1383,50 +1451,52 @@ var file_ateom_proto_goTypes = []any{ (*EgressGateway)(nil), // 4: ateom.EgressGateway (*WorkloadSpec)(nil), // 5: ateom.WorkloadSpec (*Container)(nil), // 6: ateom.Container - (*DurableDirVolumeMount)(nil), // 7: ateom.DurableDirVolumeMount - (*Readyz)(nil), // 8: ateom.Readyz - (*HTTPGetAction)(nil), // 9: ateom.HTTPGetAction - (*RunWorkloadResponse)(nil), // 10: ateom.RunWorkloadResponse - (*CheckpointWorkloadRequest)(nil), // 11: ateom.CheckpointWorkloadRequest - (*CheckpointWorkloadResponse)(nil), // 12: ateom.CheckpointWorkloadResponse - (*RestoreWorkloadRequest)(nil), // 13: ateom.RestoreWorkloadRequest - (*RestoreWorkloadResponse)(nil), // 14: ateom.RestoreWorkloadResponse - (*GetWorkloadStatsRequest)(nil), // 15: ateom.GetWorkloadStatsRequest - (*GetWorkloadStatsResponse)(nil), // 16: ateom.GetWorkloadStatsResponse - nil, // 17: ateom.RunWorkloadRequest.RuntimeAssetPathsEntry - nil, // 18: ateom.CheckpointWorkloadRequest.RuntimeAssetPathsEntry - nil, // 19: ateom.RestoreWorkloadRequest.RuntimeAssetPathsEntry + (*VolumeMount)(nil), // 7: ateom.VolumeMount + (*DurableDirVolumeMount)(nil), // 8: ateom.DurableDirVolumeMount + (*Readyz)(nil), // 9: ateom.Readyz + (*HTTPGetAction)(nil), // 10: ateom.HTTPGetAction + (*RunWorkloadResponse)(nil), // 11: ateom.RunWorkloadResponse + (*CheckpointWorkloadRequest)(nil), // 12: ateom.CheckpointWorkloadRequest + (*CheckpointWorkloadResponse)(nil), // 13: ateom.CheckpointWorkloadResponse + (*RestoreWorkloadRequest)(nil), // 14: ateom.RestoreWorkloadRequest + (*RestoreWorkloadResponse)(nil), // 15: ateom.RestoreWorkloadResponse + (*GetWorkloadStatsRequest)(nil), // 16: ateom.GetWorkloadStatsRequest + (*GetWorkloadStatsResponse)(nil), // 17: ateom.GetWorkloadStatsResponse + nil, // 18: ateom.RunWorkloadRequest.RuntimeAssetPathsEntry + nil, // 19: ateom.CheckpointWorkloadRequest.RuntimeAssetPathsEntry + nil, // 20: ateom.RestoreWorkloadRequest.RuntimeAssetPathsEntry } var file_ateom_proto_depIdxs = []int32{ 5, // 0: ateom.RunWorkloadRequest.spec:type_name -> ateom.WorkloadSpec - 17, // 1: ateom.RunWorkloadRequest.runtime_asset_paths:type_name -> ateom.RunWorkloadRequest.RuntimeAssetPathsEntry + 18, // 1: ateom.RunWorkloadRequest.runtime_asset_paths:type_name -> ateom.RunWorkloadRequest.RuntimeAssetPathsEntry 4, // 2: ateom.RunWorkloadRequest.egress_gateway:type_name -> ateom.EgressGateway 6, // 3: ateom.WorkloadSpec.containers:type_name -> ateom.Container - 8, // 4: ateom.Container.readyz:type_name -> ateom.Readyz - 7, // 5: ateom.Container.durable_dir_volume_mounts:type_name -> ateom.DurableDirVolumeMount - 9, // 6: ateom.Readyz.http_get:type_name -> ateom.HTTPGetAction - 5, // 7: ateom.CheckpointWorkloadRequest.spec:type_name -> ateom.WorkloadSpec - 18, // 8: ateom.CheckpointWorkloadRequest.runtime_asset_paths:type_name -> ateom.CheckpointWorkloadRequest.RuntimeAssetPathsEntry - 0, // 9: ateom.CheckpointWorkloadRequest.scope:type_name -> ateom.SnapshotScope - 5, // 10: ateom.RestoreWorkloadRequest.spec:type_name -> ateom.WorkloadSpec - 19, // 11: ateom.RestoreWorkloadRequest.runtime_asset_paths:type_name -> ateom.RestoreWorkloadRequest.RuntimeAssetPathsEntry - 0, // 12: ateom.RestoreWorkloadRequest.scope:type_name -> ateom.SnapshotScope - 4, // 13: ateom.RestoreWorkloadRequest.egress_gateway:type_name -> ateom.EgressGateway - 1, // 14: ateom.GetWorkloadStatsResponse.sandbox_class:type_name -> ateom.SandboxClass - 2, // 15: ateom.GetWorkloadStatsResponse.source:type_name -> ateom.StatsSource - 3, // 16: ateom.Ateom.RunWorkload:input_type -> ateom.RunWorkloadRequest - 11, // 17: ateom.Ateom.CheckpointWorkload:input_type -> ateom.CheckpointWorkloadRequest - 13, // 18: ateom.Ateom.RestoreWorkload:input_type -> ateom.RestoreWorkloadRequest - 15, // 19: ateom.Ateom.GetWorkloadStats:input_type -> ateom.GetWorkloadStatsRequest - 10, // 20: ateom.Ateom.RunWorkload:output_type -> ateom.RunWorkloadResponse - 12, // 21: ateom.Ateom.CheckpointWorkload:output_type -> ateom.CheckpointWorkloadResponse - 14, // 22: ateom.Ateom.RestoreWorkload:output_type -> ateom.RestoreWorkloadResponse - 16, // 23: ateom.Ateom.GetWorkloadStats:output_type -> ateom.GetWorkloadStatsResponse - 20, // [20:24] is the sub-list for method output_type - 16, // [16:20] is the sub-list for method input_type - 16, // [16:16] is the sub-list for extension type_name - 16, // [16:16] is the sub-list for extension extendee - 0, // [0:16] is the sub-list for field type_name + 9, // 4: ateom.Container.readyz:type_name -> ateom.Readyz + 8, // 5: ateom.Container.durable_dir_volume_mounts:type_name -> ateom.DurableDirVolumeMount + 7, // 6: ateom.Container.csi_volume_mounts:type_name -> ateom.VolumeMount + 10, // 7: ateom.Readyz.http_get:type_name -> ateom.HTTPGetAction + 5, // 8: ateom.CheckpointWorkloadRequest.spec:type_name -> ateom.WorkloadSpec + 19, // 9: ateom.CheckpointWorkloadRequest.runtime_asset_paths:type_name -> ateom.CheckpointWorkloadRequest.RuntimeAssetPathsEntry + 0, // 10: ateom.CheckpointWorkloadRequest.scope:type_name -> ateom.SnapshotScope + 5, // 11: ateom.RestoreWorkloadRequest.spec:type_name -> ateom.WorkloadSpec + 20, // 12: ateom.RestoreWorkloadRequest.runtime_asset_paths:type_name -> ateom.RestoreWorkloadRequest.RuntimeAssetPathsEntry + 0, // 13: ateom.RestoreWorkloadRequest.scope:type_name -> ateom.SnapshotScope + 4, // 14: ateom.RestoreWorkloadRequest.egress_gateway:type_name -> ateom.EgressGateway + 1, // 15: ateom.GetWorkloadStatsResponse.sandbox_class:type_name -> ateom.SandboxClass + 2, // 16: ateom.GetWorkloadStatsResponse.source:type_name -> ateom.StatsSource + 3, // 17: ateom.Ateom.RunWorkload:input_type -> ateom.RunWorkloadRequest + 12, // 18: ateom.Ateom.CheckpointWorkload:input_type -> ateom.CheckpointWorkloadRequest + 14, // 19: ateom.Ateom.RestoreWorkload:input_type -> ateom.RestoreWorkloadRequest + 16, // 20: ateom.Ateom.GetWorkloadStats:input_type -> ateom.GetWorkloadStatsRequest + 11, // 21: ateom.Ateom.RunWorkload:output_type -> ateom.RunWorkloadResponse + 13, // 22: ateom.Ateom.CheckpointWorkload:output_type -> ateom.CheckpointWorkloadResponse + 15, // 23: ateom.Ateom.RestoreWorkload:output_type -> ateom.RestoreWorkloadResponse + 17, // 24: ateom.Ateom.GetWorkloadStats:output_type -> ateom.GetWorkloadStatsResponse + 21, // [21:25] is the sub-list for method output_type + 17, // [17:21] is the sub-list for method input_type + 17, // [17:17] is the sub-list for extension type_name + 17, // [17:17] is the sub-list for extension extendee + 0, // [0:17] is the sub-list for field type_name } func init() { file_ateom_proto_init() } @@ -1435,14 +1505,14 @@ func file_ateom_proto_init() { return } file_ateom_proto_msgTypes[0].OneofWrappers = []any{} - file_ateom_proto_msgTypes[10].OneofWrappers = []any{} + file_ateom_proto_msgTypes[11].OneofWrappers = []any{} type x struct{} out := protoimpl.TypeBuilder{ File: protoimpl.DescBuilder{ GoPackagePath: reflect.TypeOf(x{}).PkgPath(), RawDescriptor: unsafe.Slice(unsafe.StringData(file_ateom_proto_rawDesc), len(file_ateom_proto_rawDesc)), NumEnums: 3, - NumMessages: 17, + NumMessages: 18, NumExtensions: 0, NumServices: 1, }, diff --git a/internal/proto/ateompb/ateom.proto b/internal/proto/ateompb/ateom.proto index 2d6a8d8be..1d72bd389 100644 --- a/internal/proto/ateompb/ateom.proto +++ b/internal/proto/ateompb/ateom.proto @@ -124,6 +124,15 @@ message Container { // durable_dir_volume_mounts are the durable-dir volumes this container // mounts, if any. repeated DurableDirVolumeMount durable_dir_volume_mounts = 4; + + // csi_volume_mounts are the CSI volumes this container mounts, if any. + repeated VolumeMount csi_volume_mounts = 5; +} + +// VolumeMount is one volume mounted into a container. +message VolumeMount { + string volume_name = 1; + string mount_path = 2; } // DurableDirVolumeMount is one durable-dir volume mounted into a container. diff --git a/manifests/ate-install/generated/ate.dev_actortemplates.yaml b/manifests/ate-install/generated/ate.dev_actortemplates.yaml index f5da2c721..971778706 100644 --- a/manifests/ate-install/generated/ate.dev_actortemplates.yaml +++ b/manifests/ate-install/generated/ate.dev_actortemplates.yaml @@ -453,9 +453,6 @@ spec: rule: '!has(self.volumes) || self.volumes.all(v, has(self.containers) && self.containers.exists(c, has(c.volumeMounts) && c.volumeMounts.exists(vm, vm.name == v.name)))' - - message: ExternalVolumes are not supported when sandboxClass is 'microvm' - rule: '!has(self.sandboxClass) || self.sandboxClass != ''microvm'' || - !has(self.volumes) || !self.volumes.exists(v, has(v.externalVolumeTemplate))' - message: 'onResume.fromData: Golden is not supported when sandboxClass is ''gvisor''' rule: '(has(self.sandboxClass) && self.sandboxClass == ''microvm'') diff --git a/pkg/api/v1alpha1/actortemplate_types.go b/pkg/api/v1alpha1/actortemplate_types.go index 358e24cc9..aaa1fbac7 100644 --- a/pkg/api/v1alpha1/actortemplate_types.go +++ b/pkg/api/v1alpha1/actortemplate_types.go @@ -355,7 +355,6 @@ type SnapshotsConfig struct { // ActorTemplateSpec defined desired spec of an actor. // // +kubebuilder:validation:XValidation:rule="!has(self.volumes) || self.volumes.all(v, has(self.containers) && self.containers.exists(c, has(c.volumeMounts) && c.volumeMounts.exists(vm, vm.name == v.name)))",message="All volumes defined in spec.volumes must be mounted by at least one container" -// +kubebuilder:validation:XValidation:rule="!has(self.sandboxClass) || self.sandboxClass != 'microvm' || !has(self.volumes) || !self.volumes.exists(v, has(v.externalVolumeTemplate))",message="ExternalVolumes are not supported when sandboxClass is 'microvm'" // +kubebuilder:validation:XValidation:rule="(has(self.sandboxClass) && self.sandboxClass == 'microvm') || !has(self.snapshotsConfig.onResume) || (has(self.snapshotsConfig.onResume.fromData) ? self.snapshotsConfig.onResume.fromData : 'ColdBoot') != 'Golden'",message="onResume.fromData: Golden is not supported when sandboxClass is 'gvisor'" type ActorTemplateSpec struct { // Containers is the workload definition. diff --git a/pkg/api/v1alpha1/actortemplate_validation_test.go b/pkg/api/v1alpha1/actortemplate_validation_test.go index 89edc6c27..ce168829f 100644 --- a/pkg/api/v1alpha1/actortemplate_validation_test.go +++ b/pkg/api/v1alpha1/actortemplate_validation_test.go @@ -1136,7 +1136,7 @@ func TestActorTemplateValidation(t *testing.T) { }, wantErr: false, }, { - name: "Volumes: ExternalVolumeTemplate volume with SandboxClass microvm is invalid", + name: "Volumes: ExternalVolumeTemplate volume with SandboxClass microvm is valid", mutate: func(at *ActorTemplate) { at.Spec.SandboxClass = SandboxClassMicroVM at.Spec.Volumes = []Volume{ @@ -1154,8 +1154,7 @@ func TestActorTemplateValidation(t *testing.T) { {Name: "vol1", MountPath: "/mnt/data"}, } }, - wantErr: true, - errMsg: "ExternalVolumes are not supported when sandboxClass is 'microvm'", + wantErr: false, }, { name: "Volumes: ExternalVolumeTemplate volume with SandboxClass gvisor is valid", mutate: func(at *ActorTemplate) {