From bc8c5b292fabdd4c6c3bc16299e632e0b0a1b658 Mon Sep 17 00:00:00 2001 From: yy Date: Wed, 25 Mar 2026 17:10:16 +0800 Subject: [PATCH 1/2] feat: support stargz snapshotter for devbox with lvm powerd --- pkg/cri/internal/devboxsnapshotter/labels.go | 41 +++++++++++++ pkg/cri/sbserver/container_create.go | 5 +- pkg/cri/sbserver/container_create_linux.go | 14 ++++- .../sbserver/container_create_linux_test.go | 59 +++++++++++++++++++ pkg/cri/sbserver/container_create_other.go | 2 +- pkg/cri/sbserver/devbox_snapshotter.go | 23 ++++++++ pkg/cri/server/container_create.go | 10 ++-- pkg/cri/server/container_create_linux.go | 13 ++-- pkg/cri/server/container_create_linux_test.go | 49 +++++++++++++++ pkg/cri/server/container_create_other.go | 5 ++ pkg/cri/server/container_stop.go | 2 +- pkg/cri/server/devbox_snapshotter.go | 28 +++++++++ pkg/cri/server/events.go | 2 +- pkg/cri/server/helpers.go | 2 +- pkg/cri/server/image_pull.go | 4 +- 15 files changed, 241 insertions(+), 18 deletions(-) create mode 100644 pkg/cri/internal/devboxsnapshotter/labels.go create mode 100644 pkg/cri/sbserver/devbox_snapshotter.go create mode 100644 pkg/cri/server/devbox_snapshotter.go diff --git a/pkg/cri/internal/devboxsnapshotter/labels.go b/pkg/cri/internal/devboxsnapshotter/labels.go new file mode 100644 index 0000000000000..074b6ab5ec584 --- /dev/null +++ b/pkg/cri/internal/devboxsnapshotter/labels.go @@ -0,0 +1,41 @@ +/* + Copyright The containerd Authors. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +package devboxsnapshotter + +const ( + StargzSnapshotter = "stargz" + SealosDevboxContentIDAnnotation = "devbox.sealos.io/content-id" + SealosDevboxStorageLimitAnnotation = "devbox.sealos.io/storage-limit" +) + +// LabelsFromAnnotations keeps only the annotations used by the devbox-backed +// writable-layer flow. containerd.WithNewSnapshot will translate these labels +// to the snapshotter-specific containerd.io/snapshot/devbox-* keys when needed. +func LabelsFromAnnotations(annotations map[string]string) map[string]string { + if len(annotations) == 0 { + return nil + } + + labels := make(map[string]string) + if contentID := annotations[SealosDevboxContentIDAnnotation]; contentID != "" { + labels[SealosDevboxContentIDAnnotation] = contentID + } + if storageLimit := annotations[SealosDevboxStorageLimitAnnotation]; storageLimit != "" { + labels[SealosDevboxStorageLimitAnnotation] = storageLimit + } + return labels +} diff --git a/pkg/cri/sbserver/container_create.go b/pkg/cri/sbserver/container_create.go index 5044734780554..26f1607633fe8 100644 --- a/pkg/cri/sbserver/container_create.go +++ b/pkg/cri/sbserver/container_create.go @@ -207,11 +207,12 @@ func (c *criService) CreateContainer(ctx context.Context, r *runtime.CreateConta log.G(ctx).Debugf("Container %q spec: %#+v", id, spew.NewFormatter(spec)) // Grab any platform specific snapshotter opts. - sOpts := snapshotterOpts(c.config.ContainerdConfig.Snapshotter, config) + runtimeSnapshotter := c.runtimeSnapshotter(ctx, ociRuntime) + sOpts := snapshotterOpts(runtimeSnapshotter, config, sandboxConfig) // Set snapshotter before any other options. opts := []containerd.NewContainerOpts{ - containerd.WithSnapshotter(c.runtimeSnapshotter(ctx, ociRuntime)), + containerd.WithSnapshotter(runtimeSnapshotter), // Prepare container rootfs. This is always writeable even if // the container wants a readonly rootfs since we want to give // the runtime (runc) a chance to modify (e.g. to create mount diff --git a/pkg/cri/sbserver/container_create_linux.go b/pkg/cri/sbserver/container_create_linux.go index f33438d65d3f7..21dfe58d9ee48 100644 --- a/pkg/cri/sbserver/container_create_linux.go +++ b/pkg/cri/sbserver/container_create_linux.go @@ -31,6 +31,7 @@ import ( "github.com/containerd/containerd/contrib/apparmor" "github.com/containerd/containerd/contrib/seccomp" "github.com/containerd/containerd/oci" + "github.com/containerd/containerd/pkg/cri/internal/devboxsnapshotter" "github.com/containerd/containerd/snapshots" customopts "github.com/containerd/containerd/pkg/cri/opts" @@ -264,6 +265,15 @@ func appArmorProfileExists(profile string) (bool, error) { } // snapshotterOpts returns any Linux specific snapshotter options for the rootfs snapshot -func snapshotterOpts(snapshotterName string, config *runtime.ContainerConfig) []snapshots.Opt { - return []snapshots.Opt{} +func snapshotterOpts(snapshotterName string, _ *runtime.ContainerConfig, sandboxConfig *runtime.PodSandboxConfig) []snapshots.Opt { + if !snapshotterNeedsDevboxLabels(snapshotterName) || sandboxConfig == nil { + return nil + } + + labels := devboxsnapshotter.LabelsFromAnnotations(sandboxConfig.Annotations) + if len(labels) == 0 { + return nil + } + + return []snapshots.Opt{snapshots.WithLabels(labels)} } diff --git a/pkg/cri/sbserver/container_create_linux_test.go b/pkg/cri/sbserver/container_create_linux_test.go index 7076c31bc18fe..cd71536b48a9d 100644 --- a/pkg/cri/sbserver/container_create_linux_test.go +++ b/pkg/cri/sbserver/container_create_linux_test.go @@ -30,6 +30,7 @@ import ( "github.com/containerd/containerd/contrib/seccomp" "github.com/containerd/containerd/mount" "github.com/containerd/containerd/oci" + "github.com/containerd/containerd/snapshots" "github.com/containerd/platforms" imagespec "github.com/opencontainers/image-spec/specs-go/v1" runtimespec "github.com/opencontainers/runtime-spec/specs-go" @@ -42,6 +43,7 @@ import ( "github.com/containerd/containerd/pkg/cap" "github.com/containerd/containerd/pkg/cri/annotations" "github.com/containerd/containerd/pkg/cri/config" + "github.com/containerd/containerd/pkg/cri/internal/devboxsnapshotter" "github.com/containerd/containerd/pkg/cri/opts" customopts "github.com/containerd/containerd/pkg/cri/opts" "github.com/containerd/containerd/pkg/cri/util" @@ -273,6 +275,63 @@ func TestContainerCapabilities(t *testing.T) { } } +func TestSnapshotterOptsForDevboxLabels(t *testing.T) { + tests := []struct { + name string + snapshotter string + sandboxConfig *runtime.PodSandboxConfig + expectedLabels map[string]string + }{ + { + name: "stargz keeps relevant devbox annotations", + snapshotter: devboxsnapshotter.StargzSnapshotter, + sandboxConfig: &runtime.PodSandboxConfig{ + Annotations: map[string]string{ + devboxsnapshotter.SealosDevboxContentIDAnnotation: "workspace-1", + devboxsnapshotter.SealosDevboxStorageLimitAnnotation: "20Gi", + "other.annotation": "ignored", + }, + }, + expectedLabels: map[string]string{ + devboxsnapshotter.SealosDevboxContentIDAnnotation: "workspace-1", + devboxsnapshotter.SealosDevboxStorageLimitAnnotation: "20Gi", + }, + }, + { + name: "non-stargz skips devbox labels", + snapshotter: "overlayfs", + sandboxConfig: &runtime.PodSandboxConfig{ + Annotations: map[string]string{ + devboxsnapshotter.SealosDevboxContentIDAnnotation: "workspace-2", + devboxsnapshotter.SealosDevboxStorageLimitAnnotation: "10Gi", + }, + }, + expectedLabels: nil, + }, + { + name: "nil sandbox config produces no opts", + snapshotter: devboxsnapshotter.StargzSnapshotter, + sandboxConfig: nil, + expectedLabels: nil, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + opts := snapshotterOpts(tt.snapshotter, &runtime.ContainerConfig{}, tt.sandboxConfig) + if tt.expectedLabels == nil { + require.Len(t, opts, 0) + return + } + + require.Len(t, opts, 1) + info := &snapshots.Info{Labels: make(map[string]string)} + opts[0](info) + assert.Equal(t, tt.expectedLabels, info.Labels) + }) + } +} + func TestContainerSpecTty(t *testing.T) { testID := "test-id" testSandboxID := "sandbox-id" diff --git a/pkg/cri/sbserver/container_create_other.go b/pkg/cri/sbserver/container_create_other.go index c97d2f568ea7f..b82a2dc00e517 100644 --- a/pkg/cri/sbserver/container_create_other.go +++ b/pkg/cri/sbserver/container_create_other.go @@ -31,6 +31,6 @@ func (c *criService) containerSpecOpts(config *runtime.ContainerConfig, imageCon } // snapshotterOpts returns snapshotter options for the rootfs snapshot -func snapshotterOpts(snapshotterName string, config *runtime.ContainerConfig) []snapshots.Opt { +func snapshotterOpts(snapshotterName string, config *runtime.ContainerConfig, sandboxConfig *runtime.PodSandboxConfig) []snapshots.Opt { return []snapshots.Opt{} } diff --git a/pkg/cri/sbserver/devbox_snapshotter.go b/pkg/cri/sbserver/devbox_snapshotter.go new file mode 100644 index 0000000000000..29f3880ed8406 --- /dev/null +++ b/pkg/cri/sbserver/devbox_snapshotter.go @@ -0,0 +1,23 @@ +/* + Copyright The containerd Authors. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +package sbserver + +import "github.com/containerd/containerd/pkg/cri/internal/devboxsnapshotter" + +func snapshotterNeedsDevboxLabels(name string) bool { + return name == devboxsnapshotter.StargzSnapshotter +} diff --git a/pkg/cri/server/container_create.go b/pkg/cri/server/container_create.go index 52870a3bdac7f..de0b300307c06 100644 --- a/pkg/cri/server/container_create.go +++ b/pkg/cri/server/container_create.go @@ -189,9 +189,11 @@ func (c *criService) CreateContainer(ctx context.Context, r *runtime.CreateConta return nil, err } - // Check if the snapshotter is devbox and add the devbox snapshotter opts - if c.runtimeSnapshotter(ctx, ociRuntime) == "devbox" { - devboxOpt, err := devboxSnapshotterOpts(c.runtimeSnapshotter(ctx, ociRuntime), r.GetSandboxConfig()) + // Add devbox snapshot labels for snapshotters that support the devbox + // writable-layer flow. + runtimeSnapshotter := c.runtimeSnapshotter(ctx, ociRuntime) + if isDevboxWritableSnapshotter(runtimeSnapshotter) { + devboxOpt, err := devboxSnapshotterOpts(runtimeSnapshotter, r.GetSandboxConfig()) if err != nil { return nil, err } @@ -202,7 +204,7 @@ func (c *criService) CreateContainer(ctx context.Context, r *runtime.CreateConta // Set snapshotter before any other options. opts := []containerd.NewContainerOpts{ - containerd.WithSnapshotter(c.runtimeSnapshotter(ctx, ociRuntime)), + containerd.WithSnapshotter(runtimeSnapshotter), // Prepare container rootfs. This is always writeable even if // the container wants a readonly rootfs since we want to give // the runtime (runc) a chance to modify (e.g. to create mount diff --git a/pkg/cri/server/container_create_linux.go b/pkg/cri/server/container_create_linux.go index 94ef3e871d7db..d8a63c01ce118 100644 --- a/pkg/cri/server/container_create_linux.go +++ b/pkg/cri/server/container_create_linux.go @@ -612,9 +612,14 @@ func generateUserString(username string, uid, gid *runtime.Int64Value) (string, } // snapshotterOpts returns any Linux specific snapshotter options for the rootfs snapshot -func devboxSnapshotterOpts(snapshotterName string, config *runtime.PodSandboxConfig) (snapshots.Opt, error) { - // fmt.Printf("devboxSnapshotterOpts: snapshotterName=%s, config=%+v\n", snapshotterName, config) - // add container annotations to snapshot labels +func devboxSnapshotterOpts(_ string, config *runtime.PodSandboxConfig) (snapshots.Opt, error) { + if config == nil || len(config.Annotations) == 0 { + return nil, nil + } + + // Preserve the original devbox annotations here and let + // containerd.WithNewSnapshot translate them for snapshotters that consume + // containerd.io/snapshot/devbox-* labels. labels := make(map[string]string) maps.Copy(labels, config.Annotations) return snapshots.WithLabels(labels), nil @@ -634,7 +639,7 @@ func snapshotterOpts(snapshotterName string, config *runtime.ContainerConfig, sa uid := sandboxConfig.Annotations[devboxUIDEnvKey] if uid != "" { snapshotOpts = append(snapshotOpts, snapshots.WithLabels(map[string]string{ - "devbox.sealos.io/uid": uid, + devboxUIDEnvKey: uid, })) } return snapshotOpts, nil diff --git a/pkg/cri/server/container_create_linux_test.go b/pkg/cri/server/container_create_linux_test.go index f9b36b7022349..ff548ecf74fa6 100644 --- a/pkg/cri/server/container_create_linux_test.go +++ b/pkg/cri/server/container_create_linux_test.go @@ -43,6 +43,7 @@ import ( "github.com/containerd/containerd/pkg/cap" "github.com/containerd/containerd/pkg/cri/annotations" "github.com/containerd/containerd/pkg/cri/config" + "github.com/containerd/containerd/pkg/cri/internal/devboxsnapshotter" "github.com/containerd/containerd/pkg/cri/opts" customopts "github.com/containerd/containerd/pkg/cri/opts" "github.com/containerd/containerd/pkg/cri/util" @@ -2285,3 +2286,51 @@ func TestSnapshotterOpts(t *testing.T) { }) } } + +func TestDevboxSnapshotterOpts(t *testing.T) { + tests := []struct { + name string + snapshotter string + annotations map[string]string + expectedLabels map[string]string + }{ + { + name: "stargz keeps original annotations", + snapshotter: devboxsnapshotter.StargzSnapshotter, + annotations: map[string]string{ + devboxsnapshotter.SealosDevboxContentIDAnnotation: "workspace-1", + devboxsnapshotter.SealosDevboxStorageLimitAnnotation: "20Gi", + }, + expectedLabels: map[string]string{ + devboxsnapshotter.SealosDevboxContentIDAnnotation: "workspace-1", + devboxsnapshotter.SealosDevboxStorageLimitAnnotation: "20Gi", + }, + }, + { + name: "devbox snapshotter keeps original annotations only", + snapshotter: DevboxSnapshotter, + annotations: map[string]string{ + devboxsnapshotter.SealosDevboxContentIDAnnotation: "workspace-3", + devboxsnapshotter.SealosDevboxStorageLimitAnnotation: "5Gi", + }, + expectedLabels: map[string]string{ + devboxsnapshotter.SealosDevboxContentIDAnnotation: "workspace-3", + devboxsnapshotter.SealosDevboxStorageLimitAnnotation: "5Gi", + }, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + opt, err := devboxSnapshotterOpts(tt.snapshotter, &runtime.PodSandboxConfig{ + Annotations: tt.annotations, + }) + require.NoError(t, err) + require.NotNil(t, opt) + + info := &snapshots.Info{Labels: make(map[string]string)} + opt(info) + assert.Equal(t, tt.expectedLabels, info.Labels) + }) + } +} diff --git a/pkg/cri/server/container_create_other.go b/pkg/cri/server/container_create_other.go index 595118c3e44b0..27400f864e2df 100644 --- a/pkg/cri/server/container_create_other.go +++ b/pkg/cri/server/container_create_other.go @@ -60,3 +60,8 @@ func (c *criService) containerSpecOpts(config *runtime.ContainerConfig, imageCon func snapshotterOpts(snapshotterName string, config *runtime.ContainerConfig, sandboxConfig *runtime.PodSandboxConfig) ([]snapshots.Opt, error) { return []snapshots.Opt{}, nil } + +// Non-Linux builds don't support the devbox-backed writable-layer flow. +func devboxSnapshotterOpts(snapshotterName string, config *runtime.PodSandboxConfig) (snapshots.Opt, error) { + return nil, nil +} diff --git a/pkg/cri/server/container_stop.go b/pkg/cri/server/container_stop.go index d881cb25724aa..e03d95879b697 100644 --- a/pkg/cri/server/container_stop.go +++ b/pkg/cri/server/container_stop.go @@ -86,7 +86,7 @@ func (c *criService) StopContainer(ctx context.Context, r *runtime.StopContainer log.G(ctx).Infof("Check snapshotter: %s", snapshotter) // Check if the snapshotter is devbox and update the devbox snapshot - if snapshotter == "devbox" { + if isDevboxWritableSnapshotter(snapshotter) { err = c.client.UpdateDevboxSnapshot(ctx, snapshotter, i.ID, unmountLvm, "true") if err != nil { log.G(ctx).WithError(err).Errorf("Failed to update devbox snapshot: %s", err) diff --git a/pkg/cri/server/devbox_snapshotter.go b/pkg/cri/server/devbox_snapshotter.go new file mode 100644 index 0000000000000..d7d427f778a3f --- /dev/null +++ b/pkg/cri/server/devbox_snapshotter.go @@ -0,0 +1,28 @@ +/* + Copyright The containerd Authors. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +package server + +import "github.com/containerd/containerd/pkg/cri/internal/devboxsnapshotter" + +func isDevboxWritableSnapshotter(name string) bool { + switch name { + case DevboxSnapshotter, devboxsnapshotter.StargzSnapshotter: + return true + default: + return false + } +} diff --git a/pkg/cri/server/events.go b/pkg/cri/server/events.go index b927216873be7..b013bf6df0a7f 100644 --- a/pkg/cri/server/events.go +++ b/pkg/cri/server/events.go @@ -448,7 +448,7 @@ func handleContainerExit(ctx context.Context, e *eventtypes.TaskExit, cntr conta return status, err } fmt.Println("Container snapshotter:", container.Snapshotter, "ID:", cntr.Container.ID()) - if container.Snapshotter == "devbox" { + if isDevboxWritableSnapshotter(container.Snapshotter) { err = c.client.UpdateDevboxSnapshot(ctx, container.Snapshotter, container.ID, unmountLvm, "true") if err != nil { logrus.WithError(err).Errorf("Failed to update devbox snapshot for container %s", cntr.Container.ID()) diff --git a/pkg/cri/server/helpers.go b/pkg/cri/server/helpers.go index 79f52ea9ee50e..50a637702af5e 100644 --- a/pkg/cri/server/helpers.go +++ b/pkg/cri/server/helpers.go @@ -91,7 +91,7 @@ const ( // runtimeRunhcsV1 is the runtime type for runhcs. runtimeRunhcsV1 = "io.containerd.runhcs.v1" - // DevboxSnapshotter is the name of the devbox snapshotter. + // DevboxSnapshotter is the name of the classic devbox snapshotter. DevboxSnapshotter = "devbox" ) diff --git a/pkg/cri/server/image_pull.go b/pkg/cri/server/image_pull.go index 9cbbb59535b16..52345a48953f9 100644 --- a/pkg/cri/server/image_pull.go +++ b/pkg/cri/server/image_pull.go @@ -210,8 +210,8 @@ func (c *criService) PullImage(ctx context.Context, r *runtime.PullImageRequest) // Create a copy of labels for each reference to avoid modifying the original map var imageLabels map[string]string - if r == repoTag && snapshotter == DevboxSnapshotter { - // Only add pin label for repoTag when snapshotter is devbox + if r == repoTag && isDevboxWritableSnapshotter(snapshotter) { + // Only add pin label for repoTag when the snapshotter participates in the devbox writable-layer flow. imageLabels = make(map[string]string) for k, v := range labels { imageLabels[k] = v From 0d69979aff7b0878e274051be4b41c7a403835c5 Mon Sep 17 00:00:00 2001 From: yy Date: Mon, 30 Mar 2026 18:48:45 +0800 Subject: [PATCH 2/2] feat: devbox with lvm powerd --- pkg/cri/sbserver/container_create_linux_test.go | 15 +++++++++++++++ pkg/cri/sbserver/devbox_snapshotter.go | 9 ++++++++- 2 files changed, 23 insertions(+), 1 deletion(-) diff --git a/pkg/cri/sbserver/container_create_linux_test.go b/pkg/cri/sbserver/container_create_linux_test.go index cd71536b48a9d..89a2bdc5f73bc 100644 --- a/pkg/cri/sbserver/container_create_linux_test.go +++ b/pkg/cri/sbserver/container_create_linux_test.go @@ -297,6 +297,21 @@ func TestSnapshotterOptsForDevboxLabels(t *testing.T) { devboxsnapshotter.SealosDevboxStorageLimitAnnotation: "20Gi", }, }, + { + name: "devbox keeps relevant devbox annotations", + snapshotter: "devbox", + sandboxConfig: &runtime.PodSandboxConfig{ + Annotations: map[string]string{ + devboxsnapshotter.SealosDevboxContentIDAnnotation: "workspace-9", + devboxsnapshotter.SealosDevboxStorageLimitAnnotation: "8Gi", + "other.annotation": "ignored", + }, + }, + expectedLabels: map[string]string{ + devboxsnapshotter.SealosDevboxContentIDAnnotation: "workspace-9", + devboxsnapshotter.SealosDevboxStorageLimitAnnotation: "8Gi", + }, + }, { name: "non-stargz skips devbox labels", snapshotter: "overlayfs", diff --git a/pkg/cri/sbserver/devbox_snapshotter.go b/pkg/cri/sbserver/devbox_snapshotter.go index 29f3880ed8406..e71279cf92728 100644 --- a/pkg/cri/sbserver/devbox_snapshotter.go +++ b/pkg/cri/sbserver/devbox_snapshotter.go @@ -18,6 +18,13 @@ package sbserver import "github.com/containerd/containerd/pkg/cri/internal/devboxsnapshotter" +const devboxSnapshotter = "devbox" + func snapshotterNeedsDevboxLabels(name string) bool { - return name == devboxsnapshotter.StargzSnapshotter + switch name { + case devboxSnapshotter, devboxsnapshotter.StargzSnapshotter: + return true + default: + return false + } }