diff --git a/.github/workflows/profiles/runc-rootless.json b/.github/workflows/profiles/runc-rootless.json index 7bc9d23..2db5fb4 100644 --- a/.github/workflows/profiles/runc-rootless.json +++ b/.github/workflows/profiles/runc-rootless.json @@ -1,5 +1,5 @@ { "shim_binary": "containerd-shim-runc-v2", "uid": 1000, - "skip": ["layers", "oom", "transfer", "uds"] + "skip": ["layers", "oom", "sandbox", "transfer", "uds"] } diff --git a/.github/workflows/profiles/runc.json b/.github/workflows/profiles/runc.json index 2c5ed76..3b1ff9e 100644 --- a/.github/workflows/profiles/runc.json +++ b/.github/workflows/profiles/runc.json @@ -1,5 +1,5 @@ { "shim_binary": "containerd-shim-runc-v2", "uid": 0, - "skip": ["layers", "transfer", "uds"] + "skip": ["layers", "sandbox", "transfer", "uds"] } diff --git a/README.md b/README.md index 7f702f4..dbed038 100644 --- a/README.md +++ b/README.md @@ -40,7 +40,7 @@ Tests are driven by one or more JSON configuration files. See | `uid` | int | UID to run as; defaults to the current user's UID. If set to a value different from the current UID and the effective UID is 0, the harness re-execs itself as that user via `sudo` | | `gid` | int | GID to run as | | `format_mounts` | bool | Provide the rootfs as formatted erofs/ext4 images with a `format/mkdir/overlay` descriptor for the shim to mount. Default (`false`) extracts the rootfs and provides a pre-mounted overlay (or plain directory when rootless) | -| `skip` | []string | Feature names to skip (`exec`, `layers`, `net`, `oom`, `transfer`, `uds`) | +| `skip` | []string | Feature names to skip (`exec`, `layers`, `net`, `oom`, `sandbox`, `transfer`, `uds`) | | `env` | map | Additional environment variables for the test run | | `debug` | bool | Enable debug logging on the shim | @@ -113,7 +113,18 @@ config, the tree is `TestShim//`. | `OutboundTCP` | net | A container's init process opens an outbound TCP connection to a host-reachable endpoint and completes a round trip. Implementation-neutral: does not assume any particular networking mechanism, only that a container has working default outbound TCP connectivity, as "host networking" would provide. | | `OutboundUDP` | net | A container's init process exchanges a UDP datagram with a host-reachable endpoint. Same neutrality as `OutboundTCP`, for the datagram path. | | `DNSResolve` | net | A container's init process resolves a real external hostname (`example.com`, forcing an actual DNS query — not answered from `/etc/hosts`) and gets back valid IP addresses. Requires outbound internet access from the test host. | -| `Stress` | (per feature) | Long-running concurrent stress run. Composes subtests from the enabled features (currently transfer: stat/write/read). Each subtest runs as a goroutine until the test deadline approaches or any one fails (which cancels the rest). Skipped under `-test.short`. | +| `Lifecycle` | sandbox | Full sandbox lifecycle: `CreateSandbox` → `StartSandbox` → `SandboxStatus`(ready) → `StopSandbox` → `SandboxStatus`(stopped) → `ShutdownSandbox`. Verifies state transitions and that the bootstrap version is ≥ 3. Linux only. | +| `Platform` | sandbox | `Platform` RPC returns a non-empty OS and Architecture. Linux only. | +| `Ping` | sandbox | `PingSandbox` succeeds while the sandbox is running. Linux only. | +| `SingleContainer` | sandbox | One member container created on the shared connection produces expected output and exits cleanly. Linux only. | +| `MultipleContainers` | sandbox | Three member containers run concurrently in one sandbox VM, each with isolated rootfs and independent output. Linux only. | +| `ContainerLifecycleIndependence` | sandbox | Deleting one member container leaves the sandbox and other containers running; `SandboxStatus` remains ready. Linux only. | +| `StatusAfterStop` | sandbox | `SandboxStatus` after `StopSandbox` does not report ready; a second `StopSandbox` is idempotent. Linux only. | +| `WaitUnblocksOnStop` | sandbox | `WaitSandbox` returns within 15 s after `StopSandbox`. Linux only. | +| `CreateTwiceRejected` | sandbox | A second `CreateSandbox` on the same shim process returns `AlreadyExists`. Linux only. | +| `StartWithoutCreateRejected` | sandbox | `StartSandbox` before `CreateSandbox` returns `FailedPrecondition`. Linux only. | +| `ResourceReleaseOnShutdown` | sandbox | After `ShutdownSandbox`, no per-container mount points remain in the shim's mount namespace. Linux only. | +| `Stress` | (per feature) | Long-running concurrent stress run. Composes subtests from the enabled features (Lifecycle, Exec, Transfer, Sandbox). Each subtest runs as a goroutine until the test deadline approaches or any one fails (which cancels the rest). Skipped under `-test.short`. The Sandbox subtest also checks host RSS growth and mount-point leaks after each run. | A separate top-level fuzz target exists alongside `TestShim`: @@ -152,6 +163,26 @@ Benchmarks live under `BenchmarkShim//`. | `StdioRoundTrip` | exec | Stdio write/read at 8B, 4KB, 4MB | | `UDSRoundTrip` | uds | UDS forwarded-socket throughput in both directions (HostToContainer, ContainerToHost) at 8B, 4KB, 4MB | | `ThirtyLayers` | layers | Bring up a container with a 30-layer erofs rootfs (same shape as `HundredLayers`, smaller). Reports per-phase metrics (`ms/shim-start`, `ms/create`, `ms/task-start`, `ms/total`) so multi-layer mount overhead can be localized. Requires `format_mounts=true` | +| `ContainerCreate` | sandbox | Per-container create/start/wait/delete cycle inside a single shared sandbox VM. The sandbox is started once before the `b.N` loop; each iteration adds one member container, runs it to completion, and removes it. Reports `ms/create`, `ms/start`, `ms/wait`, `ms/delete`, `ms/total`, and `ms/sandbox-start` (one-time amortised cost). Compare `ms/create` and `ms/total` with `RunSuite.Lifecycle` to quantify the marginal cost of a sandbox container versus a fresh-VM container. Linux only. | + +### Member-container workload contracts + +These tests (all gated on the `sandbox` feature) verify the shim API contract for member-container workloads: status fields, host-network sandboxes, exec, shared endpoints, shared namespaces, volumes, and network scoping. + +| Test | Feature | Linux only | Verifies | +|---|---|---|---| +| `StatusReportsPidAndCreatedAt` | sandbox | no | `SandboxStatus` returns a non-zero `pid` and a non-zero `created_at` after `StartSandbox`. Both let a caller reference the sandbox's namespaces (`/proc//ns/*`) and report its age (e.g. as CRI's `PodSandboxStatus.CreatedAt` does). `SandboxStatus.Info` must carry `pid` and `state` entries. | +| `HostNetworkNoNetworkSandbox` | sandbox | no | A sandbox created with an empty `netns_path` (no network sandbox provided) must succeed. Member containers must run normally. The shim must accept the no-isolation case without error. | +| `MemberContainerExec` | sandbox | no | `Task.Exec` + `Task.Start(ExecID)` must run an additional process inside a running member container with correct output and exit-status propagation. Underpins any exec-into-a-running-container use case (interactive exec, health probes, sidecar tooling). | +| `CrossContainerViaUDS` | sandbox | yes | Two exec processes running inside a sandbox member container both connect to a shared host-side UNIX domain socket forwarded in via the `uds` mount type. Proves that multiple processes in a sandbox can reach a common host-forwarded endpoint. | +| `NetworkSandboxHeldOpen` | sandbox | yes | When a non-empty `netns_path` is provided in `CreateSandboxRequest`, the shim must hold the network sandbox resource open for the sandbox lifetime. The path must remain reachable while the sandbox is ready and must be releasable after `StopSandbox`. No special privileges required. | +| `NetworkSandboxPathInStatus` | sandbox | yes | If a network sandbox path was provided in `CreateSandboxRequest`, `SandboxStatus.Info["networkSandboxPath"]` should report the same path. Informational (absence does not hard-fail); the field is optional in the base protocol. No special privileges required. | +| `ContainerOutboundTCP` | sandbox | no | A process exec'd into a member container must be able to resolve a real external hostname, proving it has a working outbound network path. Implementation-neutral: does not assume any particular networking mechanism (native netns, virtual NIC, or otherwise). DNS is used here (rather than a raw TCP round trip) because `Task.Exec` has no stdin plumbing; `NetworkSuite` separately covers TCP and UDP round trips end-to-end on the legacy path. No special privileges required. | +| `ContainerTrafficScopedToNetworkSandbox` | sandbox | yes | When a non-empty `netns_path` is provided, a member container's outbound traffic must actually originate from within that network sandbox, not merely have the path pinned open. Uses a veth pair fully contained in a real network namespace (unreachable from outside it) as a falsifiable probe: a successful round trip is only possible if the container's traffic originates inside the sandbox. Requires root (CAP_SYS_ADMIN) to create the namespace and interfaces; skipped otherwise. | +| `MemberContainersShareNetwork` | sandbox | yes | Member containers of the same sandbox must share a network stack: a listener started by one member container must be reachable from a second, independently created member container via loopback, with no explicit network configuration on either container. Implementation-neutral: does not assume any particular mechanism (a real shared network namespace, a shared virtual interface, or any other approach). No special privileges required. | +| `MemberContainerHostVolume` | sandbox | yes | A member container's OCI spec may include a "bind" mount referencing a host directory (e.g. as CRI's hostPath volumes or Kubernetes `RecursiveReadOnly=false` bind mounts produce). The shim must honor it as a live, two-way share, not a one-time copy: a file updated on the host after the container has already started must become visible inside it. Implementation-neutral: does not assume any particular sharing mechanism. No special privileges required. | +| `MemberContainersSharePID` | sandbox | yes | When a member container's OCI spec carries a host path on its PID namespace entry (e.g. as a caller uses to express Kubernetes' `shareProcessNamespace: true` or `hostPID: true`), the shim must place that container in a PID namespace shared with its sandbox peers. A process started in one member container must be visible — by PID and argv — via `/proc` in a second, independently created member container. Implementation-neutral: does not assume any particular sharing mechanism. No special privileges required. | +| `MemberContainersShareIPC` | sandbox | yes | When a member container's OCI spec carries a host path on its IPC namespace entry (e.g. as a caller uses to express Kubernetes' default of always sharing one IPC namespace across a pod's containers), the shim must place that container in an IPC namespace shared with its sandbox peers. A SysV shared memory segment created by one member container must be visible — by its well-known key — to a second, independently created member container. Implementation-neutral: does not assume any particular sharing mechanism. No special privileges required. | ## Using shimtest in your shim's CI @@ -233,7 +264,7 @@ unbounded `Stress` run, and run active fuzzing as its own step: - **`uid`**: omit to run as the runner user. Set explicitly when you want the harness to `sudo` re-exec itself or rewrite the profile. - **`skip`**: list of feature names to disable. Currently meaningful - values are `exec`, `layers`, `net`, `oom`, `transfer`, and `uds` — + values are `exec`, `layers`, `net`, `oom`, `sandbox`, `transfer`, and `uds` — useful when your shim doesn't implement transfer/UDS forwarding, multi-layer rootfs descriptors, or when running rootless without cgroup delegation. diff --git a/cmd/testbin/main.go b/cmd/testbin/main.go index a260cad..8b3131a 100644 --- a/cmd/testbin/main.go +++ b/cmd/testbin/main.go @@ -19,8 +19,8 @@ // test suite, invoked either directly (testbin [args...]) or // via symlink (e.g. /bin/cat -> /bin/testbin). // -// Commands: forever, burstexit, cat, date, echo, exit, hashverify, -// layercheck, ls, memhog, nc, tickexit +// Commands: forever, burstexit, cat, date, echo, echosrv, exit, hashverify, +// host, layercheck, ls, memhog, nc, tickexit package main import "github.com/containerd/shimtest/testbin" diff --git a/helpers.go b/helpers.go index 3f641a9..a423273 100644 --- a/helpers.go +++ b/helpers.go @@ -200,6 +200,32 @@ func withExtraMounts(mounts ...specs.Mount) func(*specs.Spec) { } } +// withHostPathNamespace returns a CreateOCISpec opt that sets (or adds) +// a namespace entry of the given type with a host path. A non-empty Path +// on an IPC/PID/network namespace entry is how an OCI spec requests that +// a container join a namespace shared with others, rather than getting a +// fresh, isolated one — for example, a host "/proc//ns/" +// path, as containerd's WithPodNamespaces sets for pod-level namespace +// sharing. The actual path value here is a placeholder — it only needs +// to be non-empty, since the shim's job is to recognize that a host path +// is present at all and substitute its own guest-side shared namespace, +// not to interpret the path itself (which is meaningless off the host +// that produced it). +func withHostPathNamespace(nsType specs.LinuxNamespaceType, path string) func(*specs.Spec) { + return func(s *specs.Spec) { + if s.Linux == nil { + s.Linux = &specs.Linux{} + } + for i, ns := range s.Linux.Namespaces { + if ns.Type == nsType { + s.Linux.Namespaces[i].Path = path + return + } + } + s.Linux.Namespaces = append(s.Linux.Namespaces, specs.LinuxNamespace{Type: nsType, Path: path}) + } +} + // withMemoryLimit returns a CreateOCISpec opt that sets the memory // limit (in bytes) on the spec, with swap clamped equal to the limit // so the container cannot grow via swap before the OOM killer fires. diff --git a/helpers_networksandbox_linux.go b/helpers_networksandbox_linux.go new file mode 100644 index 0000000..27da774 --- /dev/null +++ b/helpers_networksandbox_linux.go @@ -0,0 +1,69 @@ +//go:build linux + +/* + Copyright The containerd Authors. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +package shimtest + +import ( + "os" + "path/filepath" + "testing" + + "golang.org/x/sys/unix" +) + +// createNetworkSandbox creates a host-side network sandbox resource and +// returns its file path. Cleanup is registered on tb. +// +// The network sandbox is represented as a regular file. This is sufficient +// to test the shim's API contract — that it accepts a netns_path in +// CreateSandboxRequest, holds the resource open for the sandbox lifetime, and +// reports the path in SandboxStatus.Info — without requiring root privileges +// or kernel support for bind-mounting namespace files. +// +// In production, a caller provides a bind-mounted network namespace file +// created before calling CreateSandbox. The shim is expected to open the +// path, pin it for the sandbox lifetime, and (when running as root) enter +// the namespace so that member-container traffic originates from the +// provided netns. Entering the namespace requires CAP_SYS_ADMIN; when the +// shim lacks that capability it logs a warning and continues without +// entering. +func createNetworkSandbox(tb testing.TB) string { + tb.Helper() + + nsPath := filepath.Join(tb.TempDir(), "network-sandbox") + + f, err := os.OpenFile(nsPath, os.O_CREATE|os.O_EXCL|os.O_RDONLY, 0o444) + if err != nil { + tb.Fatalf("createNetworkSandbox: create resource file: %v", err) + } + f.Close() + + // No explicit cleanup needed: tb.TempDir() handles removal. + return nsPath +} + +// networkSandboxIsOpen returns true if the network sandbox file at path still +// exists and is stat-able. Returns false if the path is empty, does not +// exist, or cannot be accessed. +func networkSandboxIsOpen(path string) bool { + if path == "" { + return false + } + var st unix.Stat_t + return unix.Stat(path, &st) == nil +} diff --git a/helpers_networksandbox_other.go b/helpers_networksandbox_other.go new file mode 100644 index 0000000..2cd1d5b --- /dev/null +++ b/helpers_networksandbox_other.go @@ -0,0 +1,33 @@ +//go:build !linux + +/* + Copyright The containerd Authors. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +package shimtest + +import "testing" + +// createNetworkSandbox is not available on non-Linux platforms. +// Tests that call it are expected to guard with runtime.GOOS == "linux" +// or be in a linux-only file; this stub satisfies the compiler on other +// platforms. +func createNetworkSandbox(tb testing.TB) string { + tb.Skip("createNetworkSandbox is Linux-only") + return "" +} + +// networkSandboxIsOpen always returns false on non-Linux platforms. +func networkSandboxIsOpen(_ string) bool { return false } diff --git a/helpers_realnetns_linux.go b/helpers_realnetns_linux.go new file mode 100644 index 0000000..14f733a --- /dev/null +++ b/helpers_realnetns_linux.go @@ -0,0 +1,215 @@ +//go:build linux + +/* + Copyright The containerd Authors. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +package shimtest + +import ( + "fmt" + "net" + "os" + "os/exec" + "runtime" + "strings" + "sync/atomic" + "testing" + "time" + + "golang.org/x/sys/unix" +) + +// realNetworkSandbox is a genuine Linux network namespace created via the +// system "ip netns" tool — a standard unshare(CLONE_NEWNET) + bind-mount +// technique for creating an isolated, enterable network namespace. +// Unlike the fake-file sandbox used by createNetworkSandbox, a +// realNetworkSandbox can be entered with setns(2) and can carry real network +// interfaces, so it can be used to observe where a shim's network traffic +// actually originates. +// +// Creating one requires CAP_SYS_ADMIN (root) in the initial user namespace; +// callers should be prepared for createRealNetworkSandbox to skip the test. +type realNetworkSandbox struct { + name string + path string +} + +var ( + netnsCounter atomic.Int32 + vethSubnetCounter atomic.Int32 +) + +// createRealNetworkSandbox creates a genuine network namespace using the +// system "ip netns add" command. Cleanup (namespace deletion, which also +// destroys any interfaces that live only inside it) is registered on tb. +// +// Skips the test if not running as root: creating and entering a real +// network namespace requires CAP_SYS_ADMIN. +func createRealNetworkSandbox(tb testing.TB) *realNetworkSandbox { + tb.Helper() + if os.Getuid() != 0 { + tb.Skip("createRealNetworkSandbox requires root (CAP_SYS_ADMIN)") + } + if _, err := exec.LookPath("ip"); err != nil { + tb.Skip("createRealNetworkSandbox requires the \"ip\" (iproute2) command") + } + + name := fmt.Sprintf("shimtest-%d-%d", os.Getpid(), netnsCounter.Add(1)) + if out, err := exec.Command("ip", "netns", "add", name).CombinedOutput(); err != nil { + tb.Fatalf("ip netns add %s: %v: %s", name, err, out) + } + tb.Cleanup(func() { + exec.Command("ip", "netns", "delete", name).Run() //nolint:errcheck + }) + + return &realNetworkSandbox{name: name, path: "/var/run/netns/" + name} +} + +// attachVeth creates a veth pair with both ends inside the sandbox namespace +// and assigns one end a unique address on a small point-to-point subnet. +// Because neither end of the pair ever exists in the root namespace, the +// resulting subnet has no route from outside the sandbox: only a process +// actually running inside the sandbox namespace (or a caller that has +// entered it via setns) can reach the returned address. This is what makes +// the address usable as a falsifiable probe for "did this traffic originate +// inside the network sandbox". +// +// The interfaces are destroyed automatically when the sandbox namespace is +// deleted; no separate cleanup is required. +func (n *realNetworkSandbox) attachVeth(tb testing.TB) (addr string) { + tb.Helper() + + slot := vethSubnetCounter.Add(1) % 250 + addr = fmt.Sprintf("10.244.%d.1", slot) + hostIf := fmt.Sprintf("vh%d", slot) + peerIf := fmt.Sprintf("vp%d", slot) + + runIP(tb, "link", "add", hostIf, "type", "veth", "peer", "name", peerIf) + runIP(tb, "link", "set", hostIf, "netns", n.name) + runIP(tb, "link", "set", peerIf, "netns", n.name) + + runIPNetns(tb, n.name, "addr", "add", addr+"/30", "dev", hostIf) + runIPNetns(tb, n.name, "link", "set", hostIf, "up") + runIPNetns(tb, n.name, "link", "set", peerIf, "up") + runIPNetns(tb, n.name, "link", "set", "lo", "up") + + return addr +} + +// runIP runs "ip " and fails the test on error. +func runIP(tb testing.TB, args ...string) { + tb.Helper() + if out, err := exec.Command("ip", args...).CombinedOutput(); err != nil { + tb.Fatalf("ip %s: %v: %s", strings.Join(args, " "), err, out) + } +} + +// runIPNetns runs "ip netns exec ip " and fails the test on +// error. +func runIPNetns(tb testing.TB, name string, args ...string) { + tb.Helper() + full := append([]string{"netns", "exec", name, "ip"}, args...) + if out, err := exec.Command("ip", full...).CombinedOutput(); err != nil { + tb.Fatalf("ip netns exec %s ip %s: %v: %s", name, strings.Join(args, " "), err, out) + } +} + +// probeUnreachableFromCurrentNamespace asserts that addr cannot be reached +// from the calling goroutine's current network namespace. It is used as a +// sanity check that a realNetworkSandbox address is actually isolated +// before relying on it as a falsifiable probe. +func probeUnreachableFromCurrentNamespace(tb testing.TB, addr string) { + tb.Helper() + conn, err := net.DialTimeout("tcp", addr, 300*time.Millisecond) + if err == nil { + conn.Close() + tb.Fatalf("test setup error: %s is reachable from the current namespace; "+ + "the sandbox isolation this test relies on is not actually in effect", addr) + } +} + +// listenAndEchoOnceInNetns enters the network namespace at nsPath on a +// dedicated, locked OS thread, binds a TCP listener at addr, accepts a +// single connection, echoes back one line, and closes. It returns a channel +// that receives the result (nil on success) once the exchange completes or +// fails. +// +// The listener is created on the locked thread after entering the +// namespace, so binding succeeds only when addr is actually reachable from +// within that namespace. Combined with an address from attachVeth (which has +// no route from the root namespace), a successful exchange on the returned +// channel is only possible if the connecting peer's traffic actually +// originated inside the sandbox namespace. +func listenAndEchoOnceInNetns(tb testing.TB, nsPath, addr string) <-chan error { + tb.Helper() + + ready := make(chan error, 1) + result := make(chan error, 1) + + go func() { + runtime.LockOSThread() + // Intentionally never unlocked: Go retires this OS thread when the + // goroutine exits (Go 1.10+), so the namespace change does not leak + // into the shared thread pool. + + f, err := os.Open(nsPath) + if err != nil { + ready <- fmt.Errorf("open netns %q: %w", nsPath, err) + return + } + defer f.Close() + if err := unix.Setns(int(f.Fd()), unix.CLONE_NEWNET); err != nil { + ready <- fmt.Errorf("setns %q: %w", nsPath, err) + return + } + + ln, err := net.Listen("tcp", addr) + if err != nil { + ready <- fmt.Errorf("listen %s in netns: %w", addr, err) + return + } + ready <- nil + + if tcpLn, ok := ln.(*net.TCPListener); ok { + tcpLn.SetDeadline(time.Now().Add(20 * time.Second)) + } + conn, err := ln.Accept() + ln.Close() + if err != nil { + result <- fmt.Errorf("accept: %w", err) + return + } + defer conn.Close() + conn.SetDeadline(time.Now().Add(10 * time.Second)) + + buf := make([]byte, 256) + n, rerr := conn.Read(buf) + if n == 0 && rerr != nil { + result <- fmt.Errorf("read: %w", rerr) + return + } + if _, err := conn.Write(buf[:n]); err != nil { + result <- fmt.Errorf("write: %w", err) + return + } + result <- nil + }() + + if err := <-ready; err != nil { + tb.Fatalf("listenAndEchoOnceInNetns: %v", err) + } + return result +} diff --git a/helpers_sandbox.go b/helpers_sandbox.go new file mode 100644 index 0000000..42c32a4 --- /dev/null +++ b/helpers_sandbox.go @@ -0,0 +1,739 @@ +//go:build linux + +/* + Copyright The containerd Authors. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +package shimtest + +import ( + "bufio" + "bytes" + "context" + "encoding/json" + "fmt" + "os" + "os/exec" + "path/filepath" + "strings" + "sync" + "testing" + "time" + + sandboxAPI "github.com/containerd/containerd/api/runtime/sandbox/v1" + taskAPI "github.com/containerd/containerd/api/runtime/task/v3" + "github.com/containerd/containerd/api/types" + "github.com/containerd/containerd/v2/pkg/namespaces" + "github.com/containerd/ttrpc" + typeurl "github.com/containerd/typeurl/v2" + specs "github.com/opencontainers/runtime-spec/specs-go" + "google.golang.org/protobuf/types/known/anypb" +) + +// containerOutput holds the captured stdout for a member container. +type containerOutput struct { + buf *bytes.Buffer + mu *sync.Mutex +} + +// sandboxEnv holds the shared state for a running sandbox: the TTRPC +// client, sandbox and task service clients, and the list of member +// containers created so far. +type sandboxEnv struct { + ctx context.Context + client *ttrpc.Client + sc sandboxAPI.TTRPCSandboxService + tc taskAPI.TTRPCTaskService + sandboxID string + address string + + mu sync.Mutex + containers []string // member container IDs in creation order + stdoutBufs map[string]*containerOutput // cid -> captured stdout + stdinPaths map[string]string // cid -> stdin FIFO path (only set when withSandboxCtrStdin is used) +} + +// startSandboxShim starts the shim binary for a sandbox and drives it +// through CreateSandbox + StartSandbox. It returns a *sandboxEnv +// backed by the shared TTRPC connection. +// +// API contract enforced here: +// - Bootstrap version ≥ 3 (enables per-connection task routing). +// - CreateSandbox and StartSandbox must succeed. +// - StartSandboxResponse.Pid must be > 0. +// - StartSandboxResponse.CreatedAt must be set and non-zero. +// +// Cleanup (ShutdownSandbox + shim delete) is registered on tb. +// startSandboxShim starts a sandbox shim with no network sandbox (host-network +// or platforms where network sandboxes are not supported). +// Use startSandboxShimWithNetworkSandbox to pass a network sandbox path. +func startSandboxShim(tb testing.TB, cfg Config, sandboxID string) *sandboxEnv { + tb.Helper() + return startSandboxShimInner(tb, cfg, sandboxID, "") +} + +// writeSandboxOCISpec writes a minimal OCI config.json suitable for a +// pod-sandbox bundle. The spec carries resource annotations so that +// VM-based shims start with a small VM; shims that do not use these +// annotations ignore them. +func writeSandboxOCISpec(tb testing.TB, bundleDir string) { + tb.Helper() + spec := struct { + OciVersion string `json:"ociVersion"` + Annotations map[string]string `json:"annotations,omitempty"` + }{ + OciVersion: "1.0.2", + Annotations: map[string]string{ + "io.containerd.nerdbox.resources.cpu": "2", + "io.containerd.nerdbox.resources.memory": "2048", + }, + } + data, err := json.Marshal(spec) + if err != nil { + tb.Fatal("marshal sandbox OCI spec:", err) + } + if err := os.WriteFile(filepath.Join(bundleDir, "config.json"), data, 0o644); err != nil { + tb.Fatal("write sandbox config.json:", err) + } +} + +// shutdownSandboxShim is the cleanup function registered by +// startSandboxShim. It tears down any remaining member containers +// then calls StopSandbox and ShutdownSandbox. Errors are logged but +// not fatal so that cleanup proceeds even after a failed test. +func shutdownSandboxShim(tb testing.TB, env *sandboxEnv) { + tb.Helper() + ctx, cancel := context.WithTimeout(env.ctx, 30*time.Second) + defer cancel() + + env.mu.Lock() + ctrs := make([]string, len(env.containers)) + copy(ctrs, env.containers) + env.mu.Unlock() + + for _, cid := range ctrs { + env.tc.Kill(ctx, &taskAPI.KillRequest{ID: cid, Signal: 9, All: true}) //nolint:errcheck + env.tc.Wait(ctx, &taskAPI.WaitRequest{ID: cid}) //nolint:errcheck + env.tc.Delete(ctx, &taskAPI.DeleteRequest{ID: cid}) //nolint:errcheck + } + + env.sc.StopSandbox(ctx, &sandboxAPI.StopSandboxRequest{SandboxID: env.sandboxID}) //nolint:errcheck + env.sc.ShutdownSandbox(ctx, &sandboxAPI.ShutdownSandboxRequest{SandboxID: env.sandboxID}) //nolint:errcheck +} + +// createContainerInSandbox creates (and by default starts) a member +// container inside an already-running sandbox. It uses the same +// TTRPC connection that was used for the sandbox lifecycle RPCs, which +// is the routing mechanism containerd uses in production. +// +// Stdout is captured into an internal buffer; callers read it via +// readContainerOutput(env, cid). Cleanup (kill/wait/delete) is +// registered on tb. +// +// Returns the container ID. +func createContainerInSandbox(tb testing.TB, env *sandboxEnv, args []string, specOpts ...func(*sandboxCtrSpec)) string { + tb.Helper() + + so := &sandboxCtrSpec{} + for _, opt := range specOpts { + opt(so) + } + + cid := containerID(tb) + + bundleDir := tb.TempDir() + bundleDir, err := filepath.EvalSymlinks(bundleDir) + if err != nil { + tb.Fatal("evalSymlinks member bundleDir:", err) + } + + // Member containers: build the rootfs from the embedded testbin. + // For the sandbox path, ShareRootfs on the host will assemble the + // rootfs from whatever mounts are provided. We must give the shim + // a mount spec it can execute on the host. + // + // When running as root, use FormatMounts=true (erofs images with an + // overlay descriptor) so the shim assembles the overlay properly. + // + // When running as non-root, FormatMounts=false extracts the rootfs + // directly into bundleDir/rootfs and returns nil mounts. In that + // case we provide a single bind mount of that pre-extracted dir so + // ShareRootfs can bind it into the shared dir. + cfg := Config{FormatMounts: os.Getuid() == 0} + rootfsMounts := buildEmbeddedRootfs(tb, bundleDir, cfg) + + // Non-root / pre-extracted path: nil mounts means the rootfs is + // already in bundleDir/rootfs — present it as a bind mount. + if len(rootfsMounts) == 0 { + rootfsMounts = []*types.Mount{{ + Type: "bind", + Source: filepath.Join(bundleDir, "rootfs"), + Options: []string{"ro", "rbind"}, + }} + } + + var ociOpts []func(*specs.Spec) + if len(so.extraMounts) > 0 { + ociOpts = append(ociOpts, withExtraMounts(so.extraMounts...)) + } + ociOpts = append(ociOpts, so.ociOpts...) + createOCISpec(tb, bundleDir, args, cfg, ociOpts...) + + var stdinPath, stdoutPath, stderrPath string + if so.stdin { + stdinPath, stdoutPath, stderrPath = createStdioFifos(tb, bundleDir) + } else { + stdoutPath, stderrPath = createIOFifos(tb, bundleDir) + } + // Start capturing stdout into a buffer before Task.Create so the + // shim's forwardIO can open the write end without blocking. + var stdoutBuf bytes.Buffer + var stdoutMu sync.Mutex + drainFifoInto(tb, env.ctx, stdoutPath, &stdoutBuf, &stdoutMu) + // Stderr is discarded. + drainFifo(tb, env.ctx, stderrPath) + + var req *taskAPI.CreateTaskRequest + if so.stdin { + req = newCreateTaskRequestStdin(tb, cid, bundleDir, stdinPath, stdoutPath, stderrPath, rootfsMounts) + } else { + req = newCreateTaskRequest(tb, cid, bundleDir, stdoutPath, stderrPath, rootfsMounts) + } + if _, err := env.tc.Create(env.ctx, req); err != nil { + tb.Fatalf("Task.Create member %s: %v", cid, err) + } + + if !so.noStart { + if _, err := env.tc.Start(env.ctx, &taskAPI.StartRequest{ID: cid}); err != nil { + tb.Fatalf("Task.Start member %s: %v", cid, err) + } + } + + env.mu.Lock() + env.containers = append(env.containers, cid) + env.stdoutBufs[cid] = &containerOutput{buf: &stdoutBuf, mu: &stdoutMu} + if so.stdin { + if env.stdinPaths == nil { + env.stdinPaths = make(map[string]string) + } + env.stdinPaths[cid] = stdinPath + } + env.mu.Unlock() + + tb.Cleanup(func() { + ctx, cancel := context.WithTimeout(env.ctx, 10*time.Second) + defer cancel() + env.tc.Kill(ctx, &taskAPI.KillRequest{ID: cid, Signal: 9, All: true}) //nolint:errcheck + env.tc.Wait(ctx, &taskAPI.WaitRequest{ID: cid}) //nolint:errcheck + env.tc.Delete(ctx, &taskAPI.DeleteRequest{ID: cid}) //nolint:errcheck + + env.mu.Lock() + for i, id := range env.containers { + if id == cid { + env.containers = append(env.containers[:i], env.containers[i+1:]...) + break + } + } + delete(env.stdoutBufs, cid) + env.mu.Unlock() + }) + + return cid +} + +// sandboxCtrSpec carries options for createContainerInSandbox. +type sandboxCtrSpec struct { + noStart bool + stdin bool + extraMounts []specs.Mount // extra OCI mounts appended to the container spec + ociOpts []func(*specs.Spec) // extra low-level OCI spec opts (e.g. namespace sharing) +} + +// withSandboxCtrStdin requests that createContainerInSandbox wire up a +// stdin FIFO for the member container, in addition to stdout/stderr. Use +// writeContainerStdin to send data once the container is running. +func withSandboxCtrStdin() func(*sandboxCtrSpec) { + return func(o *sandboxCtrSpec) { o.stdin = true } +} + +// withSandboxCtrExtraMounts appends mounts to the container's OCI spec. +// Use this to inject shared volumes, /dev/shm bind-mounts, or UDS-mount +// entries into a sandbox member container. +func withSandboxCtrExtraMounts(mounts ...specs.Mount) func(*sandboxCtrSpec) { + return func(o *sandboxCtrSpec) { + o.extraMounts = append(o.extraMounts, mounts...) + } +} + +// withSandboxCtrNamespace requests that the given namespace type be set +// to a (placeholder) host path in the container's OCI spec, signaling +// that the container should join a namespace shared with its sandbox +// peers rather than a fresh, isolated one. See withHostPathNamespace for +// why the specific path value does not matter. +func withSandboxCtrNamespace(nsType specs.LinuxNamespaceType, path string) func(*sandboxCtrSpec) { + return func(o *sandboxCtrSpec) { + o.ociOpts = append(o.ociOpts, withHostPathNamespace(nsType, path)) + } +} + +// createSandboxContainerFast creates and starts a member container using +// pre-built rootfs images. It is the stress-loop counterpart of +// createContainerInSandbox: it avoids calling writeRootfsErofs / +// writeBigFileErofs on every iteration (which would exhaust tmpfs over +// thousands of iterations) by accepting images built once before the loop. +// +// preExtractedRootfs is a pre-populated directory used as the bind-mount +// source on non-root systems (where loop mounts are unavailable). Pass "" +// on root systems where FormatMounts is true (the erofs+overlay path is used +// instead). +// +// Unlike createContainerInSandbox it does NOT register a tb.Cleanup, and it +// does NOT capture stdout into a buffer. Callers must call +// releaseSandboxContainer when done with each container. Stdout/stderr are +// drained and discarded. +func createSandboxContainerFast(ctx context.Context, tb testing.TB, env *sandboxEnv, cfg Config, imgs shimImages, preExtractedRootfs string, args []string) (string, error) { + cid := containerID(tb) + + bundleDir := tb.TempDir() + bundleDir, err := filepath.EvalSymlinks(bundleDir) + if err != nil { + return "", fmt.Errorf("evalSymlinks: %w", err) + } + + rootfsDir := filepath.Join(bundleDir, "rootfs") + if err := os.MkdirAll(rootfsDir, 0755); err != nil { + return "", fmt.Errorf("mkdir rootfs: %w", err) + } + + rootfsMounts := buildSandboxMemberMountsFromImages(tb, cfg, imgs, rootfsDir, preExtractedRootfs) + + createOCISpec(tb, bundleDir, args, Config{FormatMounts: cfg.FormatMounts}) + + // Use null (empty) IO paths so the shim skips FIFO creation entirely. + // This avoids the per-container FIFO files and the goroutines that drain + // them from accumulating in the test's temp directory over thousands of + // iterations. The shim treats empty Stdout/Stderr as "discard IO". + req := newCreateTaskRequest(tb, cid, bundleDir, "", "", rootfsMounts) + if _, err := env.tc.Create(ctx, req); err != nil { + return "", fmt.Errorf("Task.Create: %w", err) + } + if _, err := env.tc.Start(ctx, &taskAPI.StartRequest{ID: cid}); err != nil { + return "", fmt.Errorf("Task.Start: %w", err) + } + + env.mu.Lock() + env.containers = append(env.containers, cid) + env.mu.Unlock() + + return cid, nil +} + +// buildSandboxMemberMountsFromImages builds rootfs mount specs for a stress +// iteration using pre-built images. It only creates the per-iteration +// writable parts (ext4 scratch or overlay upper/work), reusing the read-only +// erofs images across iterations to avoid O(N) disk consumption. +// +// When running as root with FormatMounts, the full erofs+ext4+overlay path +// is used (same as benchContainerCreate). Otherwise a bind mount of the +// given preExtractedRootfs directory is returned so ShareRootfs can copy it +// into the sandbox shared dir without needing loop devices. +// preExtractedRootfs must be pre-populated by the caller once before the +// loop; it is read-only and reused across all iterations. +func buildSandboxMemberMountsFromImages(tb testing.TB, cfg Config, imgs shimImages, rootfsDir, preExtractedRootfs string) []*types.Mount { + tb.Helper() + if cfg.FormatMounts && os.Getuid() == 0 { + // Root + format mounts: use the erofs+ext4+overlay path. + // rootfsDir gets a fresh ext4 scratch on each iteration. + return buildRootfsMountsFromImages(tb, cfg, imgs, rootfsDir) + } + // Non-root or no format mounts: point at the pre-extracted directory. + // ShareRootfs will copy it into the sandbox shared dir per container. + if preExtractedRootfs == "" { + // Fallback if caller did not pre-extract (shouldn't happen). + extractErofsIntoDir(tb, imgs.erofsImg, rootfsDir) + preExtractedRootfs = rootfsDir + } + return []*types.Mount{{ + Type: "bind", + Source: preExtractedRootfs, + Options: []string{"ro", "rbind"}, + }} +} + +// releaseSandboxContainer immediately releases a container that was created +// with createContainerInSandbox. It issues Task.Delete on the shim (which +// triggers host-side rootfs cleanup via Unshare) and removes the container +// from the env tracking maps so the memory is reclaimed during the run. +// +// This is the per-iteration counterpart to the tb.Cleanup registered by +// createContainerInSandbox. Call it in stress loops where containers are +// short-lived: it prevents env.stdoutBufs from growing unboundedly across +// thousands of iterations and avoids stacking O(N) redundant tb.Cleanup +// registrations that would fire at test teardown. +// +// After releaseSandboxContainer returns the tb.Cleanup registered at +// creation time will still fire, but it becomes a no-op: the container is +// gone from env.containers and env.stdoutBufs, so the Kill/Wait/Delete RPCs +// will return NotFound and the map deletes are idempotent. +func releaseSandboxContainer(ctx context.Context, env *sandboxEnv, cid string) error { + _, err := env.tc.Delete(ctx, &taskAPI.DeleteRequest{ID: cid}) + + env.mu.Lock() + for i, id := range env.containers { + if id == cid { + env.containers = append(env.containers[:i], env.containers[i+1:]...) + break + } + } + delete(env.stdoutBufs, cid) + env.mu.Unlock() + + return err +} + +// withSandboxCtrNoStart creates the task without issuing Task.Start. +func withSandboxCtrNoStart() func(*sandboxCtrSpec) { + return func(o *sandboxCtrSpec) { o.noStart = true } +} + +// readContainerOutput waits up to timeout for want to appear in the +// captured stdout for the container with the given ID. The container +// must have been created via createContainerInSandbox on env. +func readContainerOutput(tb testing.TB, env *sandboxEnv, cid, want string, timeout time.Duration) string { + tb.Helper() + env.mu.Lock() + co := env.stdoutBufs[cid] + env.mu.Unlock() + if co == nil { + tb.Fatalf("no captured stdout for container %s", cid) + } + deadline := time.After(timeout) + for { + co.mu.Lock() + got := co.buf.String() + co.mu.Unlock() + if strings.Contains(got, want) { + return got + } + select { + case <-deadline: + co.mu.Lock() + final := co.buf.String() + co.mu.Unlock() + tb.Fatalf("timed out waiting for %q in stdout of %s, got: %q", want, cid, final) + case <-time.After(20 * time.Millisecond): + } + } +} + +// writeContainerStdin writes data to the stdin FIFO of a member container +// created with withSandboxCtrStdin, then closes the write end. The +// container must have been created via createContainerInSandbox with the +// withSandboxCtrStdin option. +func writeContainerStdin(tb testing.TB, env *sandboxEnv, cid, data string) { + tb.Helper() + env.mu.Lock() + stdinPath := env.stdinPaths[cid] + env.mu.Unlock() + if stdinPath == "" { + tb.Fatalf("no stdin FIFO for container %s (was it created with withSandboxCtrStdin?)", cid) + } + w, err := openPipeWriter(env.ctx, stdinPath) + if err != nil { + tb.Fatalf("open stdin fifo for %s: %v", cid, err) + } + if _, err := w.Write([]byte(data)); err != nil { + tb.Fatalf("write stdin for %s: %v", cid, err) + } + if err := w.Close(); err != nil { + tb.Fatalf("close stdin fifo for %s: %v", cid, err) + } +} + +// readSandboxOutput waits up to timeout for want to appear in the FIFO +// at stdoutPath, returning the full accumulated output. Prefer +// readContainerOutput when the container was created with +// createContainerInSandbox. +func readSandboxOutput(tb testing.TB, ctx context.Context, stdoutPath, want string, timeout time.Duration) string { + tb.Helper() + var buf bytes.Buffer + var mu sync.Mutex + drainFifoInto(tb, ctx, stdoutPath, &buf, &mu) + deadline := time.After(timeout) + for { + mu.Lock() + got := buf.String() + mu.Unlock() + if strings.Contains(got, want) { + return got + } + select { + case <-deadline: + mu.Lock() + final := buf.String() + mu.Unlock() + tb.Fatalf("timed out waiting for %q in stdout, got: %q", want, final) + case <-time.After(20 * time.Millisecond): + } + } +} + +// sandboxShimPID resolves the shim OS PID via the Task.Connect RPC +// after the first member container exists. Returns 0 if unavailable. +func sandboxShimPID(env *sandboxEnv, memberCID string) int { + pid, err := shimPidViaConnect(env.address, memberCID, 2*time.Second) + if err != nil { + return 0 + } + return pid +} + +// sandboxMountTargets returns all mount targets visible in the shim +// process's mount namespace by parsing /proc//mountinfo. +// Returns nil if pid == 0 or the file is unreadable. +func sandboxMountTargets(pid int) []string { + if pid == 0 { + return nil + } + f, err := os.Open(fmt.Sprintf("/proc/%d/mountinfo", pid)) + if err != nil { + return nil + } + defer f.Close() + + var targets []string + scanner := bufio.NewScanner(f) + for scanner.Scan() { + // mountinfo: id parent major:minor root mountpoint options ... + fields := strings.Fields(scanner.Text()) + if len(fields) >= 5 { + targets = append(targets, fields[4]) + } + } + return targets +} + +// sandboxContainersMounts returns mount targets that fall under the +// sandbox shared containers directory (i.e. paths containing +// "/containers/"). Used by the mount-leak detector. +func sandboxContainersMounts(pid int) []string { + all := sandboxMountTargets(pid) + var matched []string + for _, t := range all { + if strings.Contains(t, "/containers/") { + matched = append(matched, t) + } + } + return matched +} + +// waitForSandboxStatus polls SandboxStatus until the state matches +// want or the deadline is exceeded. +func waitForSandboxStatus(ctx context.Context, sc sandboxAPI.TTRPCSandboxService, sandboxID, want string, timeout time.Duration) error { + deadline := time.Now().Add(timeout) + for { + resp, err := sc.SandboxStatus(ctx, &sandboxAPI.SandboxStatusRequest{SandboxID: sandboxID}) + if err == nil && resp.GetState() == want { + return nil + } + if time.Now().After(deadline) { + state := "unknown" + if err == nil { + state = resp.GetState() + } + return fmt.Errorf("timed out waiting for state %q, last state %q (err: %v)", want, state, err) + } + select { + case <-ctx.Done(): + return ctx.Err() + case <-time.After(50 * time.Millisecond): + } + } +} + +// Unused import guard: types is used only to satisfy the compiler when +// buildRootfsMountsForSandbox is called from sandbox_suite.go. The +// function body references the types package via buildEmbeddedRootfs. +var _ *types.Mount + +// startSandboxShimWithNetworkSandbox starts a sandbox shim and passes +// networkSandboxPath in the CreateSandboxRequest. It is otherwise identical +// to startSandboxShim. Pass an empty string for the host-network case +// (no network sandbox). +// +// The API contract: the shim must hold the host-side network sandbox open +// for the lifetime of the sandbox so that CNI and other host tooling can +// operate on it while the sandbox is running. +func startSandboxShimWithNetworkSandbox(tb testing.TB, cfg Config, sandboxID, networkSandboxPath string) *sandboxEnv { + tb.Helper() + return startSandboxShimInner(tb, cfg, sandboxID, networkSandboxPath) +} + +// startSandboxShimInner is the shared implementation; exposed via +// startSandboxShim (empty path) and startSandboxShimWithNetworkSandbox. +func startSandboxShimInner(tb testing.TB, cfg Config, sandboxID, networkSandboxPath string) *sandboxEnv { + tb.Helper() + + shimBin, err := exec.LookPath(cfg.ShimBinary) + if err != nil { + tb.Fatalf("shim binary %q not found in PATH: %v", cfg.ShimBinary, err) + } + shimDir := filepath.Dir(shimBin) + if !strings.Contains(os.Getenv("PATH"), shimDir) { + os.Setenv("PATH", shimDir+string(os.PathListSeparator)+os.Getenv("PATH")) + } + + bundleDir := tb.TempDir() + bundleDir, err = filepath.EvalSymlinks(bundleDir) + if err != nil { + tb.Fatal("evalSymlinks bundleDir:", err) + } + + writeSandboxOCISpec(tb, bundleDir) + startEventsRecorder(tb, bundleDir) + + ns := uniqueTestNamespace(tb, "sandbox") + ctx := namespaces.WithNamespace(tb.Context(), ns) + + params := startShim(tb, shimBin, bundleDir, sandboxID, ns, cfg) + + if params.Version < 3 { + tb.Fatalf("sandbox API requires bootstrap version ≥ 3, shim returned %d", params.Version) + } + + conn := connectShim(tb, params.Address) + client := ttrpc.NewClient(conn) + tb.Cleanup(func() { client.Close() }) + + sc := sandboxAPI.NewTTRPCSandboxClient(client) + tc := taskAPI.NewTTRPCTaskClient(client) + + env := &sandboxEnv{ + ctx: ctx, + client: client, + sc: sc, + tc: tc, + sandboxID: sandboxID, + address: params.Address, + stdoutBufs: make(map[string]*containerOutput), + } + + if _, err := sc.CreateSandbox(ctx, &sandboxAPI.CreateSandboxRequest{ + SandboxID: sandboxID, + BundlePath: bundleDir, + NetnsPath: networkSandboxPath, + }); err != nil { + tb.Fatalf("CreateSandbox: %v", err) + } + + startResp, err := sc.StartSandbox(ctx, &sandboxAPI.StartSandboxRequest{ + SandboxID: sandboxID, + }) + if err != nil { + tb.Fatalf("StartSandbox: %v", err) + } + if startResp.GetPid() == 0 { + tb.Error("StartSandbox returned pid=0; shim must report a non-zero pid") + } + if ts := startResp.GetCreatedAt(); ts == nil || ts.AsTime().IsZero() { + tb.Error("StartSandbox returned zero createdAt") + } + + tb.Cleanup(func() { + shutdownSandboxShim(tb, env) + }) + + return env +} + +// sandboxStatusInfo calls SandboxStatus with verbose=true and returns the +// state string and the Info map. If the RPC fails the test is failed. +func sandboxStatusInfo(tb testing.TB, env *sandboxEnv) (state string, info map[string]string) { + tb.Helper() + resp, err := env.sc.SandboxStatus(env.ctx, &sandboxAPI.SandboxStatusRequest{ + SandboxID: env.sandboxID, + Verbose: true, + }) + if err != nil { + tb.Fatalf("SandboxStatus: %v", err) + } + return resp.GetState(), resp.GetInfo() +} + +// execInSandboxContainer execs a process in a running member container and +// returns its stdout output and exit status. It blocks until the exec +// completes or timeout elapses. stderr is captured and included in the +// returned output (interleaved) so that callers can inspect error messages. +// +// The API contract: Task.Exec followed by Task.Start(ExecID) must run the +// command inside the container; Task.Wait must return the exit status after +// the process terminates. +func execInSandboxContainer(tb testing.TB, env *sandboxEnv, cid string, args []string, timeout time.Duration) (output string, exitStatus uint32) { + tb.Helper() + + execID := containerID(tb) // unique exec ID derived from test name + + execStdout, execStderr := createIOFifos(tb, tb.TempDir()) + var outBuf bytes.Buffer + var outMu sync.Mutex + drainFifoInto(tb, env.ctx, execStdout, &outBuf, &outMu) + // Also capture stderr so callers can see error messages from the exec'd process. + drainFifoInto(tb, env.ctx, execStderr, &outBuf, &outMu) + + procSpec, err := typeurl.MarshalAnyToProto(&specs.Process{ + Args: args, + Cwd: "/", + Env: []string{"PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin"}, + }) + if err != nil { + tb.Fatalf("marshal exec process spec: %v", err) + } + specAny := &anypb.Any{TypeUrl: procSpec.TypeUrl, Value: procSpec.Value} + + if _, err := env.tc.Exec(env.ctx, &taskAPI.ExecProcessRequest{ + ID: cid, + ExecID: execID, + Stdout: execStdout, + Stderr: execStderr, + Spec: specAny, + }); err != nil { + tb.Fatalf("Task.Exec in %s: %v", cid, err) + } + if _, err := env.tc.Start(env.ctx, &taskAPI.StartRequest{ + ID: cid, + ExecID: execID, + }); err != nil { + tb.Fatalf("Task.Start exec %s/%s: %v", cid, execID, err) + } + + ctx, cancel := context.WithTimeout(env.ctx, timeout) + defer cancel() + + waitResp, err := env.tc.Wait(ctx, &taskAPI.WaitRequest{ + ID: cid, + ExecID: execID, + }) + if err != nil { + tb.Fatalf("Task.Wait exec %s/%s: %v", cid, execID, err) + } + + // Allow a moment for the FIFO data to drain. + time.Sleep(50 * time.Millisecond) + outMu.Lock() + output = outBuf.String() + outMu.Unlock() + + return output, waitResp.GetExitStatus() +} diff --git a/helpers_sandbox_other.go b/helpers_sandbox_other.go new file mode 100644 index 0000000..808637b --- /dev/null +++ b/helpers_sandbox_other.go @@ -0,0 +1,63 @@ +//go:build !linux + +/* + Copyright The containerd Authors. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +// Package shimtest provides sandbox helpers. On non-Linux platforms the +// sandbox suite is not supported (virtiofs-backed shared container +// filesystems require Linux). The helpers here are stubs that satisfy +// the compiler; the SandboxSuite.Run method skips the entire suite at +// runtime. + +package shimtest + +import ( + "context" + "testing" + "time" + + sandboxAPI "github.com/containerd/containerd/api/runtime/sandbox/v1" +) + +type sandboxEnv struct{} + +func startSandboxShim(_ testing.TB, _ Config, _ string) *sandboxEnv { + return &sandboxEnv{} +} + +func createContainerInSandbox(_ testing.TB, _ *sandboxEnv, _ []string, _ ...func(*sandboxCtrSpec)) (string, string) { + return "", "" +} + +type sandboxCtrSpec struct{} + +func withSandboxCtrNoStart() func(*sandboxCtrSpec) { return func(*sandboxCtrSpec) {} } + +func readSandboxOutput(_ testing.TB, _ context.Context, _, _ string, _ time.Duration) string { + return "" +} + +func sandboxShimPID(_ *sandboxEnv, _ string) int { return 0 } + +func sandboxContainersMounts(_ int) []string { return nil } + +func sandboxMountTargets(_ int) []string { return nil } + +func waitForSandboxStatus(_ context.Context, _ sandboxAPI.TTRPCSandboxService, _, _ string, _ time.Duration) error { + return nil +} + +func shutdownSandboxShim(_ testing.TB, _ *sandboxEnv) {} diff --git a/rootfs.go b/rootfs.go index 06244e9..bc0d47b 100644 --- a/rootfs.go +++ b/rootfs.go @@ -125,7 +125,7 @@ func testbinAssetName(goarch string) string { // testbinCommands lists the commands provided by the testbin binary. // Symlinks are created in /bin for each command in the embedded // rootfs. -var testbinCommands = []string{"forever", "burstexit", "cat", "date", "echo", "exit", "hashverify", "host", "layercheck", "ls", "memhog", "nc", "tickexit"} +var testbinCommands = []string{"forever", "burstexit", "cat", "date", "echo", "echosrv", "exit", "hashverify", "host", "layercheck", "ls", "memhog", "nc", "pidscan", "shmread", "shmwrite", "tickexit"} // bigFileSize is the size of the IO benchmark fixture file. Large // enough to swamp small per-call overheads while still building / diff --git a/runner_test.go b/runner_test.go index 4f26de7..78fb7fb 100644 --- a/runner_test.go +++ b/runner_test.go @@ -57,8 +57,12 @@ func TestShim(t *testing.T) { if !featureSkipped("layers") { shimtest.NewLayersSuite(c).Run(t) } + if !featureSkipped("sandbox") { + shimtest.NewSandboxSuite(c).Run(t) + } t.Run("Stress", shimtest.NewStressSuite(c, shimtest.StressOptions{ Transfer: !featureSkipped("transfer"), + Sandbox: !featureSkipped("sandbox"), }).Run) }) } @@ -89,6 +93,9 @@ func BenchmarkShim(b *testing.B) { if !featureSkipped("layers") { shimtest.NewLayersSuite(c).Bench(b) } + if !featureSkipped("sandbox") { + shimtest.NewSandboxSuite(c).Bench(b) + } }) } } diff --git a/sandbox_bench_linux.go b/sandbox_bench_linux.go new file mode 100644 index 0000000..ae8eda5 --- /dev/null +++ b/sandbox_bench_linux.go @@ -0,0 +1,200 @@ +//go:build linux + +/* + Copyright The containerd Authors. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +package shimtest + +import ( + "context" + "fmt" + "os" + "os/exec" + "path/filepath" + "strings" + "testing" + "time" + + taskAPI "github.com/containerd/containerd/api/runtime/task/v3" + "github.com/containerd/containerd/api/types" +) + +// Bench runs every benchmark in the SandboxSuite as a sub-benchmark of b. +func (s *SandboxSuite) Bench(b *testing.B) { + b.Helper() + b.Run("ContainerCreate", s.benchContainerCreate) +} + +// benchContainerCreate measures the per-container create/start/wait/delete +// cycle inside a single running sandbox VM. The sandbox is started once +// before the iteration loop so the VM boot cost is paid only once; each +// b.N iteration adds one member container, runs it to completion, and +// removes it. +// +// This benchmark is the sandbox-API counterpart to RunSuite.benchLifecycle. +// Because the VM is shared across all iterations, per-iteration cost reflects +// only the marginal work needed to create and run a new container: rootfs +// assembly on the host, guest bundle/mount/task RPCs, and cleanup. The +// sandbox start time is reported separately as ms/sandbox-start so the +// amortised overhead is visible. +// +// Reported metrics (all in milliseconds, averaged over b.N): +// +// - ms/create — Task.Create RPC (rootfs assembly + guest bundle/mount/task) +// - ms/start — Task.Start RPC +// - ms/wait — Task.Wait until exit +// - ms/delete — Task.Delete RPC (rootfs unshare + cleanup) +// - ms/total — sum of the four phases above +// +// Reported once (not per-iteration): +// +// - ms/sandbox-start — time from sandbox shim launch to StartSandbox response +func (s *SandboxSuite) benchContainerCreate(b *testing.B) { + shimBin, err := exec.LookPath(s.cfg.ShimBinary) + if err != nil { + b.Fatalf("shim binary %q not found in PATH: %v", s.cfg.ShimBinary, err) + } + if shimDir := filepath.Dir(shimBin); !strings.Contains(os.Getenv("PATH"), shimDir) { + os.Setenv("PATH", shimDir+string(os.PathListSeparator)+os.Getenv("PATH")) + } + + // Pre-build the read-only rootfs images once so per-iteration setup + // only needs to construct the writable layer. For the sandbox path, + // buildSandboxMemberMounts uses these to produce the mounts passed to + // ShareRootfs on each iteration. + imgs := buildShimImages(b, s.cfg) + + sandboxID := containerID(b) + base := containerID(b) + + // ── Start the sandbox (timed separately, not part of b.N loop) ─────── + tSandboxStart := time.Now() + env := startSandboxShim(b, s.cfg, sandboxID) + sandboxStartMs := float64(time.Since(tSandboxStart).Microseconds()) / 1000.0 + + b.ReportMetric(sandboxStartMs, "ms/sandbox-start") + + // ── Per-iteration state ─────────────────────────────────────────────── + var sumCreate, sumStart, sumWait, sumDelete time.Duration + + b.ResetTimer() + for i := 0; i < b.N; i++ { + b.StopTimer() + + cid := fmt.Sprintf("%s-%d", base, i) + + // Build a fresh member-container bundle and rootfs mounts. + bundleDir := b.TempDir() + bundleDir, err = filepath.EvalSymlinks(bundleDir) + if err != nil { + b.Fatal("resolve member bundle dir:", err) + } + rootfsDir := filepath.Join(bundleDir, "rootfs") + if err := os.MkdirAll(rootfsDir, 0755); err != nil { + b.Fatal("mkdir rootfs:", err) + } + rootfsMounts := buildSandboxMemberMounts(b, s.cfg, imgs, rootfsDir, bundleDir) + cfg := Config{FormatMounts: s.cfg.FormatMounts} + createOCISpec(b, bundleDir, []string{"/bin/exit", "0"}, cfg) + + stdoutPath, stderrPath := createIOFifos(b, bundleDir) + drainFifo(b, env.ctx, stdoutPath) + drainFifo(b, env.ctx, stderrPath) + + req := newCreateTaskRequest(b, cid, bundleDir, stdoutPath, stderrPath, rootfsMounts) + + b.StartTimer() + + // Create + t := time.Now() + if _, err := env.tc.Create(env.ctx, req); err != nil { + b.Fatalf("Create %s: %v", cid, err) + } + sumCreate += time.Since(t) + + // Start + t = time.Now() + if _, err := env.tc.Start(env.ctx, &taskAPI.StartRequest{ID: cid}); err != nil { + b.Fatalf("Start %s: %v", cid, err) + } + sumStart += time.Since(t) + + // Wait + t = time.Now() + if _, err := env.tc.Wait(env.ctx, &taskAPI.WaitRequest{ID: cid}); err != nil { + b.Fatalf("Wait %s: %v", cid, err) + } + sumWait += time.Since(t) + + // Delete (triggers host-side rootfs cleanup via SharedFS.Unshare) + t = time.Now() + if _, err := env.tc.Delete(env.ctx, &taskAPI.DeleteRequest{ID: cid}); err != nil { + b.Fatalf("Delete %s: %v", cid, err) + } + sumDelete += time.Since(t) + + b.StopTimer() + + // Remove from env tracking so memory does not accumulate. + env.mu.Lock() + for i, id := range env.containers { + if id == cid { + env.containers = append(env.containers[:i], env.containers[i+1:]...) + break + } + } + delete(env.stdoutBufs, cid) + env.mu.Unlock() + } + + n := float64(b.N) + reportMs := func(d time.Duration, name string) { + b.ReportMetric(float64(d.Microseconds())/n/1000.0, name) + } + reportMs(sumCreate, "ms/create") + reportMs(sumStart, "ms/start") + reportMs(sumWait, "ms/wait") + reportMs(sumDelete, "ms/delete") + reportMs(sumCreate+sumStart+sumWait+sumDelete, "ms/total") +} + +// buildSandboxMemberMounts builds the rootfs mount specs for a sandbox member +// container benchmark iteration. It mirrors the logic in +// createContainerInSandbox but is optimised for benchmarks: when FormatMounts +// is true the pre-built erofs images are reused; otherwise a bind mount of the +// pre-extracted rootfs dir is returned (same fallback that ShareRootfs handles +// by copying into the shared dir). +func buildSandboxMemberMounts(tb testing.TB, cfg Config, imgs shimImages, rootfsDir, bundleDir string) []*types.Mount { + tb.Helper() + if cfg.FormatMounts && os.Getuid() == 0 { + return buildRootfsMountsFromImages(tb, cfg, imgs, rootfsDir) + } + // Non-root or non-format: extract once into rootfsDir, then wrap as + // a bind mount so ShareRootfs can copy it into the shared directory. + if os.Getuid() != 0 { + extractErofsIntoDir(tb, imgs.erofsImg, rootfsDir) + return []*types.Mount{{ + Type: "bind", + Source: rootfsDir, + Options: []string{"ro", "rbind"}, + }} + } + _ = bundleDir + return buildRootfsMountsFromImages(tb, cfg, imgs, rootfsDir) +} + +// Ensure the context package is used (env.ctx references it implicitly). +var _ context.Context diff --git a/sandbox_bench_other.go b/sandbox_bench_other.go new file mode 100644 index 0000000..c7543d5 --- /dev/null +++ b/sandbox_bench_other.go @@ -0,0 +1,26 @@ +//go:build !linux + +/* + Copyright The containerd Authors. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +package shimtest + +import "testing" + +// Bench skips sandbox benchmarks on non-Linux platforms. +func (s *SandboxSuite) Bench(b *testing.B) { + b.Skip("SandboxSuite benchmarks are Linux-only") +} diff --git a/sandbox_suite.go b/sandbox_suite.go new file mode 100644 index 0000000..c6af67f --- /dev/null +++ b/sandbox_suite.go @@ -0,0 +1,509 @@ +//go:build linux + +/* + Copyright The containerd Authors. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +package shimtest + +import ( + "fmt" + "strings" + "sync" + "syscall" + "testing" + "time" + + sandboxAPI "github.com/containerd/containerd/api/runtime/sandbox/v1" + taskAPI "github.com/containerd/containerd/api/runtime/task/v3" + tasktypes "github.com/containerd/containerd/api/types/task" + "github.com/containerd/containerd/v2/pkg/namespaces" + "github.com/containerd/ttrpc" +) + +// sandboxStateReady and sandboxStateNotReady are the two values a shim's +// SandboxStatusResponse.State must use. +// +// The runtime/sandbox/v1 proto itself only documents State as a plain +// string, with no enumerated values on the wire. But an unconstrained +// free-form string is not a usable API contract: any caller that needs to +// branch on sandbox readiness has to match against *some* fixed vocabulary, +// and a shim that invents its own spelling (a plausible one like "ready" +// included) breaks every caller that expects the specified names. State is +// therefore specified here as exactly one of these two strings, not an +// arbitrary human-readable state name. This is exactly the kind of +// externally-observable contract shimtest exists to check: it is invisible +// in the base shim-v2 sandbox protocol's type signature, but load-bearing +// for any caller that inspects sandbox readiness — for example, +// containerd's CRI layer maps these exact names onto the CRI v1 +// PodSandboxState enum when the shim sandboxer is configured. +const ( + sandboxStateReady = "SANDBOX_READY" + sandboxStateNotReady = "SANDBOX_NOTREADY" +) + +// SandboxSuite verifies the containerd sandbox shim API contract +// (runtime/sandbox/v1). Tests in this suite cover: +// +// - Sandbox lifecycle (create → start → status → stop → shutdown) +// - Platform and Ping RPCs +// - Member container routing: tasks created on the shared connection +// after StartSandbox must run correctly inside the sandbox +// - Multiple concurrent containers sharing one sandbox +// - Per-container Delete independence (does not tear down the sandbox) +// - WaitSandbox unblocks on stop +// - Protocol error cases (duplicate Create, Start before Create) +// - Resource release: no mount leaks after shutdown +// +// The suite is gated on the "sandbox" feature key; it is never skipped +// once enabled — every failure is a conformance failure. +type SandboxSuite struct { + cfg Config +} + +// NewSandboxSuite constructs a SandboxSuite from cfg. +func NewSandboxSuite(cfg Config) *SandboxSuite { + return &SandboxSuite{cfg: cfg} +} + +// Run runs every test in the suite as a subtest of t. +func (s *SandboxSuite) Run(t *testing.T) { + t.Helper() + registerShimLeakCheck(t, s.cfg.ShimBinary) + + t.Run("Lifecycle", s.testLifecycle) + t.Run("Platform", s.testPlatform) + t.Run("Ping", s.testPing) + t.Run("SingleContainer", s.testSingleContainer) + t.Run("MultipleContainers", s.testMultipleContainers) + t.Run("ContainerLifecycleIndependence", s.testContainerLifecycleIndependence) + t.Run("StatusAfterStop", s.testStatusAfterStop) + t.Run("WaitUnblocksOnStop", s.testWaitUnblocksOnStop) + t.Run("CreateTwiceRejected", s.testCreateTwiceRejected) + t.Run("StartWithoutCreateRejected", s.testStartWithoutCreateRejected) + t.Run("ResourceReleaseOnShutdown", s.testResourceReleaseOnShutdown) + + // Member-container workload contracts (exec, shared namespaces, + // volumes, networking). + t.Run("StatusReportsPidAndCreatedAt", s.testStatusReportsPidAndCreatedAt) + t.Run("HostNetworkNoNetworkSandbox", s.testHostNetworkNoNetworkSandbox) + t.Run("MemberContainerExec", s.testMemberContainerExec) + t.Run("CrossContainerViaUDS", s.testCrossContainerViaUDS) + t.Run("NetworkSandboxHeldOpen", s.testNetworkSandboxHeldOpen) + t.Run("NetworkSandboxPathInStatus", s.testNetworkSandboxPathInStatus) + t.Run("ContainerOutboundTCP", s.testContainerOutboundTCP) + t.Run("ContainerTrafficScopedToNetworkSandbox", s.testContainerTrafficScopedToNetworkSandbox) + t.Run("MemberContainersShareNetwork", s.testMemberContainersShareNetwork) + t.Run("MemberContainerHostVolume", s.testMemberContainerHostVolume) + t.Run("MemberContainersSharePID", s.testMemberContainersSharePID) + t.Run("MemberContainersShareIPC", s.testMemberContainersShareIPC) +} + +// testLifecycle drives the sandbox through the full lifecycle: +// +// CreateSandbox → StartSandbox → SandboxStatus(ready) → +// StopSandbox → SandboxStatus(stopped) → ShutdownSandbox +// +// The shim must transition through the expected states and the +// ShutdownSandbox RPC must succeed. +func (s *SandboxSuite) testLifecycle(t *testing.T) { + sandboxID := containerID(t) + env := startSandboxShim(t, s.cfg, sandboxID) + + // After StartSandbox the status must reflect a running sandbox. + status, err := env.sc.SandboxStatus(env.ctx, &sandboxAPI.SandboxStatusRequest{ + SandboxID: sandboxID, + }) + if err != nil { + t.Fatalf("SandboxStatus: %v", err) + } + if status.GetState() != sandboxStateReady { + t.Errorf("SandboxStatus state after start: got %q, want %q", status.GetState(), sandboxStateReady) + } + if status.GetPid() == 0 { + t.Error("SandboxStatus Pid must be > 0 after start") + } + t.Logf("sandbox running: state=%s pid=%d", status.GetState(), status.GetPid()) + + // StopSandbox must succeed and transition state to stopped. + if _, err := env.sc.StopSandbox(env.ctx, &sandboxAPI.StopSandboxRequest{ + SandboxID: sandboxID, + }); err != nil { + t.Fatalf("StopSandbox: %v", err) + } + + // Poll status: the shim must report SANDBOX_NOTREADY after stop. + if err := waitForSandboxStatus(env.ctx, env.sc, sandboxID, sandboxStateNotReady, 10*time.Second); err != nil { + t.Errorf("SandboxStatus state after stop: %v", err) + } + + // ShutdownSandbox must succeed even though the sandbox is already stopped. + if _, err := env.sc.ShutdownSandbox(env.ctx, &sandboxAPI.ShutdownSandboxRequest{ + SandboxID: sandboxID, + }); err != nil { + t.Fatalf("ShutdownSandbox: %v", err) + } + + t.Log("sandbox lifecycle complete") +} + +// testPlatform verifies that Platform returns a valid OS/architecture. +// The shim must always honour this RPC; containerd uses it to generate +// a correct OCI spec for member containers. +func (s *SandboxSuite) testPlatform(t *testing.T) { + sandboxID := containerID(t) + env := startSandboxShim(t, s.cfg, sandboxID) + + resp, err := env.sc.Platform(env.ctx, &sandboxAPI.PlatformRequest{SandboxID: sandboxID}) + if err != nil { + t.Fatalf("Platform: %v", err) + } + p := resp.GetPlatform() + if p == nil { + t.Fatal("Platform response missing platform field") + } + if p.GetOS() == "" { + t.Error("Platform response: OS must not be empty") + } + if p.GetArchitecture() == "" { + t.Error("Platform response: Architecture must not be empty") + } + t.Logf("platform: os=%s arch=%s variant=%s", p.GetOS(), p.GetArchitecture(), p.GetVariant()) +} + +// testPing verifies that PingSandbox succeeds while the sandbox is +// running. PingSandbox is a lightweight liveness check; it must +// return without error from a live sandbox. +func (s *SandboxSuite) testPing(t *testing.T) { + sandboxID := containerID(t) + env := startSandboxShim(t, s.cfg, sandboxID) + + if _, err := env.sc.PingSandbox(env.ctx, &sandboxAPI.PingRequest{SandboxID: sandboxID}); err != nil { + t.Fatalf("PingSandbox: %v", err) + } + t.Log("PingSandbox succeeded") +} + +// testSingleContainer verifies that a single member container can be +// created, started, and produce output inside the sandbox. +// +// The API contract: after StartSandbox, Task.Create on the shared +// connection must create a container that runs inside the sandbox. +// Task.Start must make the container's init process runnable, and +// Task.Wait must return the init process exit status. +func (s *SandboxSuite) testSingleContainer(t *testing.T) { + sandboxID := containerID(t) + env := startSandboxShim(t, s.cfg, sandboxID) + + cid := createContainerInSandbox(t, env, []string{"/bin/echo", "hello-sandbox"}) + + // Read output — the container must produce the expected string. + readContainerOutput(t, env, cid, "hello-sandbox", 30*time.Second) + + // Wait for the container's natural exit and verify exit status. + waitResp, err := env.tc.Wait(env.ctx, &taskAPI.WaitRequest{ID: cid}) + if err != nil { + t.Fatalf("Task.Wait: %v", err) + } + if waitResp.GetExitStatus() != 0 { + t.Errorf("expected exit status 0, got %d", waitResp.GetExitStatus()) + } + + // Delete the container task. + if _, err := env.tc.Delete(env.ctx, &taskAPI.DeleteRequest{ID: cid}); err != nil { + t.Fatalf("Task.Delete: %v", err) + } + t.Log("single container complete") +} + +// testMultipleContainers verifies that three member containers can run +// concurrently inside one sandbox, each receiving its own isolated +// rootfs and producing the expected output. +// +// The API contract: the sandbox must support N concurrent member +// containers. Each container's output must be independent; the sandbox +// VM must not be torn down when one container exits. +func (s *SandboxSuite) testMultipleContainers(t *testing.T) { + const n = 3 + sandboxID := containerID(t) + env := startSandboxShim(t, s.cfg, sandboxID) + + type ctrResult struct { + cid string + token string + } + + // Create all containers first, then collect output concurrently. + ctrs := make([]ctrResult, n) + for i := range ctrs { + token := fmt.Sprintf("ctr%d-token-%s", i, randomSuffix()) + cid := createContainerInSandbox(t, env, []string{"/bin/echo", token}) + ctrs[i] = ctrResult{cid: cid, token: token} + } + + // Verify each container's output in parallel. + var wg sync.WaitGroup + for _, cr := range ctrs { + wg.Add(1) + go func(cr ctrResult) { + defer wg.Done() + readContainerOutput(t, env, cr.cid, cr.token, 30*time.Second) + }(cr) + } + wg.Wait() + + // All containers should have exited cleanly by now. + for _, cr := range ctrs { + waitResp, err := env.tc.Wait(env.ctx, &taskAPI.WaitRequest{ID: cr.cid}) + if err != nil { + t.Errorf("Task.Wait %s: %v", cr.cid, err) + continue + } + if waitResp.GetExitStatus() != 0 { + t.Errorf("container %s exit status: got %d, want 0", cr.cid, waitResp.GetExitStatus()) + } + env.tc.Delete(env.ctx, &taskAPI.DeleteRequest{ID: cr.cid}) //nolint:errcheck + } + + t.Logf("all %d containers completed", n) +} + +// testContainerLifecycleIndependence verifies that deleting one member +// container leaves the sandbox and other containers running. +// +// The API contract: Task.Delete for a member container must clean up +// that container's resources without affecting the sandbox VM or any +// other member containers. +func (s *SandboxSuite) testContainerLifecycleIndependence(t *testing.T) { + sandboxID := containerID(t) + env := startSandboxShim(t, s.cfg, sandboxID) + + // Start a long-lived "observer" container. + observerCID := createContainerInSandbox(t, env, []string{"/bin/forever", "observer"}) + + // Start a short-lived container that exits naturally. + shortCID := createContainerInSandbox(t, env, []string{"/bin/echo", "short-lived"}) + readContainerOutput(t, env, shortCID, "short-lived", 30*time.Second) + + // Wait for the short container to exit and delete it. + env.tc.Wait(env.ctx, &taskAPI.WaitRequest{ID: shortCID}) //nolint:errcheck + if _, err := env.tc.Delete(env.ctx, &taskAPI.DeleteRequest{ID: shortCID}); err != nil { + t.Fatalf("Delete short container: %v", err) + } + + // The observer container must still be running. + stateResp, err := env.tc.State(env.ctx, &taskAPI.StateRequest{ID: observerCID}) + if err != nil { + t.Fatalf("State for observer after short-container delete: %v", err) + } + if stateResp.GetStatus() != tasktypes.Status_RUNNING { + t.Errorf("observer container status after peer delete: got %v, want RUNNING", stateResp.GetStatus()) + } + + // The sandbox itself must also still be running. + status, err := env.sc.SandboxStatus(env.ctx, &sandboxAPI.SandboxStatusRequest{SandboxID: sandboxID}) + if err != nil { + t.Fatalf("SandboxStatus after peer delete: %v", err) + } + if status.GetState() != sandboxStateReady { + t.Errorf("sandbox state after peer-container delete: got %q, want %q", status.GetState(), sandboxStateReady) + } + + t.Log("observer still running after peer delete; sandbox intact") + + // Clean up the observer. + env.tc.Kill(env.ctx, &taskAPI.KillRequest{ID: observerCID, Signal: uint32(syscall.SIGKILL), All: true}) //nolint:errcheck + env.tc.Wait(env.ctx, &taskAPI.WaitRequest{ID: observerCID}) //nolint:errcheck + env.tc.Delete(env.ctx, &taskAPI.DeleteRequest{ID: observerCID}) //nolint:errcheck +} + +// testStatusAfterStop verifies that SandboxStatus after StopSandbox +// reports a non-ready state, and that a second StopSandbox is +// idempotent (must not error). +func (s *SandboxSuite) testStatusAfterStop(t *testing.T) { + sandboxID := containerID(t) + env := startSandboxShim(t, s.cfg, sandboxID) + + // First stop. + if _, err := env.sc.StopSandbox(env.ctx, &sandboxAPI.StopSandboxRequest{SandboxID: sandboxID}); err != nil { + t.Fatalf("StopSandbox (first): %v", err) + } + + // Status must not be sandboxStateReady after stop. + status, err := env.sc.SandboxStatus(env.ctx, &sandboxAPI.SandboxStatusRequest{SandboxID: sandboxID}) + if err != nil { + t.Logf("SandboxStatus after stop returned error (may be acceptable): %v", err) + } else if status.GetState() == sandboxStateReady { + t.Errorf("SandboxStatus after stop: state is still %q; shim must not report ready after stop", status.GetState()) + } + + // Second stop must be idempotent — must not return an error. + if _, err := env.sc.StopSandbox(env.ctx, &sandboxAPI.StopSandboxRequest{SandboxID: sandboxID}); err != nil { + t.Errorf("StopSandbox (second, idempotency check): %v", err) + } + + t.Log("status-after-stop and idempotency checks passed") +} + +// testWaitUnblocksOnStop verifies that WaitSandbox returns after the +// sandbox is stopped. +// +// The API contract: WaitSandbox must unblock when the sandbox exits +// (via StopSandbox or ShutdownSandbox). Callers rely on this to +// detect sandbox death. +func (s *SandboxSuite) testWaitUnblocksOnStop(t *testing.T) { + sandboxID := containerID(t) + env := startSandboxShim(t, s.cfg, sandboxID) + + waitDone := make(chan error, 1) + go func() { + _, err := env.sc.WaitSandbox(env.ctx, &sandboxAPI.WaitSandboxRequest{SandboxID: sandboxID}) + waitDone <- err + }() + + // Give WaitSandbox a moment to start blocking. + time.Sleep(200 * time.Millisecond) + + if _, err := env.sc.StopSandbox(env.ctx, &sandboxAPI.StopSandboxRequest{SandboxID: sandboxID}); err != nil { + t.Fatalf("StopSandbox: %v", err) + } + + select { + case err := <-waitDone: + if err != nil { + t.Logf("WaitSandbox returned error after stop (may be acceptable for ttrpc shutdown): %v", err) + } else { + t.Log("WaitSandbox returned cleanly after stop") + } + case <-time.After(15 * time.Second): + t.Fatal("WaitSandbox did not return within 15s after StopSandbox") + } +} + +// testCreateTwiceRejected verifies that calling CreateSandbox a second +// time on the same shim returns an AlreadyExists error. +// +// The API contract: a sandbox shim process hosts exactly one sandbox. +// A second CreateSandbox must be rejected with AlreadyExists. +func (s *SandboxSuite) testCreateTwiceRejected(t *testing.T) { + sandboxID := containerID(t) + env := startSandboxShim(t, s.cfg, sandboxID) + + _, err := env.sc.CreateSandbox(env.ctx, &sandboxAPI.CreateSandboxRequest{ + SandboxID: sandboxID + "-dup", + BundlePath: ".", + }) + if err == nil { + t.Fatal("second CreateSandbox must fail; got nil error") + } + errStr := strings.ToLower(err.Error()) + if !strings.Contains(errStr, "already exists") && !strings.Contains(errStr, "alreadyexists") { + t.Errorf("second CreateSandbox: expected AlreadyExists error, got: %v", err) + } + t.Logf("second CreateSandbox correctly rejected: %v", err) +} + +// testStartWithoutCreateRejected verifies that StartSandbox before +// CreateSandbox fails with FailedPrecondition. +// +// The API contract: CreateSandbox must precede StartSandbox. The shim +// must reject StartSandbox if CreateSandbox has not been called first. +func (s *SandboxSuite) testStartWithoutCreateRejected(t *testing.T) { + shimBin, bundleDir, _ := shimSetup(t, s.cfg) + sandboxID := containerID(t) + ns := uniqueTestNamespace(t, "sandbox") + ctx := namespaces.WithNamespace(t.Context(), ns) + + // The shim reads config.json from its working directory for the + // grouping label. Write a minimal sandbox spec so the shim can start. + writeSandboxOCISpec(t, bundleDir) + + // Start a fresh shim without calling CreateSandbox. + params := startShim(t, shimBin, bundleDir, sandboxID, ns, s.cfg) + conn := connectShim(t, params.Address) + client := ttrpc.NewClient(conn) + defer client.Close() + + sc := sandboxAPI.NewTTRPCSandboxClient(client) + tc := taskAPI.NewTTRPCTaskClient(client) + + _, err := sc.StartSandbox(ctx, &sandboxAPI.StartSandboxRequest{SandboxID: sandboxID}) + if err == nil { + t.Fatal("StartSandbox without CreateSandbox must fail; got nil error") + } + errStr := strings.ToLower(err.Error()) + if !strings.Contains(errStr, "precondition") && !strings.Contains(errStr, "failed_precondition") { + t.Errorf("StartSandbox without create: expected FailedPrecondition, got: %v", err) + } + t.Logf("StartSandbox before CreateSandbox correctly rejected: %v", err) + + // Shut the shim down cleanly. + shutdownTask(ctx, tc, sandboxID) +} + +// testResourceReleaseOnShutdown verifies that ShutdownSandbox releases +// per-container host resources. Specifically, if the shim uses a +// shared virtiofs directory, paths under that directory must not appear +// as mount points in the shim's mount namespace after shutdown. +// +// The API contract: ShutdownSandbox must release all resources +// allocated for the sandbox and its member containers. Leaked mount +// points can prevent bundle-directory cleanup and exhaust kernel mount +// table entries. +func (s *SandboxSuite) testResourceReleaseOnShutdown(t *testing.T) { + sandboxID := containerID(t) + env := startSandboxShim(t, s.cfg, sandboxID) + + // Run two member containers to create per-container rootfs mounts. + cid1 := createContainerInSandbox(t, env, []string{"/bin/echo", "ctr1"}) + cid2 := createContainerInSandbox(t, env, []string{"/bin/echo", "ctr2"}) + + readContainerOutput(t, env, cid1, "ctr1", 30*time.Second) + readContainerOutput(t, env, cid2, "ctr2", 30*time.Second) + + env.tc.Wait(env.ctx, &taskAPI.WaitRequest{ID: cid1}) //nolint:errcheck + env.tc.Wait(env.ctx, &taskAPI.WaitRequest{ID: cid2}) //nolint:errcheck + env.tc.Delete(env.ctx, &taskAPI.DeleteRequest{ID: cid1}) //nolint:errcheck + env.tc.Delete(env.ctx, &taskAPI.DeleteRequest{ID: cid2}) //nolint:errcheck + + // Capture the shim PID before shutdown so we can inspect its + // namespace after. Use a probe container-ID; Connect returns the + // shim PID regardless of which ID is used on some shims. + shimPID := sandboxShimPID(env, cid1) + mountsBefore := sandboxContainersMounts(shimPID) + t.Logf("shim PID: %d, container mounts before shutdown: %v", shimPID, mountsBefore) + + // Stop and shut down the sandbox. + env.sc.StopSandbox(env.ctx, &sandboxAPI.StopSandboxRequest{SandboxID: sandboxID}) //nolint:errcheck + env.sc.ShutdownSandbox(env.ctx, &sandboxAPI.ShutdownSandboxRequest{SandboxID: sandboxID}) //nolint:errcheck + + // Give the shim time to clean up. + time.Sleep(500 * time.Millisecond) + + // After shutdown, no per-container mounts should remain in the + // shim's namespace. We check the shim's /proc//mountinfo + // if the shim runs in a private mount namespace; if the shim + // exited (pid gone) that is also a clean result. + mountsAfter := sandboxContainersMounts(shimPID) + if len(mountsAfter) > 0 { + t.Errorf("shim left %d per-container mount(s) after ShutdownSandbox: %v", + len(mountsAfter), mountsAfter) + } else { + t.Log("no per-container mounts remain after shutdown") + } +} + +// Ensure ttrpc import is used (consumed in testStartWithoutCreateRejected). +var _ *ttrpc.Client diff --git a/sandbox_suite_member.go b/sandbox_suite_member.go new file mode 100644 index 0000000..4f7df79 --- /dev/null +++ b/sandbox_suite_member.go @@ -0,0 +1,301 @@ +//go:build linux + +/* + Copyright The containerd Authors. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +// This file contains SandboxSuite tests that verify the shim API contract +// for member-container workloads: status fields, host-network sandboxes, +// exec, shared endpoints, and outbound networking. Every test is framed +// in terms of the shim API specification, not any particular +// implementation. + +package shimtest + +import ( + "net" + "os" + "strings" + "syscall" + "testing" + "time" + + sandboxAPI "github.com/containerd/containerd/api/runtime/sandbox/v1" + taskAPI "github.com/containerd/containerd/api/runtime/task/v3" + specs "github.com/opencontainers/runtime-spec/specs-go" +) + +// testStatusReportsPidAndCreatedAt verifies that SandboxStatus always returns +// a non-zero pid and a non-zero created_at timestamp. +// +// The API contract: a caller must be able to reference the sandbox's +// namespaces (e.g. /proc//ns/*) and report the sandbox's age, so a +// shim must populate a non-zero pid and a non-zero created_at after a +// successful StartSandbox. +func (s *SandboxSuite) testStatusReportsPidAndCreatedAt(t *testing.T) { + sandboxID := containerID(t) + env := startSandboxShim(t, s.cfg, sandboxID) + + _, info := sandboxStatusInfo(t, env) + + resp, err := env.sc.SandboxStatus(env.ctx, &sandboxAPI.SandboxStatusRequest{ + SandboxID: sandboxID, + }) + if err != nil { + t.Fatalf("SandboxStatus: %v", err) + } + + if resp.GetPid() == 0 { + t.Error("SandboxStatus.Pid must be non-zero after StartSandbox") + } + ts := resp.GetCreatedAt() + if ts == nil || ts.AsTime().IsZero() { + t.Error("SandboxStatus.CreatedAt must be non-zero after StartSandbox") + } + if info["pid"] == "" || info["pid"] == "0" { + t.Errorf("SandboxStatus.Info[pid] must be a non-zero pid string, got %q", info["pid"]) + } + if info["state"] == "" { + t.Errorf("SandboxStatus.Info[state] must not be empty, got %q", info["state"]) + } + t.Logf("status ok: pid=%d created_at=%s info=%v", resp.GetPid(), resp.GetCreatedAt().AsTime(), info) +} + +// testHostNetworkNoNetworkSandbox verifies that a sandbox created with an +// empty NetnsPath (i.e. no network sandbox provided) succeeds and that +// member containers run normally. +// +// The API contract: an empty netns_path in CreateSandboxRequest means the +// sandbox uses the host's network stack (no isolation). The shim must accept +// this case without error and member containers must run successfully. +func (s *SandboxSuite) testHostNetworkNoNetworkSandbox(t *testing.T) { + sandboxID := containerID(t) + env := startSandboxShimWithNetworkSandbox(t, s.cfg, sandboxID, "") + + cid := createContainerInSandbox(t, env, []string{"/bin/echo", "host-network-ok"}) + readContainerOutput(t, env, cid, "host-network-ok", 30*time.Second) + + waitResp, err := env.tc.Wait(env.ctx, &taskAPI.WaitRequest{ID: cid}) + if err != nil { + t.Fatalf("Task.Wait: %v", err) + } + if waitResp.GetExitStatus() != 0 { + t.Errorf("container exit status: got %d, want 0", waitResp.GetExitStatus()) + } + env.tc.Delete(env.ctx, &taskAPI.DeleteRequest{ID: cid}) //nolint:errcheck + t.Log("host-network sandbox: member container ran successfully") +} + +// testMemberContainerExec verifies that a process can be exec'd into a running +// member container and that its output and exit status are correctly propagated. +// +// The API contract: after Task.Create + Task.Start, a shim must accept +// Task.Exec to run an additional process inside the container. The exec +// process must run inside the container's namespace and filesystem, its +// output must arrive on the configured stdio path, and Task.Wait on the +// ExecID must return the correct exit status. +// +// This contract underpins any exec-into-a-running-container use case +// (e.g. interactive exec, health/liveness probes, sidecar tooling). +func (s *SandboxSuite) testMemberContainerExec(t *testing.T) { + sandboxID := containerID(t) + env := startSandboxShim(t, s.cfg, sandboxID) + + // Start a long-lived container to exec into. + observerCID := createContainerInSandbox(t, env, []string{"/bin/forever", "exec-target"}) + + const token = "exec-probe-ok" + out, exitStatus := execInSandboxContainer(t, env, observerCID, []string{"/bin/echo", token}, 30*time.Second) + if !strings.Contains(out, token) { + t.Errorf("exec output: want %q in output, got %q", token, out) + } + if exitStatus != 0 { + t.Errorf("exec exit status: got %d, want 0", exitStatus) + } + + // Verify non-zero exit status propagation. + _, nonZeroStatus := execInSandboxContainer(t, env, observerCID, []string{"/bin/exit", "42"}, 30*time.Second) + if nonZeroStatus != 42 { + t.Errorf("exec exit-code propagation: got %d, want 42", nonZeroStatus) + } + + env.tc.Kill(env.ctx, &taskAPI.KillRequest{ID: observerCID, Signal: uint32(syscall.SIGKILL), All: true}) //nolint:errcheck + env.tc.Wait(env.ctx, &taskAPI.WaitRequest{ID: observerCID}) //nolint:errcheck + env.tc.Delete(env.ctx, &taskAPI.DeleteRequest{ID: observerCID}) //nolint:errcheck + t.Log("exec in sandbox member container: ok") +} + +// testCrossContainerViaUDS verifies that two member containers in one sandbox +// can both reach a shared host-side UNIX domain socket endpoint by exec'ing +// into a container that has the socket forwarded into its filesystem. +// +// The API contract: a member container that has a "uds" mount type in its +// OCI spec must receive the corresponding host-side UNIX socket forwarded into +// its filesystem. A process exec'd into the container must be able to connect +// to that socket. This is the general contract behind any shared, pre-forwarded +// host endpoint: multiple processes in a sandbox must be able to reach the +// same endpoint through it. +// +// Test topology: +// +// host UNIX socket listener (the shared endpoint) +// └── forwarded into the shared container at /run/shared.sock +// ├── exec A: nc -U /run/shared.sock (connects, verified by host accept) +// └── exec B: nc -U /run/shared.sock (connects, verified by host accept) +// +// Using exec into a single container avoids the per-container socket-forward +// routing ambiguity that arises when multiple containers each have their own +// accept stream. +func (s *SandboxSuite) testCrossContainerViaUDS(t *testing.T) { + sandboxID := containerID(t) + env := startSandboxShim(t, s.cfg, sandboxID) + + // Create a host-side UNIX socket listener (the shared endpoint). + hostSockPath, err := makeUnixSockPath(t) + if err != nil { + t.Fatalf("create host sock path: %v", err) + } + ln, err := net.Listen("unix", hostSockPath) + if err != nil { + t.Fatalf("host unix listen: %v", err) + } + t.Cleanup(func() { ln.Close() }) + + const containerSockPath = "/run/shared.sock" + + // Accept connections from the container and immediately close them. + // Closing causes nc to see EOF on the socket and exit cleanly. + hostDone := make(chan error, 2) + acceptOne := func() { + conn, err := ln.Accept() + if err != nil { + hostDone <- err + return + } + conn.Close() + hostDone <- nil + } + go acceptOne() + go acceptOne() + + // Start a long-lived container with the host socket forwarded into it. + sharedCID := createContainerInSandbox(t, env, + []string{"/bin/forever", "uds-shared-container"}, + withSandboxCtrExtraMounts(specs.Mount{ + Type: "uds", + Source: hostSockPath, + Destination: containerSockPath, + }), + ) + + // Exec nc twice into the shared container; each connection proves that + // a process running in the container can reach the shared host endpoint. + // These model two different processes reaching a common host-forwarded + // endpoint, as multiple containers in a sandbox would. + for i := 0; i < 2; i++ { + out, exitCode := execInSandboxContainer( + t, env, sharedCID, + []string{"/bin/nc", "-U", containerSockPath}, + 30*time.Second, + ) + if exitCode != 0 { + t.Errorf("nc exec %d: exit code %d, output: %q", i+1, exitCode, out) + } + // The host must have seen a connection for this exec. + select { + case err := <-hostDone: + if err != nil { + t.Fatalf("host accept connection %d: %v", i+1, err) + } + t.Logf("cross-container UDS connection %d: ok", i+1) + case <-time.After(5 * time.Second): + t.Fatalf("host did not see connection %d within 5s", i+1) + } + } + + // Kill the shared container. + env.tc.Kill(env.ctx, &taskAPI.KillRequest{ID: sharedCID, Signal: uint32(syscall.SIGKILL), All: true}) //nolint:errcheck + env.tc.Wait(env.ctx, &taskAPI.WaitRequest{ID: sharedCID}) //nolint:errcheck + env.tc.Delete(env.ctx, &taskAPI.DeleteRequest{ID: sharedCID}) //nolint:errcheck + t.Log("cross-container UDS: both execs reached the shared host endpoint") +} + +// testContainerOutboundTCP verifies that a process running inside a member +// container has a working outbound network path by resolving a real +// external hostname. +// +// The API contract: a shim must give member containers a working network +// stack, regardless of the mechanism used to provide it (native network +// namespace membership, a virtual NIC, or any other in-guest networking +// approach). This test does not care how connectivity is achieved — only +// that a container can reach a resolver and get back a valid answer, +// exactly as any container workload that depends on DNS would. +// +// DNS resolution (rather than a raw TCP round trip) is used here because +// Task.Exec — the mechanism this suite uses to run a process inside an +// already-running member container — has no stdin plumbing, and a TCP +// round trip needs a way to send data. /bin/host takes its input purely +// from argv and writes its result to stdout, so it fits Task.Exec's +// existing capabilities. NetworkSuite (legacy path) separately covers TCP +// and UDP round trips end-to-end. +func (s *SandboxSuite) testContainerOutboundTCP(t *testing.T) { + sandboxID := containerID(t) + env := startSandboxShim(t, s.cfg, sandboxID) + + cid := createContainerInSandbox(t, env, []string{"/bin/forever", "outbound-container"}) + + // host : prints " has address " for each resolved address. + out, exitStatus := execInSandboxContainer(t, env, cid, []string{"/bin/host", dnsTestHostname}, 30*time.Second) + if exitStatus != 0 { + t.Fatalf("host exec exit status: got %d, want 0; output: %q", exitStatus, out) + } + + var addrs []string + for _, line := range strings.Split(strings.TrimSpace(out), "\n") { + line = strings.TrimSpace(line) + if line == "" { + continue + } + const marker = " has address " + idx := strings.Index(line, marker) + if idx < 0 { + t.Errorf("host %s: unexpected output line %q", dnsTestHostname, line) + continue + } + ip := line[idx+len(marker):] + if net.ParseIP(ip) == nil { + t.Errorf("host %s: %q is not a valid IP address", dnsTestHostname, ip) + continue + } + addrs = append(addrs, ip) + } + if len(addrs) == 0 { + t.Fatalf("host %s produced no addresses; output: %q", dnsTestHostname, out) + } + + t.Log("container outbound DNS resolution: ok, addresses:", addrs) +} + +// makeUnixSockPath returns a UNIX socket path under a temp directory that +// satisfies the 104-byte AF_UNIX path limit on macOS. +func makeUnixSockPath(tb testing.TB) (string, error) { + tb.Helper() + dir, err := os.MkdirTemp(unixSafeDir(), "nb-uds-") + if err != nil { + return "", err + } + tb.Cleanup(func() { os.RemoveAll(dir) }) + return dir + "/shared.sock", nil +} diff --git a/sandbox_suite_member_linux.go b/sandbox_suite_member_linux.go new file mode 100644 index 0000000..e2a3acc --- /dev/null +++ b/sandbox_suite_member_linux.go @@ -0,0 +1,103 @@ +//go:build linux + +/* + Copyright The containerd Authors. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +// This file contains SandboxSuite member-container workload tests that +// require Linux kernel features (network namespaces). + +package shimtest + +import ( + "testing" + "time" + + sandboxAPI "github.com/containerd/containerd/api/runtime/sandbox/v1" + taskAPI "github.com/containerd/containerd/api/runtime/task/v3" +) + +// testNetworkSandboxHeldOpen verifies that the shim holds the host-side +// network sandbox open for the lifetime of the sandbox and releases it after +// the sandbox stops. +// +// The API contract: a shim that receives a non-empty netns_path in +// CreateSandboxRequest must pin the network sandbox resource (e.g. a Linux +// network namespace bind-mount) for the duration of the sandbox. This allows +// CNI and other host-side tools to inspect or manipulate the network sandbox +// while the sandbox is running. After StopSandbox the shim must release its +// pin so that the caller can unmount the bind-mount and reclaim the resource. +// +// The test creates a real host-side network sandbox (bind-mounted netns), +// passes its path to CreateSandbox, asserts the path is reachable while the +// sandbox is ready, then stops the sandbox and verifies the state transition. +func (s *SandboxSuite) testNetworkSandboxHeldOpen(t *testing.T) { + nsPath := createNetworkSandbox(t) + + sandboxID := containerID(t) + env := startSandboxShimWithNetworkSandbox(t, s.cfg, sandboxID, nsPath) + + // While the sandbox is running the bind-mount must still be reachable. + // A missing path means the shim (or something else) unmounted it + // prematurely — violating the hold-open contract. + if !networkSandboxIsOpen(nsPath) { + t.Fatal("network sandbox disappeared while sandbox is running; shim must hold it open") + } + t.Logf("network sandbox %q is reachable while sandbox is running", nsPath) + + // Run a member container to confirm the sandbox is fully operational. + cid := createContainerInSandbox(t, env, []string{"/bin/echo", "netns-ok"}) + readContainerOutput(t, env, cid, "netns-ok", 30*time.Second) + env.tc.Wait(env.ctx, &taskAPI.WaitRequest{ID: cid}) //nolint:errcheck + env.tc.Delete(env.ctx, &taskAPI.DeleteRequest{ID: cid}) //nolint:errcheck + + // Stop the sandbox explicitly so we can inspect the final state. + if _, err := env.sc.StopSandbox(env.ctx, &sandboxAPI.StopSandboxRequest{ + SandboxID: sandboxID, + }); err != nil { + t.Fatalf("StopSandbox: %v", err) + } + + if err := waitForSandboxStatus(env.ctx, env.sc, sandboxID, sandboxStateNotReady, 10*time.Second); err != nil { + t.Errorf("SandboxStatus state after stop: %v", err) + } + t.Log("network sandbox held open while running; sandbox stopped cleanly") +} + +// testNetworkSandboxPathInStatus verifies that SandboxStatus may report the +// network sandbox path in its Info map under "networkSandboxPath". +// +// The API contract: the base sandbox TTRPC protocol does not mandate specific +// Info keys. A shim that accepts a network sandbox path is encouraged to +// expose it in Info so callers can confirm which network resource is pinned +// without side-channel lookups. The test treats absence of the key as an +// informational result rather than a hard failure. +func (s *SandboxSuite) testNetworkSandboxPathInStatus(t *testing.T) { + nsPath := createNetworkSandbox(t) + + sandboxID := containerID(t) + env := startSandboxShimWithNetworkSandbox(t, s.cfg, sandboxID, nsPath) + + _, info := sandboxStatusInfo(t, env) + reported := info["networkSandboxPath"] + if reported == "" { + t.Logf("SandboxStatus.Info does not include 'networkSandboxPath' (optional field); info=%v", info) + return + } + if reported != nsPath { + t.Errorf("SandboxStatus.Info[networkSandboxPath]: got %q, want %q", reported, nsPath) + } + t.Logf("SandboxStatus.Info[networkSandboxPath]=%q (matches provided path)", reported) +} diff --git a/sandbox_suite_netns_linux.go b/sandbox_suite_netns_linux.go new file mode 100644 index 0000000..f2eeb3e --- /dev/null +++ b/sandbox_suite_netns_linux.go @@ -0,0 +1,106 @@ +//go:build linux + +/* + Copyright The containerd Authors. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +// This file contains a SandboxSuite test that verifies container network +// traffic is actually scoped to the network sandbox the shim was given, as +// opposed to merely holding the resource's path open. It requires root to +// create a real network namespace and network interfaces, and is skipped +// otherwise. + +package shimtest + +import ( + "testing" + "time" + + taskAPI "github.com/containerd/containerd/api/runtime/task/v3" +) + +// testContainerTrafficScopedToNetworkSandbox verifies that when a sandbox is +// given a network sandbox (a non-empty netns_path in CreateSandboxRequest), +// a member container's outbound network traffic actually originates from +// within that network sandbox, rather than from whatever network context +// the shim process happens to run in. +// +// The API contract: passing a non-empty netns_path in CreateSandboxRequest +// must result in member container traffic being scoped to that network +// sandbox — regardless of the mechanism the shim uses internally (native +// namespace membership, a virtual NIC bridged into the namespace, or any +// other approach). This test only observes the externally visible result: +// a container must be able to reach an endpoint that exists only inside the +// provided network sandbox, exactly as any container workload reaching a +// service scoped to that network sandbox would. +// +// Test topology: a veth pair is created with both ends inside a real network +// namespace, giving one end an address on a subnet that has no route from +// outside that namespace (see realNetworkSandbox.attachVeth). A listener is +// bound to that address from inside the namespace. The sandbox is started +// with the namespace's path, and a member container runs nc(1) in TCP mode +// (nc ) to reach the listener's address, sending a token via +// its stdin FIFO and printing the echoed response to stdout. Since the +// address is unreachable from any context other than the namespace itself, +// a successful round trip is only possible if the container's traffic +// actually originates there. +// +// Requires root (CAP_SYS_ADMIN) to create a network namespace and attach +// network interfaces; skipped otherwise. +func (s *SandboxSuite) testContainerTrafficScopedToNetworkSandbox(t *testing.T) { + netns := createRealNetworkSandbox(t) + addr := netns.attachVeth(t) + const port = "9191" + endpoint := addr + ":" + port + + // Sanity check: confirm the address really is unreachable from the + // namespace this test process runs in, so that a later successful + // connection can only be explained by the container's traffic + // originating inside the sandbox namespace. + probeUnreachableFromCurrentNamespace(t, endpoint) + + done := listenAndEchoOnceInNetns(t, netns.path, endpoint) + + sandboxID := containerID(t) + env := startSandboxShimWithNetworkSandbox(t, s.cfg, sandboxID, netns.path) + + const token = "netns-scoped-ok" + cid := createContainerInSandbox(t, env, []string{"/bin/nc", addr, port}, withSandboxCtrStdin()) + writeContainerStdin(t, env, cid, token+"\n") + readContainerOutput(t, env, cid, token, 30*time.Second) + + waitResp, err := env.tc.Wait(env.ctx, &taskAPI.WaitRequest{ID: cid}) + if err != nil { + t.Fatalf("Task.Wait: %v", err) + } + if waitResp.GetExitStatus() != 0 { + t.Fatalf("container exit status: got %d, want 0 "+ + "(container could not reach the network-sandbox-scoped endpoint; "+ + "its traffic may not be originating inside the provided network sandbox)", + waitResp.GetExitStatus()) + } + env.tc.Delete(env.ctx, &taskAPI.DeleteRequest{ID: cid}) //nolint:errcheck + + select { + case err := <-done: + if err != nil { + t.Fatalf("network-sandbox-scoped endpoint did not observe a successful exchange: %v", err) + } + case <-time.After(5 * time.Second): + t.Fatal("network-sandbox-scoped endpoint did not observe a connection from the container within 5s") + } + + t.Log("container traffic is scoped to the provided network sandbox") +} diff --git a/sandbox_suite_other.go b/sandbox_suite_other.go new file mode 100644 index 0000000..2f73c69 --- /dev/null +++ b/sandbox_suite_other.go @@ -0,0 +1,37 @@ +//go:build !linux + +/* + Copyright The containerd Authors. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +package shimtest + +import "testing" + +// SandboxSuite is the sandbox conformance suite. On non-Linux +// platforms the suite is not supported and every test is skipped. +type SandboxSuite struct { + cfg Config +} + +// NewSandboxSuite constructs a SandboxSuite. +func NewSandboxSuite(cfg Config) *SandboxSuite { + return &SandboxSuite{cfg: cfg} +} + +// Run skips the entire suite on non-Linux platforms. +func (s *SandboxSuite) Run(t *testing.T) { + t.Skip("SandboxSuite is Linux-only (virtiofs-backed shared filesystem)") +} diff --git a/sandbox_suite_shared_ipcns_linux.go b/sandbox_suite_shared_ipcns_linux.go new file mode 100644 index 0000000..ca04cd9 --- /dev/null +++ b/sandbox_suite_shared_ipcns_linux.go @@ -0,0 +1,94 @@ +//go:build linux + +/* + Copyright The containerd Authors. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +package shimtest + +import ( + "strings" + "testing" + "time" + + taskAPI "github.com/containerd/containerd/api/runtime/task/v3" + specs "github.com/opencontainers/runtime-spec/specs-go" +) + +// testMemberContainersShareIPC verifies that member containers of the +// same sandbox can share an IPC namespace: a SysV shared memory segment +// created by one member container is visible — by its well-known key — +// to a second, independently created member container. +// +// SysV IPC objects are chosen (rather than, say, a shared file) because +// their visibility is governed entirely by the process's IPC namespace, +// independent of mount namespace or any bind-mounted directory. A +// successful cross-container round trip through the same key is +// conclusive proof of a shared IPC namespace specifically, not an +// artifact of some other sharing mechanism. +// +// The API contract: when a member container's OCI spec carries a host +// path on its IPC namespace entry, the shim must place that container +// in an IPC namespace shared with its sandbox peers (e.g. this is how a +// caller expresses Kubernetes' default of always sharing one IPC +// namespace across a pod's containers). This test only observes the +// externally visible result and does not assume any particular +// mechanism a shim uses to provide it. It intentionally uses a +// placeholder host path (see withSandboxCtrNamespace) since only a +// live host has an actual sandbox PID to put there. +func (s *SandboxSuite) testMemberContainersShareIPC(t *testing.T) { + sandboxID := containerID(t) + env := startSandboxShim(t, s.cfg, sandboxID) + + const ( + shmKey = "424242" + marker = "shared-ipcns-ok" + ) + + writerCID := createContainerInSandbox(t, env, []string{"/bin/shmwrite", shmKey, marker}, + withSandboxCtrNamespace(specs.IPCNamespace, "/proc/1/ns/ipc")) + + writerWait, err := env.tc.Wait(env.ctx, &taskAPI.WaitRequest{ID: writerCID}) + if err != nil { + t.Fatalf("Task.Wait writer: %v", err) + } + if writerWait.GetExitStatus() != 0 { + t.Fatalf("writer container exit status: got %d, want 0", writerWait.GetExitStatus()) + } + env.tc.Delete(env.ctx, &taskAPI.DeleteRequest{ID: writerCID}) //nolint:errcheck + + // The shm segment created by the writer persists in the IPC + // namespace after the writer container exits (nothing calls + // IPC_RMID on it), so there is no ordering requirement beyond the + // writer having already exited. + readerCID := createContainerInSandbox(t, env, []string{"/bin/shmread", shmKey}, + withSandboxCtrNamespace(specs.IPCNamespace, "/proc/1/ns/ipc")) + + out := readContainerOutput(t, env, readerCID, marker, 30*time.Second) + if !strings.Contains(out, marker) { + t.Fatalf("shmread output did not contain marker %q: %q", marker, out) + } + + readerWait, err := env.tc.Wait(env.ctx, &taskAPI.WaitRequest{ID: readerCID}) + if err != nil { + t.Fatalf("Task.Wait reader: %v", err) + } + if readerWait.GetExitStatus() != 0 { + t.Fatalf("reader container exit status: got %d, want 0", readerWait.GetExitStatus()) + } + env.tc.Delete(env.ctx, &taskAPI.DeleteRequest{ID: readerCID}) //nolint:errcheck + + t.Log("member containers share an IPC namespace: shared memory segment visible across containers") +} diff --git a/sandbox_suite_shared_netns_linux.go b/sandbox_suite_shared_netns_linux.go new file mode 100644 index 0000000..1135c6b --- /dev/null +++ b/sandbox_suite_shared_netns_linux.go @@ -0,0 +1,79 @@ +//go:build linux + +/* + Copyright The containerd Authors. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +package shimtest + +import ( + "testing" + "time" + + taskAPI "github.com/containerd/containerd/api/runtime/task/v3" +) + +// testMemberContainersShareNetwork verifies that member containers of the +// same sandbox share a network stack: a listener started by one member +// container is reachable from a second, independently created member +// container via loopback, with no explicit network configuration on either +// container. +// +// The API contract: a shim's sandbox model requires all member containers +// of one sandbox to share a single network identity (e.g. this is what +// backs a Kubernetes pod's shared network namespace). This test only +// observes the externally visible result of that contract — +// cross-container loopback connectivity — and does not assume any +// particular mechanism a shim uses to provide it (a real shared network +// namespace, a shared virtual interface, or any other approach). +func (s *SandboxSuite) testMemberContainersShareNetwork(t *testing.T) { + sandboxID := containerID(t) + env := startSandboxShim(t, s.cfg, sandboxID) + + const port = "9192" + + listenerCID := createContainerInSandbox(t, env, []string{"/bin/echosrv", port}) + + // Give the listener container a moment to actually bind and start + // accepting before the client attempts to connect. There is no + // synchronous "ready" signal available across two containers created + // independently via the Task API. + time.Sleep(200 * time.Millisecond) + + const token = "shared-netns-ok" + clientCID := createContainerInSandbox(t, env, []string{"/bin/nc", "127.0.0.1", port}, withSandboxCtrStdin()) + writeContainerStdin(t, env, clientCID, token) + readContainerOutput(t, env, clientCID, token, 30*time.Second) + + clientWait, err := env.tc.Wait(env.ctx, &taskAPI.WaitRequest{ID: clientCID}) + if err != nil { + t.Fatalf("Task.Wait client: %v", err) + } + if clientWait.GetExitStatus() != 0 { + t.Fatalf("client container exit status: got %d, want 0", clientWait.GetExitStatus()) + } + env.tc.Delete(env.ctx, &taskAPI.DeleteRequest{ID: clientCID}) //nolint:errcheck + + listenerWait, err := env.tc.Wait(env.ctx, &taskAPI.WaitRequest{ID: listenerCID}) + if err != nil { + t.Fatalf("Task.Wait listener: %v", err) + } + if listenerWait.GetExitStatus() != 0 { + t.Fatalf("listener container exit status: got %d, want 0", listenerWait.GetExitStatus()) + } + env.tc.Delete(env.ctx, &taskAPI.DeleteRequest{ID: listenerCID}) //nolint:errcheck + + t.Log("member containers share a network namespace: loopback connectivity confirmed") +} diff --git a/sandbox_suite_shared_pidns_linux.go b/sandbox_suite_shared_pidns_linux.go new file mode 100644 index 0000000..9e0de5b --- /dev/null +++ b/sandbox_suite_shared_pidns_linux.go @@ -0,0 +1,87 @@ +//go:build linux + +/* + Copyright The containerd Authors. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +package shimtest + +import ( + "strings" + "testing" + "time" + + taskAPI "github.com/containerd/containerd/api/runtime/task/v3" + specs "github.com/opencontainers/runtime-spec/specs-go" +) + +// testMemberContainersSharePID verifies that member containers of the +// same sandbox can share a PID namespace: a process started by one +// member container is visible — by PID and argv — to a second, +// independently created member container. +// +// The API contract: when a member container's OCI spec carries a host +// path on its PID namespace entry, the shim must place that container +// in a PID namespace shared with its sandbox peers rather than a fresh, +// isolated one (e.g. this is how a caller expresses Kubernetes' +// shareProcessNamespace: true or hostPID: true for a pod). This test +// only observes the externally visible result — cross-container process +// visibility via /proc — and does not assume any particular mechanism a +// shim uses to provide it (a real shared PID namespace, or any other +// approach). It intentionally uses a placeholder host path (see +// withSandboxCtrNamespace) since only a live host has an actual sandbox +// PID to put there; the shim's job is to recognize that a host path was +// requested at all and substitute its own equivalent, not to interpret +// the specific value. +func (s *SandboxSuite) testMemberContainersSharePID(t *testing.T) { + sandboxID := containerID(t) + env := startSandboxShim(t, s.cfg, sandboxID) + + const marker = "pid-share-marker-forever" + + sentinelCID := createContainerInSandbox(t, env, []string{"/bin/forever", marker}, + withSandboxCtrNamespace(specs.PIDNamespace, "/proc/1/ns/pid")) + + // Give the sentinel process a moment to actually start before the + // scanner container looks for it; there is no synchronous "ready" + // signal available across two containers created independently via + // the Task API. + time.Sleep(200 * time.Millisecond) + + scannerCID := createContainerInSandbox(t, env, []string{"/bin/pidscan"}, + withSandboxCtrNamespace(specs.PIDNamespace, "/proc/1/ns/pid")) + + out := readContainerOutput(t, env, scannerCID, marker, 30*time.Second) + if !strings.Contains(out, marker) { + t.Fatalf("pidscan output did not contain sentinel marker %q: %q", marker, out) + } + + scannerWait, err := env.tc.Wait(env.ctx, &taskAPI.WaitRequest{ID: scannerCID}) + if err != nil { + t.Fatalf("Task.Wait scanner: %v", err) + } + if scannerWait.GetExitStatus() != 0 { + t.Fatalf("scanner container exit status: got %d, want 0", scannerWait.GetExitStatus()) + } + env.tc.Delete(env.ctx, &taskAPI.DeleteRequest{ID: scannerCID}) //nolint:errcheck + + if _, err := env.tc.Kill(env.ctx, &taskAPI.KillRequest{ID: sentinelCID, Signal: 9, All: true}); err != nil { + t.Fatalf("Task.Kill sentinel: %v", err) + } + env.tc.Wait(env.ctx, &taskAPI.WaitRequest{ID: sentinelCID}) //nolint:errcheck + env.tc.Delete(env.ctx, &taskAPI.DeleteRequest{ID: sentinelCID}) //nolint:errcheck + + t.Log("member containers share a PID namespace: scanner observed the sentinel process's argv") +} diff --git a/sandbox_suite_volumes_linux.go b/sandbox_suite_volumes_linux.go new file mode 100644 index 0000000..3618fe5 --- /dev/null +++ b/sandbox_suite_volumes_linux.go @@ -0,0 +1,104 @@ +//go:build linux + +/* + Copyright The containerd Authors. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +package shimtest + +import ( + "os" + "path/filepath" + "strings" + "testing" + "time" + + specs "github.com/opencontainers/runtime-spec/specs-go" +) + +// testMemberContainerHostVolume verifies that a member container can be +// given a host directory as a bind-mount volume (an OCI "bind" mount type +// in the container spec), and that the mount is a real, live share — not +// a one-time copy: a file updated on the host after the container has +// already started must become visible inside it. +// +// The API contract: a container's OCI spec may include "bind" mounts +// referencing host paths (e.g. this is how a caller expresses Kubernetes +// hostPath volumes or RecursiveReadOnly=false bind mounts) and the shim +// must honor them for member containers, exactly as it does for the +// top-level bundle rootfs. This test only observes the externally +// visible behavior: it does not assume any particular implementation +// mechanism (a hot-added virtual filesystem, a pre-existing shared tree, +// or anything else a shim might use to satisfy the mount). +func (s *SandboxSuite) testMemberContainerHostVolume(t *testing.T) { + sandboxID := containerID(t) + env := startSandboxShim(t, s.cfg, sandboxID) + + hostDir := t.TempDir() + const ( + fileName = "data.txt" + hostToken = "AAA-initial" + containerMP = "/data" + ) + hostFile := filepath.Join(hostDir, fileName) + if err := os.WriteFile(hostFile, []byte(hostToken+"\n"), 0o644); err != nil { + t.Fatalf("write host file: %v", err) + } + + cid := createContainerInSandbox(t, env, []string{"/bin/forever", "volume-container"}, + withSandboxCtrExtraMounts(specs.Mount{ + Type: "bind", + Source: hostDir, + Destination: containerMP, + Options: []string{"rbind", "rw"}, + }), + ) + + // Host-to-container direction: the file written before the container + // started must be readable inside it. Tokens deliberately use short, + // mutually-exclusive prefixes ("AAA"/"BBB") rather than distinguishing + // suffixes: execInSandboxContainer's output capture allows only a + // fixed, short grace period for FIFO data to drain after the exec'd + // process exits, so a real but short read (e.g. just "AAA-in") must + // still unambiguously identify which file version was seen. + out, exitCode := execInSandboxContainer(t, env, cid, + []string{"/bin/cat", containerMP + "/" + fileName}, 30*time.Second) + if exitCode != 0 { + t.Fatalf("cat host file from container: exit code %d, output %q", exitCode, out) + } + if !strings.HasPrefix(out, "AAA") { + t.Errorf("container read of host file: got %q, want a prefix of %q", out, hostToken) + } + + // Container-to-host direction is exercised the other way around: update + // the file on the host after the container has already started and + // confirm the container sees the change live, proving this is a real + // shared mount and not a one-shot copy taken when the container + // started. + const updatedToken = "BBB-updated" + if err := os.WriteFile(hostFile, []byte(updatedToken+"\n"), 0o644); err != nil { + t.Fatalf("update host file: %v", err) + } + out, exitCode = execInSandboxContainer(t, env, cid, + []string{"/bin/cat", containerMP + "/" + fileName}, 30*time.Second) + if exitCode != 0 { + t.Fatalf("cat updated host file from container: exit code %d, output %q", exitCode, out) + } + if !strings.HasPrefix(out, "BBB") { + t.Errorf("container read of updated host file: got %q, want a prefix of %q", out, updatedToken) + } + + t.Log("member container host volume: live bind mount confirmed both at creation and after a host-side update") +} diff --git a/stress_sandbox_linux.go b/stress_sandbox_linux.go new file mode 100644 index 0000000..cfabdf9 --- /dev/null +++ b/stress_sandbox_linux.go @@ -0,0 +1,229 @@ +//go:build linux + +/* + Copyright The containerd Authors. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +package shimtest + +import ( + "context" + "fmt" + "os" + "sync" + "sync/atomic" + "testing" + "time" + + sandboxAPI "github.com/containerd/containerd/api/runtime/sandbox/v1" + taskAPI "github.com/containerd/containerd/api/runtime/task/v3" +) + +// stressSandboxConcurrency is the number of member containers created +// per iteration of the sandbox stress test. +const stressSandboxConcurrency = 3 + +// stressSandboxMaxRSSGrowth is the upper bound on shim RSS growth (in +// bytes) for the sandbox stress run. VM-based shims exhibit a large +// one-time RSS step on first boot (guest RAM, VMM structures) that +// saturates quickly; growth beyond that is the signal of a per-container +// leak. +// +// 512 MiB accommodates the observed one-time VM boot step (typically +// ~200–300 MiB on Linux) with headroom. Per-container growth at +// steady state should be < 1 KiB/container. +const stressSandboxMaxRSSGrowth = 512 * 1024 * 1024 + +// testSandbox exercises the sandbox shim API under sustained container +// churn: one long-lived sandbox VM hosts repeated bursts of concurrent +// member containers. Each burst creates, starts, waits, and deletes +// stressSandboxConcurrency containers before the next burst begins. +// +// Leak-detection components: +// +// 1. Process leak: no shim processes remain after the run (enforced +// by the top-level registerShimLeakCheck in StressSuite.Run). +// 2. Host RSS growth: the shim RSS must not exceed +// stressSandboxMaxRSSGrowth bytes over the run duration. +// 3. Mount leak: no per-container mount points remain in the shim's +// mount namespace after ShutdownSandbox. +func (s *StressSuite) testSandbox(t *testing.T) { + if testing.Short() { + t.Skip("skipping sandbox stress in short mode") + } + + // Pre-build read-only rootfs images once. Reusing them across all + // iterations keeps disk consumption O(1) instead of O(iterations). + // Per-iteration setup only creates the small writable parts (ext4 + // scratch image or overlay upper/work dirs). + imgs := buildShimImages(t, s.cfg) + + // On non-root systems (where loop mounts are unavailable), pre-extract + // the rootfs erofs into a single directory that every iteration reuses + // as the bind-mount source. ShareRootfs copies it into the sandbox + // shared dir per container and Unshare removes it promptly, so disk + // consumption stays O(1) across the run. + var preExtractedRootfs string + if !s.cfg.FormatMounts || os.Getuid() != 0 { + preExtractedRootfs = t.TempDir() + extractErofsIntoDir(t, imgs.erofsImg, preExtractedRootfs) + } + + sandboxID := containerID(t) + env := startSandboxShim(t, s.cfg, sandboxID) + + // Bootstrap: create a probe container to seed the shimPID lookup + // and establish that the sandbox is functional before the loop. + probeCID := createContainerInSandbox(t, env, []string{"/bin/echo", "probe"}) + readContainerOutput(t, env, probeCID, "probe", 30*time.Second) + env.tc.Wait(env.ctx, &taskAPI.WaitRequest{ID: probeCID}) //nolint:errcheck + + shimPID := sandboxShimPID(env, probeCID) + releaseSandboxContainer(env.ctx, env, probeCID) //nolint:errcheck + t.Logf("sandbox stress: shim PID=%d", shimPID) + + // Sample RSS before the churn loop. + var rssBefore int64 + var rssOK bool + if shimPID != 0 { + var err error + rssBefore, err = readRSS(shimPID) + if err != nil { + t.Logf("cannot read pre-stress shim RSS (PID %d): %v — disabling RSS check", shimPID, err) + } else { + rssOK = true + } + } + + var iterIdx atomic.Int64 + ctx, cancel := stressCtx(t, env.ctx) + defer cancel() + + iters, elapsed, stressErr := runStress(ctx, func(iterCtx context.Context) error { + i := iterIdx.Add(1) + name := fmt.Sprintf("sbiter%05d", i) + + // Each iteration creates stressSandboxConcurrency containers using + // the pre-built images. Commands cycle: + // j%3 == 0: /bin/exit 0 (exit-code propagation) + // j%3 == 1: /bin/exit 0 + // j%3 == 2: /bin/exit 0 + // All containers exit cleanly; we just verify the exit status. + // Output is discarded (stdout/stderr FIFOs are drained silently) + // so the test exercises the create/start/wait/delete path under + // sustained load without accumulating output buffers. + type ctrInfo struct { + cid string + } + ctrs := make([]ctrInfo, stressSandboxConcurrency) + for j := range ctrs { + args := []string{"/bin/echo", fmt.Sprintf("%s-j%d", name, j)} + cid, err := createSandboxContainerFast(iterCtx, t, env, s.cfg, imgs, preExtractedRootfs, args) + if err != nil { + return fmt.Errorf("create %d: %w", j, err) + } + ctrs[j] = ctrInfo{cid: cid} + } + + // Wait for all containers concurrently. + var wg sync.WaitGroup + errs := make([]error, stressSandboxConcurrency) + for j, ci := range ctrs { + wg.Add(1) + go func(j int, ci ctrInfo) { + defer wg.Done() + subCtx, subCancel := context.WithTimeout(iterCtx, stressIterationTimeout) + defer subCancel() + + waitResp, err := env.tc.Wait(subCtx, &taskAPI.WaitRequest{ID: ci.cid}) + if err != nil { + errs[j] = fmt.Errorf("wait %s: %w", ci.cid, err) + return + } + if waitResp.GetExitStatus() != 0 { + errs[j] = fmt.Errorf("container %s exited with status %d", + ci.cid, waitResp.GetExitStatus()) + } + }(j, ci) + } + wg.Wait() + + for _, e := range errs { + if e != nil { + return e + } + } + + // Delete all containers and release tracking state immediately. + for _, ci := range ctrs { + if err := releaseSandboxContainer(iterCtx, env, ci.cid); err != nil { + return fmt.Errorf("delete %s: %w", ci.cid, err) + } + } + + return nil + }) + + rate := float64(iters) / elapsed.Seconds() + t.Logf("sandbox stress: %d iterations × %d containers = %d total containers in %s (%.1f iter/s)", + iters, stressSandboxConcurrency, iters*stressSandboxConcurrency, + elapsed.Round(time.Millisecond), rate) + + if stressErr != nil { + t.Fatalf("sandbox stress: %v", stressErr) + } + + // ── RSS growth check ────────────────────────────────────────────── + if rssOK && shimPID != 0 { + rssAfter, err := readRSS(shimPID) + if err != nil { + t.Logf("cannot read post-stress shim RSS: %v", err) + } else { + growth := rssAfter - rssBefore + threshold := int64(stressSandboxMaxRSSGrowth) + if s.options.SandboxRSSGrowthOverride > 0 { + threshold = s.options.SandboxRSSGrowthOverride + } + t.Logf("shim RSS: before=%d MiB after=%d MiB growth=%d MiB (threshold %d MiB)", + rssBefore>>20, rssAfter>>20, growth>>20, threshold>>20) + if growth > threshold { + t.Errorf("shim RSS grew %d bytes during sandbox stress (threshold %d bytes); "+ + "possible per-container memory leak", + growth, threshold) + } + } + } + + // ── Mount-leak check ───────────────────────────────────────────── + // Snapshot mounts before shutdown, then verify they are gone after. + mountsBefore := sandboxContainersMounts(shimPID) + t.Logf("per-container mounts before shutdown: %d", len(mountsBefore)) + + // Trigger shutdown (cleanup is also registered by startSandboxShim, + // but we drive it explicitly here so we can inspect state after). + env.sc.StopSandbox(env.ctx, &sandboxAPI.StopSandboxRequest{SandboxID: sandboxID}) //nolint:errcheck + env.sc.ShutdownSandbox(env.ctx, &sandboxAPI.ShutdownSandboxRequest{SandboxID: sandboxID}) //nolint:errcheck + + // Allow shim to finish cleanup. + time.Sleep(500 * time.Millisecond) + + mountsAfter := sandboxContainersMounts(shimPID) + if len(mountsAfter) > 0 { + t.Errorf("sandbox stress: %d per-container mount(s) leaked after ShutdownSandbox: %v", + len(mountsAfter), mountsAfter) + } else { + t.Log("no per-container mounts remain after shutdown (mount-leak check passed)") + } +} diff --git a/stress_sandbox_other.go b/stress_sandbox_other.go new file mode 100644 index 0000000..372d22e --- /dev/null +++ b/stress_sandbox_other.go @@ -0,0 +1,26 @@ +//go:build !linux + +/* + Copyright The containerd Authors. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +package shimtest + +import "testing" + +// testSandbox skips on non-Linux platforms. +func (s *StressSuite) testSandbox(t *testing.T) { + t.Skip("sandbox stress is Linux-only") +} diff --git a/stress_suite.go b/stress_suite.go index 921ad6f..adca907 100644 --- a/stress_suite.go +++ b/stress_suite.go @@ -64,12 +64,24 @@ type StressOptions struct { // test. The shim under test must implement the transfer service. Transfer bool + // Sandbox enables the sandbox container-churn stress test. The + // shim under test must implement the containerd sandbox shim API + // (runtime/sandbox/v1). + Sandbox bool + // ExecRSSGrowthOverride, when non-zero, replaces the platform default // RSS growth threshold for the exec stress test. Use this for shims // that host a VM or other large runtime in-process and therefore have // a higher expected one-time RSS step than a thin supervisor shim. // The value is in bytes. ExecRSSGrowthOverride int64 + + // SandboxRSSGrowthOverride, when non-zero, replaces the platform default + // RSS growth threshold for the sandbox stress test. VM-based shims + // have a large one-time RSS step from the VM boot that saturates + // early; set this to accommodate the expected baseline. + // The value is in bytes. + SandboxRSSGrowthOverride int64 } // NewStressSuite constructs a StressSuite from cfg and options. @@ -92,6 +104,9 @@ func (s *StressSuite) Run(t *testing.T) { if s.options.Transfer { t.Run("Transfer", s.testTransfer) } + if s.options.Sandbox { + t.Run("Sandbox", s.testSandbox) + } } // testLifecycle exercises the full create/start/run/kill/wait/delete diff --git a/testbin/testbin.go b/testbin/testbin.go index fcc1bae..95daa6f 100644 --- a/testbin/testbin.go +++ b/testbin/testbin.go @@ -28,6 +28,7 @@ package testbin import ( + "bytes" "fmt" "hash/crc32" "io" @@ -38,7 +39,9 @@ import ( "strconv" "strings" "sync" + "syscall" "time" + "unsafe" ) // Main is the entry point for the testbin multicall binary. It dispatches @@ -86,8 +89,16 @@ func Main() { cmdNC(args) case "host": cmdHost(args) + case "echosrv": + cmdEchoServer(args) case "tickexit": cmdTickexit(args) + case "pidscan": + cmdPidscan(args) + case "shmwrite": + cmdShmWrite(args) + case "shmread": + cmdShmRead(args) default: fmt.Fprintf(os.Stderr, "testbin: unknown command: %s\n", cmd) os.Exit(127) @@ -460,6 +471,15 @@ func cmdBurstexit(args []string) { // ReadFrom, i.e. sendto/recvfrom) so that shim networking layers cannot // short-circuit routing based on the local connect(2) call, which for UDP // always succeeds regardless of whether any peer is listening. +// +// There is deliberately no listen mode: nc's stream modes are a symmetric +// bidirectional pipe that only terminates when the peer closes the +// connection, which works when the peer is a host-side Go program that can +// explicitly Close() once it's done (see testOutboundTCP and +// testContainerTrafficScopedToNetworkSandbox), but deadlocks if both ends are +// nc processes with neither designed to close first. For "one container +// listens, another connects" scenarios, see echosrv, a purpose-built +// one-shot responder that always terminates on its own. func cmdNC(args []string) { if len(args) < 2 { fmt.Fprintln(os.Stderr, "usage: nc [-u] | nc -U ") @@ -578,3 +598,183 @@ func cmdHost(args []string) { fmt.Printf("%s has address %s\n", name, a) } } + +// cmdEchoServer listens on TCP port (all interfaces), accepts exactly +// one connection, reads exactly one chunk of data (up to 4096 bytes), writes +// the same bytes back verbatim, closes the connection, and exits 0. +// +// Unlike "nc -l", which is a general-purpose bidirectional stream pipe (and +// so never closes the connection on its own — the peer must close it), +// echosrv is purpose-built as a one-shot round-trip responder: it always +// terminates on its own once one exchange completes, which is what makes it +// usable as a container's main process in a test that needs the container to +// exit cleanly after proving connectivity (e.g. two containers exchanging +// data over a shared network namespace, where neither side is a host-side Go +// program that can explicitly Close() to signal completion). +// +// Usage: echosrv +func cmdEchoServer(args []string) { + if len(args) < 2 { + fmt.Fprintln(os.Stderr, "usage: echosrv ") + os.Exit(1) + } + // tcp4/0.0.0.0 explicitly, not "tcp"/":" (which defaults to a + // dual-stack IPv6 socket on Linux): shimtest does not assume a shim's + // default container networking path supports IPv6, only IPv4. + ln, err := net.Listen("tcp4", "0.0.0.0:"+args[1]) + if err != nil { + fmt.Fprintf(os.Stderr, "echosrv: listen 0.0.0.0:%s: %v\n", args[1], err) + os.Exit(1) + } + conn, err := ln.Accept() + ln.Close() + if err != nil { + fmt.Fprintf(os.Stderr, "echosrv: accept: %v\n", err) + os.Exit(1) + } + defer conn.Close() + + buf := make([]byte, 4096) + n, err := conn.Read(buf) + if n == 0 && err != nil { + fmt.Fprintf(os.Stderr, "echosrv: read: %v\n", err) + os.Exit(1) + } + if _, err := conn.Write(buf[:n]); err != nil { + fmt.Fprintf(os.Stderr, "echosrv: write: %v\n", err) + os.Exit(1) + } +} + +// cmdPidscan lists every PID visible in this process's PID namespace +// along with its cmdline, by scanning /proc. Used by shimtest to verify +// PID namespace sharing across member containers: the test does not +// know the PID number of the sentinel process it is looking for ahead +// of time (only a unique marker string baked into that process's +// argv), so it scans every visible PID's cmdline rather than checking +// one specific PID. +func cmdPidscan(_ []string) { + entries, err := os.ReadDir("/proc") + if err != nil { + fmt.Fprintf(os.Stderr, "pidscan: readdir /proc: %v\n", err) + os.Exit(1) + } + for _, e := range entries { + name := e.Name() + if _, err := strconv.Atoi(name); err != nil { + continue // not a PID directory + } + data, err := os.ReadFile(filepath.Join("/proc", name, "cmdline")) + if err != nil { + // The process may have exited between the readdir and this + // read; that race is expected and not an error. + continue + } + cmdline := strings.ReplaceAll(strings.TrimRight(string(data), "\x00"), "\x00", " ") + fmt.Printf("%s %s\n", name, cmdline) + } +} + +const ( + shmSize = 4096 + // ipcCreat is IPC_CREAT, from linux/ipc.h. The stdlib syscall + // package exposes SysV shm's syscall numbers (SYS_SHMGET etc.) but + // not its flag constants, so this is hardcoded. + ipcCreat = 0o1000 +) + +// cmdShmWrite creates (or reuses) a SysV shared memory segment +// identified by a fixed numeric key and writes a marker string into it, +// then detaches — but does not remove — the segment, leaving it behind +// for a later shmread call to find. +// +// Used by shimtest to verify IPC namespace sharing: SysV IPC objects +// are keyed and visible only within the creating process's IPC +// namespace, independent of mount namespace or any bind-mounted +// /dev/shm, so a successful cross-container shmwrite/shmread round +// trip through the same key is conclusive proof of a shared IPC +// namespace (and not, for instance, an artifact of a shared /dev/shm +// bind mount). +// +// Usage: shmwrite +func cmdShmWrite(args []string) { + if len(args) < 3 { + fmt.Fprintln(os.Stderr, "usage: shmwrite ") + os.Exit(1) + } + key, err := strconv.ParseInt(args[1], 10, 64) + if err != nil { + fmt.Fprintf(os.Stderr, "shmwrite: invalid key %q: %v\n", args[1], err) + os.Exit(1) + } + marker := args[2] + if len(marker) >= shmSize { + fmt.Fprintln(os.Stderr, "shmwrite: marker too large") + os.Exit(1) + } + + shmid, _, errno := syscall.Syscall(syscall.SYS_SHMGET, uintptr(key), shmSize, ipcCreat|0600) + if errno != 0 { + fmt.Fprintf(os.Stderr, "shmwrite: shmget: %v\n", errno) + os.Exit(1) + } + addr, _, errno := syscall.Syscall(syscall.SYS_SHMAT, shmid, 0, 0) + if errno != 0 { + fmt.Fprintf(os.Stderr, "shmwrite: shmat: %v\n", errno) + os.Exit(1) + } + // addr is a raw address returned by the shmat(2) syscall, not derived + // from a Go pointer, so it doesn't fit vet's recognized safe-conversion + // patterns even though the conversion itself is valid here. + buf := (*[shmSize]byte)(unsafe.Pointer(addr)) //nolint:govet + n := copy(buf[:], marker) + buf[n] = 0 + syscall.Syscall(syscall.SYS_SHMDT, addr, 0, 0) //nolint:errcheck + + fmt.Println("shmwrite: ok") +} + +// cmdShmRead attaches to an existing SysV shared memory segment +// identified by a fixed numeric key (created by a prior shmwrite call, +// possibly in a different container) and prints the marker string +// found in it. +// +// It deliberately omits IPC_CREAT: if the segment does not already +// exist in this process's IPC namespace, that is exactly the "not +// shared" case and must be reported as a failure, rather than silently +// creating a fresh, empty segment that would make a broken test look +// like it passed. +// +// Usage: shmread +func cmdShmRead(args []string) { + if len(args) < 2 { + fmt.Fprintln(os.Stderr, "usage: shmread ") + os.Exit(1) + } + key, err := strconv.ParseInt(args[1], 10, 64) + if err != nil { + fmt.Fprintf(os.Stderr, "shmread: invalid key %q: %v\n", args[1], err) + os.Exit(1) + } + + shmid, _, errno := syscall.Syscall(syscall.SYS_SHMGET, uintptr(key), shmSize, 0600) + if errno != 0 { + fmt.Println("shmread: NOTFOUND") + os.Exit(1) + } + addr, _, errno := syscall.Syscall(syscall.SYS_SHMAT, shmid, 0, 0) + if errno != 0 { + fmt.Fprintf(os.Stderr, "shmread: shmat: %v\n", errno) + os.Exit(1) + } + // See the matching comment in cmdShmWrite: addr comes from shmat(2), + // not from a Go pointer, so vet can't recognize this as a safe + // conversion even though it is one. + buf := (*[shmSize]byte)(unsafe.Pointer(addr)) //nolint:govet + end := bytes.IndexByte(buf[:], 0) + if end < 0 { + end = shmSize + } + fmt.Println(string(buf[:end])) + syscall.Syscall(syscall.SYS_SHMDT, addr, 0, 0) //nolint:errcheck +}