diff --git a/internal/pkg/libcheck/libcheck.go b/internal/pkg/libcheck/libcheck.go index 5361cf93..c2db790c 100644 --- a/internal/pkg/libcheck/libcheck.go +++ b/internal/pkg/libcheck/libcheck.go @@ -36,6 +36,7 @@ package libcheck import ( "context" + "errors" "fmt" "io/fs" "os" @@ -45,6 +46,10 @@ import ( "slices" "strconv" "strings" + "sync" + "time" + + "github.com/golang/glog" ) // Libc identifies the C library (or absence of one) an image's entrypoint @@ -124,13 +129,44 @@ type Inspector struct { CtrPath string ContainerdSocket string Namespace string // containerd namespace images were pulled into, usually "k8s.io" + + pulling sync.Map // image refs with a background pull in flight +} + +// ErrImageNotPresent is returned when the image is not in containerd yet. The +// kubelet device manager calls Allocate at pod admission, before it pulls the +// image, so the first Pod to use an image sees this; the pull is started in +// the background and a retry finds the image. +var ErrImageNotPresent = errors.New("image is not on the node yet") + +// pullTimeout bounds one background pull. +const pullTimeout = 30 * time.Minute + +// pullInBackground starts one "ctr images pull" per image ref at a time. +// Allocate must not wait for it: images are large and admission is blocking. +func (ins *Inspector) pullInBackground(imageRef string) { + if _, running := ins.pulling.LoadOrStore(imageRef, struct{}{}); running { + return + } + go func() { + defer ins.pulling.Delete(imageRef) + ctx, cancel := context.WithTimeout(context.Background(), pullTimeout) + defer cancel() + args := []string{"--namespace", ins.Namespace, "--address", ins.ContainerdSocket, "images", "pull", "--", imageRef} + if out, err := exec.CommandContext(ctx, ins.CtrPath, args...).CombinedOutput(); err != nil { + glog.Warningf("libcheck: ctr images pull %s: %v (%s)", imageRef, err, strings.TrimSpace(string(out))) + return + } + glog.Infof("libcheck: pulled %s so its libc can be inspected", imageRef) + }() } // Inspect mounts imageRef read-only, inspects it for a musl or glibc -// dynamic loader, and unmounts it again. The image must already be present -// in the inspector's containerd (kubelet pulls it before Allocate runs in -// the normal flow); a missing image is reported as an error, not Unknown, -// so the caller can decide how to fail closed on "could not verify". +// dynamic loader, and unmounts it again. The image must be present in the +// inspector's containerd. Allocate runs at pod admission, before kubelet +// pulls the image, so a missing image starts a background pull and is +// reported as ErrImageNotPresent, not Unknown, so the caller fails closed on +// "could not verify" and the next Pod finds the image. func (ins *Inspector) Inspect(ctx context.Context, imageRef string) (Libc, error) { mountDir, err := os.MkdirTemp("", "amd-dp-libcheck-") if err != nil { @@ -142,6 +178,10 @@ func (ins *Inspector) Inspect(ctx context.Context, imageRef string) (Libc, error // "--" ends flag parsing, so a ref starting with '-' is never a flag. mountArgs := []string{"--namespace", ins.Namespace, "--address", ins.ContainerdSocket, "images", "mount", "--", imageRef, mountDir} if out, err := exec.CommandContext(ctx, ins.CtrPath, mountArgs...).CombinedOutput(); err != nil { + if strings.Contains(string(out), "not found") { + ins.pullInBackground(imageRef) + return Unknown, fmt.Errorf("%w: %s; pulling it in the background, retry the Pod once the pull finishes", ErrImageNotPresent, imageRef) + } return Unknown, fmt.Errorf("ctr images mount %s: %w (%s)", imageRef, err, strings.TrimSpace(string(out))) } defer func() { diff --git a/internal/pkg/libcheck/libcheck_test.go b/internal/pkg/libcheck/libcheck_test.go index e52f9861..f1062a30 100644 --- a/internal/pkg/libcheck/libcheck_test.go +++ b/internal/pkg/libcheck/libcheck_test.go @@ -18,10 +18,12 @@ package libcheck import ( "context" + "errors" "os" "path/filepath" "strings" "testing" + "time" ) func touch(t *testing.T, path string) { @@ -297,3 +299,75 @@ func TestInspectRootFollowsAbsoluteLibcLinkInsideImage(t *testing.T) { t.Error("link outside the image accepted") } } + +// Allocate runs before kubelet pulls the image, so the first Inspect of an +// image that is not in containerd must start a pull and report +// ErrImageNotPresent, and the next Inspect must find the pulled image. +func TestInspectPullsMissingImage(t *testing.T) { + dir := t.TempDir() + pulled := filepath.Join(dir, "pulled") + pullCount := filepath.Join(dir, "pulls") + fixture := filepath.Join(dir, "image") + touch(t, filepath.Join(fixture, "lib64", "ld-linux-x86-64.so.2")) + libc := filepath.Join(fixture, "lib", "x86_64-linux-gnu", "libc.so.6") + touch(t, libc) + if err := os.WriteFile(libc, []byte("GLIBC_2.38\x00"), 0o644); err != nil { + t.Fatal(err) + } + ctr := filepath.Join(dir, "ctr") + script := `#!/bin/sh +case " $* " in +*" pull "*) echo x >> ` + pullCount + `; touch ` + pulled + `; exit 0;; +*" unmount "*) exit 0;; +*" mount "*) + if [ ! -f ` + pulled + ` ]; then echo 'ctr: image "docker.io/library/x:1": not found'; exit 1; fi + for last; do :; done + cp -r ` + fixture + `/. "$last"/; exit 0;; +esac +exit 1 +` + if err := os.WriteFile(ctr, []byte(script), 0o755); err != nil { + t.Fatal(err) + } + ins := &Inspector{CtrPath: ctr, ContainerdSocket: "/sock", Namespace: "k8s.io"} + + if _, err := ins.Inspect(context.Background(), "x:1"); !errors.Is(err, ErrImageNotPresent) { + t.Fatalf("first Inspect error = %v, want ErrImageNotPresent", err) + } + deadline := time.Now().Add(5 * time.Second) + for { + if _, err := os.Stat(pulled); err == nil { + break + } + if time.Now().After(deadline) { + t.Fatal("no background pull started") + } + time.Sleep(10 * time.Millisecond) + } + var got Libc + for { + var err error + if got, err = ins.Inspect(context.Background(), "x:1"); err == nil { + break + } + if !errors.Is(err, ErrImageNotPresent) || time.Now().After(deadline) { + t.Fatalf("Inspect after the pull = %v", err) + } + time.Sleep(10 * time.Millisecond) // pull flag set, in-flight marker not yet cleared + } + if got != Glibc { + t.Errorf("Inspect after the pull = %v, want Glibc", got) + } + if n := strings.Count(readFile(t, pullCount), "x"); n != 1 { + t.Errorf("ctr images pull ran %d times, want 1", n) + } +} + +func readFile(t *testing.T, path string) string { + t.Helper() + b, err := os.ReadFile(path) + if err != nil { + t.Fatal(err) + } + return string(b) +}