fix(libcheck): pull an image that is not on the node yet

internal/pkg/libcheck/libcheck.go

@@ -36,6 +36,7 @@ package libcheck
 
 import (
 	"context"
+	"errors"
 	"fmt"
 	"io/fs"
 	"os"
@@ -45,6 +46,10 @@ import (
 	"slices"
 	"strconv"
 	"strings"
+	"sync"
+	"time"
+
+	"github.com/golang/glog"
 )
 
 // Libc identifies the C library (or absence of one) an image's entrypoint
@@ -124,13 +129,44 @@ type Inspector struct {
 	CtrPath          string
 	ContainerdSocket string
 	Namespace        string // containerd namespace images were pulled into, usually "k8s.io"
+
+	pulling sync.Map // image refs with a background pull in flight
+}
+
+// ErrImageNotPresent is returned when the image is not in containerd yet. The
+// kubelet device manager calls Allocate at pod admission, before it pulls the
+// image, so the first Pod to use an image sees this; the pull is started in
+// the background and a retry finds the image.
+var ErrImageNotPresent = errors.New("image is not on the node yet")
+
+// pullTimeout bounds one background pull.
+const pullTimeout = 30 * time.Minute
+
+// pullInBackground starts one "ctr images pull" per image ref at a time.
+// Allocate must not wait for it: images are large and admission is blocking.
+func (ins *Inspector) pullInBackground(imageRef string) {
+	if _, running := ins.pulling.LoadOrStore(imageRef, struct{}{}); running {
+		return
+	}
+	go func() {
+		defer ins.pulling.Delete(imageRef)
+		ctx, cancel := context.WithTimeout(context.Background(), pullTimeout)
+		defer cancel()
+		args := []string{"--namespace", ins.Namespace, "--address", ins.ContainerdSocket, "images", "pull", "--", imageRef}
+		if out, err := exec.CommandContext(ctx, ins.CtrPath, args...).CombinedOutput(); err != nil {
+			glog.Warningf("libcheck: ctr images pull %s: %v (%s)", imageRef, err, strings.TrimSpace(string(out)))
+			return
+		}
+		glog.Infof("libcheck: pulled %s so its libc can be inspected", imageRef)
+	}()
 }
 
 // Inspect mounts imageRef read-only, inspects it for a musl or glibc
-// dynamic loader, and unmounts it again. The image must already be present
-// in the inspector's containerd (kubelet pulls it before Allocate runs in
-// the normal flow); a missing image is reported as an error, not Unknown,
-// so the caller can decide how to fail closed on "could not verify".
+// dynamic loader, and unmounts it again. The image must be present in the
+// inspector's containerd. Allocate runs at pod admission, before kubelet
+// pulls the image, so a missing image starts a background pull and is
+// reported as ErrImageNotPresent, not Unknown, so the caller fails closed on
+// "could not verify" and the next Pod finds the image.
 func (ins *Inspector) Inspect(ctx context.Context, imageRef string) (Libc, error) {
 	mountDir, err := os.MkdirTemp("", "amd-dp-libcheck-")
 	if err != nil {
@@ -142,6 +178,10 @@ func (ins *Inspector) Inspect(ctx context.Context, imageRef string) (Libc, error
 	// "--" ends flag parsing, so a ref starting with '-' is never a flag.
 	mountArgs := []string{"--namespace", ins.Namespace, "--address", ins.ContainerdSocket, "images", "mount", "--", imageRef, mountDir}
 	if out, err := exec.CommandContext(ctx, ins.CtrPath, mountArgs...).CombinedOutput(); err != nil {
+		if strings.Contains(string(out), "not found") {
+			ins.pullInBackground(imageRef)
+			return Unknown, fmt.Errorf("%w: %s; pulling it in the background, retry the Pod once the pull finishes", ErrImageNotPresent, imageRef)
+		}
 		return Unknown, fmt.Errorf("ctr images mount %s: %w (%s)", imageRef, err, strings.TrimSpace(string(out)))
 	}
 	defer func() {

internal/pkg/libcheck/libcheck_test.go

@@ -18,10 +18,12 @@ package libcheck
 
 import (
 	"context"
+	"errors"
 	"os"
 	"path/filepath"
 	"strings"
 	"testing"
+	"time"
 )
 
 func touch(t *testing.T, path string) {
@@ -297,3 +299,75 @@ func TestInspectRootFollowsAbsoluteLibcLinkInsideImage(t *testing.T) {
 		t.Error("link outside the image accepted")
 	}
 }
+
+// Allocate runs before kubelet pulls the image, so the first Inspect of an
+// image that is not in containerd must start a pull and report
+// ErrImageNotPresent, and the next Inspect must find the pulled image.
+func TestInspectPullsMissingImage(t *testing.T) {
+	dir := t.TempDir()
+	pulled := filepath.Join(dir, "pulled")
+	pullCount := filepath.Join(dir, "pulls")
+	fixture := filepath.Join(dir, "image")
+	touch(t, filepath.Join(fixture, "lib64", "ld-linux-x86-64.so.2"))
+	libc := filepath.Join(fixture, "lib", "x86_64-linux-gnu", "libc.so.6")
+	touch(t, libc)
+	if err := os.WriteFile(libc, []byte("GLIBC_2.38\x00"), 0o644); err != nil {
+		t.Fatal(err)
+	}
+	ctr := filepath.Join(dir, "ctr")
+	script := `#!/bin/sh
+case " $* " in
+*" pull "*) echo x >> ` + pullCount + `; touch ` + pulled + `; exit 0;;
+*" unmount "*) exit 0;;
+*" mount "*)
+  if [ ! -f ` + pulled + ` ]; then echo 'ctr: image "docker.io/library/x:1": not found'; exit 1; fi
+  for last; do :; done
+  cp -r ` + fixture + `/. "$last"/; exit 0;;
+esac
+exit 1
+`
+	if err := os.WriteFile(ctr, []byte(script), 0o755); err != nil {
+		t.Fatal(err)
+	}
+	ins := &Inspector{CtrPath: ctr, ContainerdSocket: "/sock", Namespace: "k8s.io"}
+
+	if _, err := ins.Inspect(context.Background(), "x:1"); !errors.Is(err, ErrImageNotPresent) {
+		t.Fatalf("first Inspect error = %v, want ErrImageNotPresent", err)
+	}
+	deadline := time.Now().Add(5 * time.Second)
+	for {
+		if _, err := os.Stat(pulled); err == nil {
+			break
+		}
+		if time.Now().After(deadline) {
+			t.Fatal("no background pull started")
+		}
+		time.Sleep(10 * time.Millisecond)
+	}
+	var got Libc
+	for {
+		var err error
+		if got, err = ins.Inspect(context.Background(), "x:1"); err == nil {
+			break
+		}
+		if !errors.Is(err, ErrImageNotPresent) || time.Now().After(deadline) {
+			t.Fatalf("Inspect after the pull = %v", err)
+		}
+		time.Sleep(10 * time.Millisecond) // pull flag set, in-flight marker not yet cleared
+	}
+	if got != Glibc {
+		t.Errorf("Inspect after the pull = %v, want Glibc", got)
+	}
+	if n := strings.Count(readFile(t, pullCount), "x"); n != 1 {
+		t.Errorf("ctr images pull ran %d times, want 1", n)
+	}
+}
+
+func readFile(t *testing.T, path string) string {
+	t.Helper()
+	b, err := os.ReadFile(path)
+	if err != nil {
+		t.Fatal(err)
+	}
+	return string(b)
+}