fix(plugin): serve whole GPUs when no HAMi-tracked pod is pending

internal/pkg/plugin/plugin.go

@@ -20,6 +20,7 @@ package plugin
 import (
 	"context"
 	"encoding/json"
+	"errors"
 	"fmt"
 	"os"
 	"os/signal"
@@ -1039,8 +1040,20 @@ func (p *AMDGPUPlugin) Allocate(ctx context.Context, r *pluginapi.AllocateReques
 	glog.Infof("Allocate request: %+v", r)
 	nodename := os.Getenv(utils.NodeNameEnvName)
 	current, err := utils.GetPendingPod(ctx, nodename)
-	if err != nil {
-		return &pluginapi.AllocateResponse{}, err
+	trackedPod := err == nil
+	if !trackedPod {
+		if !errors.Is(err, utils.ErrNoPendingPod) {
+			return &pluginapi.AllocateResponse{}, err
+		}
+		// No Pod on this node carries HAMi's bind-time annotations, most
+		// often because a different scheduler or a second device plugin
+		// placed this Pod directly on the kubelet-reported device ids
+		// (see besteffort_policy.go's same assumption on the list side).
+		// Serve the whole GPUs kubelet asked for instead of failing the
+		// Pod; there is no HAMi-tracked Pod to persist CU state against
+		// or to release a node lock for.
+		glog.Warningf("no HAMi-tracked pod pending on node %s (%v); serving kubelet device ids %v as whole GPUs", nodename, err, r.ContainerRequests)
+		current = &corev1.Pod{}
 	}
 	podCuAllocList := map[string]string{}
 	if raw := strings.TrimSpace(current.Annotations[utils.CuAllocation]); raw != "" {
@@ -1261,13 +1274,15 @@ func (p *AMDGPUPlugin) Allocate(ctx context.Context, r *pluginapi.AllocateReques
 						ReadOnly:      true,
 					})
 				}
-				if p.dmemEnabled {
+				if p.dmemEnabled && trackedPod {
 					// The pod's cgroup can take several seconds to appear
 					// after binding (measured up to ~7s on a real RKE2
 					// node), well past what Allocate should ever block for.
 					// Apply the cap in the background; libamvgpu's
 					// HIP_DEVICE_MEMORY_LIMIT_<i> above is already the
 					// synchronous enforcement path when auditCompatible.
+					// Skipped without a HAMi-tracked pod: there is no real
+					// namespace/name/UID to resolve a cgroup path from.
 					podNamespace, podName, podUID, qos := current.Namespace, current.Name, string(current.UID), current.Status.QOSClass
 					for _, d := range devreq {
 						go p.applyDmemCap(podNamespace, podName, podUID, qos, d)
@@ -1308,7 +1323,9 @@ func (p *AMDGPUPlugin) Allocate(ctx context.Context, r *pluginapi.AllocateReques
 	}
 
 	glog.Infoln("Allocate Response", response.ContainerResponses)
-	utils.PodAllocationTrySuccess(nodename, NodeLockName, current)
+	if trackedPod {
+		utils.PodAllocationTrySuccess(nodename, NodeLockName, current)
+	}
 
 	return response, nil
 }

internal/pkg/plugin/plugin_test.go

@@ -301,6 +301,60 @@ func TestAllocateWholeGPUSkipsCUCommit(t *testing.T) {
 	}
 }
 
+// A Pod a different scheduler or a second device plugin placed on this
+// node's kubelet-reported device ids carries none of HAMi's bind-time
+// annotations, so GetPendingPod finds nothing. Allocate must still serve the
+// kubelet device ids as a whole GPU instead of failing the Pod.
+func TestAllocateWithoutHAMiTrackedPod(t *testing.T) {
+	node := &corev1.Node{ObjectMeta: metav1.ObjectMeta{Name: "n"}}
+	cs := fake.NewSimpleClientset(node) // no Pod carries the bind-time annotations
+	old := utils.KubeClient
+	t.Cleanup(func() { utils.KubeClient = old })
+	utils.KubeClient = cs
+	t.Setenv(utils.NodeNameEnvName, "n")
+
+	p := &AMDGPUPlugin{
+		AMDGPUs:              map[string]map[string]interface{}{"0000:06:00.0": {"card": 1, "renderD": 129}},
+		amdSMIUUIDToTopology: map[string]string{"uuid-a": "0000:06:00.0"},
+		amdSMIUUIDToROCrUUID: map[string]string{"uuid-a": "GPU-1"},
+		deviceCache:          []*utils.DeviceInfo{{ID: "uuid-a", Devcore: 32, Devmem: 16304}},
+	}
+	resp, err := p.Allocate(context.Background(), &pluginapi.AllocateRequest{
+		ContainerRequests: []*pluginapi.ContainerAllocateRequest{{DevicesIds: []string{"uuid-a"}}},
+	})
+	if err != nil {
+		t.Fatalf("Allocate without a HAMi-tracked pod = %v, want the kubelet device ids served as a whole GPU", err)
+	}
+	if got := len(resp.ContainerResponses); got != 1 {
+		t.Fatalf("ContainerResponses = %d, want 1", got)
+	}
+	if _, maskSet := resp.ContainerResponses[0].Envs["HSA_CU_MASK"]; !maskSet {
+		t.Error("whole-GPU fallback must still set HSA_CU_MASK")
+	}
+	if got := resp.ContainerResponses[0].Envs["ROCR_VISIBLE_DEVICES"]; got != "GPU-1" {
+		t.Errorf("ROCR_VISIBLE_DEVICES = %q, want GPU-1", got)
+	}
+}
+
+// A real error getting the pending pod (API server trouble, not simply no
+// bound pod found) must still fail Allocate rather than silently falling
+// back, so kubelet retries instead of starting a container without the
+// scheduler's intended device.
+func TestAllocateFailsOnRealPendingPodError(t *testing.T) {
+	cs := fake.NewSimpleClientset() // no Node "n" either: GetAllocatePodByNode errors
+	old := utils.KubeClient
+	t.Cleanup(func() { utils.KubeClient = old })
+	utils.KubeClient = cs
+	t.Setenv(utils.NodeNameEnvName, "n")
+
+	p := &AMDGPUPlugin{}
+	if _, err := p.Allocate(context.Background(), &pluginapi.AllocateRequest{
+		ContainerRequests: []*pluginapi.ContainerAllocateRequest{{DevicesIds: []string{"uuid-a"}}},
+	}); err == nil {
+		t.Fatal("Allocate = nil error on a real pending-pod lookup failure, want it to fail")
+	}
+}
+
 func TestIsSchedulableTopologyKey(t *testing.T) {
 	defer func(n int) { splitCount = n }(splitCount)
 	splitCount = 10 // these tests pin one slot count for every GPU

internal/pkg/utils/utils.go

@@ -19,6 +19,7 @@ package utils
 import (
 	"context"
 	"encoding/json"
+	"errors"
 	"fmt"
 
 	corev1 "k8s.io/api/core/v1"
@@ -28,6 +29,13 @@ import (
 	"k8s.io/klog/v2"
 )
 
+// ErrNoPendingPod is returned by GetPendingPod when no Pod on the node
+// carries HAMi's bind-time annotations, typically because kubelet is
+// asking Allocate for a Pod that a different scheduler or a second device
+// plugin placed on this node's kubelet-reported device ids, outside HAMi's
+// own scheduling flow.
+var ErrNoPendingPod = errors.New("no binding pod found")
+
 func GetNode(nodename string) (*corev1.Node, error) {
 	if nodename == "" {
 		klog.ErrorS(nil, "Node name is empty")
@@ -85,7 +93,7 @@ func GetPendingPod(ctx context.Context, node string) (*corev1.Pod, error) {
 			return &p, nil
 		}
 	}
-	return nil, fmt.Errorf("no binding pod found on node %s", node)
+	return nil, fmt.Errorf("%w on node %s", ErrNoPendingPod, node)
 }
 
 func GetAllocatePodByNode(ctx context.Context, nodeName string) (*corev1.Pod, error) {