From 6930b76e50793aff92705508e37667cbc7438783 Mon Sep 17 00:00:00 2001 From: mesutoezdil Date: Sun, 11 Oct 2026 13:32:07 +0200 Subject: [PATCH] fix(plugin): serve whole GPUs when no HAMi-tracked pod is pending Allocate returned "no binding pod found" for a Pod that carries none of HAMi's bind-time annotations (placed by another scheduler or alongside a second device plugin), so the whole-GPU fallback from the kubelet device ids was never reached and the Pod failed with UnexpectedAdmissionError. Treat ErrNoPendingPod as that case: use an empty Pod, serve the kubelet ids as whole GPUs, and skip the node-lock release and dmem cap that need a real Pod. Other lookup errors still fail Allocate. Signed-off-by: mesutoezdil --- internal/pkg/plugin/plugin.go | 25 +++++++++++--- internal/pkg/plugin/plugin_test.go | 54 ++++++++++++++++++++++++++++++ internal/pkg/utils/utils.go | 10 +++++- 3 files changed, 84 insertions(+), 5 deletions(-) diff --git a/internal/pkg/plugin/plugin.go b/internal/pkg/plugin/plugin.go index c8e97661..cc1f1794 100644 --- a/internal/pkg/plugin/plugin.go +++ b/internal/pkg/plugin/plugin.go @@ -20,6 +20,7 @@ package plugin import ( "context" "encoding/json" + "errors" "fmt" "os" "os/signal" @@ -1039,8 +1040,20 @@ func (p *AMDGPUPlugin) Allocate(ctx context.Context, r *pluginapi.AllocateReques glog.Infof("Allocate request: %+v", r) nodename := os.Getenv(utils.NodeNameEnvName) current, err := utils.GetPendingPod(ctx, nodename) - if err != nil { - return &pluginapi.AllocateResponse{}, err + trackedPod := err == nil + if !trackedPod { + if !errors.Is(err, utils.ErrNoPendingPod) { + return &pluginapi.AllocateResponse{}, err + } + // No Pod on this node carries HAMi's bind-time annotations, most + // often because a different scheduler or a second device plugin + // placed this Pod directly on the kubelet-reported device ids + // (see besteffort_policy.go's same assumption on the list side). + // Serve the whole GPUs kubelet asked for instead of failing the + // Pod; there is no HAMi-tracked Pod to persist CU state against + // or to release a node lock for. + glog.Warningf("no HAMi-tracked pod pending on node %s (%v); serving kubelet device ids %v as whole GPUs", nodename, err, r.ContainerRequests) + current = &corev1.Pod{} } podCuAllocList := map[string]string{} if raw := strings.TrimSpace(current.Annotations[utils.CuAllocation]); raw != "" { @@ -1261,13 +1274,15 @@ func (p *AMDGPUPlugin) Allocate(ctx context.Context, r *pluginapi.AllocateReques ReadOnly: true, }) } - if p.dmemEnabled { + if p.dmemEnabled && trackedPod { // The pod's cgroup can take several seconds to appear // after binding (measured up to ~7s on a real RKE2 // node), well past what Allocate should ever block for. // Apply the cap in the background; libamvgpu's // HIP_DEVICE_MEMORY_LIMIT_ above is already the // synchronous enforcement path when auditCompatible. + // Skipped without a HAMi-tracked pod: there is no real + // namespace/name/UID to resolve a cgroup path from. podNamespace, podName, podUID, qos := current.Namespace, current.Name, string(current.UID), current.Status.QOSClass for _, d := range devreq { go p.applyDmemCap(podNamespace, podName, podUID, qos, d) @@ -1308,7 +1323,9 @@ func (p *AMDGPUPlugin) Allocate(ctx context.Context, r *pluginapi.AllocateReques } glog.Infoln("Allocate Response", response.ContainerResponses) - utils.PodAllocationTrySuccess(nodename, NodeLockName, current) + if trackedPod { + utils.PodAllocationTrySuccess(nodename, NodeLockName, current) + } return response, nil } diff --git a/internal/pkg/plugin/plugin_test.go b/internal/pkg/plugin/plugin_test.go index 9bba2755..c2e69d93 100644 --- a/internal/pkg/plugin/plugin_test.go +++ b/internal/pkg/plugin/plugin_test.go @@ -301,6 +301,60 @@ func TestAllocateWholeGPUSkipsCUCommit(t *testing.T) { } } +// A Pod a different scheduler or a second device plugin placed on this +// node's kubelet-reported device ids carries none of HAMi's bind-time +// annotations, so GetPendingPod finds nothing. Allocate must still serve the +// kubelet device ids as a whole GPU instead of failing the Pod. +func TestAllocateWithoutHAMiTrackedPod(t *testing.T) { + node := &corev1.Node{ObjectMeta: metav1.ObjectMeta{Name: "n"}} + cs := fake.NewSimpleClientset(node) // no Pod carries the bind-time annotations + old := utils.KubeClient + t.Cleanup(func() { utils.KubeClient = old }) + utils.KubeClient = cs + t.Setenv(utils.NodeNameEnvName, "n") + + p := &AMDGPUPlugin{ + AMDGPUs: map[string]map[string]interface{}{"0000:06:00.0": {"card": 1, "renderD": 129}}, + amdSMIUUIDToTopology: map[string]string{"uuid-a": "0000:06:00.0"}, + amdSMIUUIDToROCrUUID: map[string]string{"uuid-a": "GPU-1"}, + deviceCache: []*utils.DeviceInfo{{ID: "uuid-a", Devcore: 32, Devmem: 16304}}, + } + resp, err := p.Allocate(context.Background(), &pluginapi.AllocateRequest{ + ContainerRequests: []*pluginapi.ContainerAllocateRequest{{DevicesIds: []string{"uuid-a"}}}, + }) + if err != nil { + t.Fatalf("Allocate without a HAMi-tracked pod = %v, want the kubelet device ids served as a whole GPU", err) + } + if got := len(resp.ContainerResponses); got != 1 { + t.Fatalf("ContainerResponses = %d, want 1", got) + } + if _, maskSet := resp.ContainerResponses[0].Envs["HSA_CU_MASK"]; !maskSet { + t.Error("whole-GPU fallback must still set HSA_CU_MASK") + } + if got := resp.ContainerResponses[0].Envs["ROCR_VISIBLE_DEVICES"]; got != "GPU-1" { + t.Errorf("ROCR_VISIBLE_DEVICES = %q, want GPU-1", got) + } +} + +// A real error getting the pending pod (API server trouble, not simply no +// bound pod found) must still fail Allocate rather than silently falling +// back, so kubelet retries instead of starting a container without the +// scheduler's intended device. +func TestAllocateFailsOnRealPendingPodError(t *testing.T) { + cs := fake.NewSimpleClientset() // no Node "n" either: GetAllocatePodByNode errors + old := utils.KubeClient + t.Cleanup(func() { utils.KubeClient = old }) + utils.KubeClient = cs + t.Setenv(utils.NodeNameEnvName, "n") + + p := &AMDGPUPlugin{} + if _, err := p.Allocate(context.Background(), &pluginapi.AllocateRequest{ + ContainerRequests: []*pluginapi.ContainerAllocateRequest{{DevicesIds: []string{"uuid-a"}}}, + }); err == nil { + t.Fatal("Allocate = nil error on a real pending-pod lookup failure, want it to fail") + } +} + func TestIsSchedulableTopologyKey(t *testing.T) { defer func(n int) { splitCount = n }(splitCount) splitCount = 10 // these tests pin one slot count for every GPU diff --git a/internal/pkg/utils/utils.go b/internal/pkg/utils/utils.go index c83173b1..704972a6 100644 --- a/internal/pkg/utils/utils.go +++ b/internal/pkg/utils/utils.go @@ -19,6 +19,7 @@ package utils import ( "context" "encoding/json" + "errors" "fmt" corev1 "k8s.io/api/core/v1" @@ -28,6 +29,13 @@ import ( "k8s.io/klog/v2" ) +// ErrNoPendingPod is returned by GetPendingPod when no Pod on the node +// carries HAMi's bind-time annotations, typically because kubelet is +// asking Allocate for a Pod that a different scheduler or a second device +// plugin placed on this node's kubelet-reported device ids, outside HAMi's +// own scheduling flow. +var ErrNoPendingPod = errors.New("no binding pod found") + func GetNode(nodename string) (*corev1.Node, error) { if nodename == "" { klog.ErrorS(nil, "Node name is empty") @@ -85,7 +93,7 @@ func GetPendingPod(ctx context.Context, node string) (*corev1.Pod, error) { return &p, nil } } - return nil, fmt.Errorf("no binding pod found on node %s", node) + return nil, fmt.Errorf("%w on node %s", ErrNoPendingPod, node) } func GetAllocatePodByNode(ctx context.Context, nodeName string) (*corev1.Pod, error) {