From 042de95474228fa6da211af51c500448c6768340 Mon Sep 17 00:00:00 2001 From: Reza Jelveh Date: Wed, 19 Aug 2026 13:56:22 +0800 Subject: [PATCH 1/5] fix: admit whole-GPU allocations via kubelet device ids The fork scheduler writes hami.io/amd-devices-allocated keyed by AMDGPU- with Usedcores=0, but the plugin reads hami.io/amd-devices-to-allocate, so Allocate failed before returning. When the annotation is unavailable, fall back to kubelet's own device ids, resolving split ids ("#") through the topology BDF map. Whole-GPU allocations (Usedcores=0) mask every CU (HSA_CU_MASK 0:0-255) and keep the LD_AUDIT hook; CU occupancy persistence and the node lock check apply only to sliced (cores>0) allocations. Verified: vllm-amd (Qwen2.5-0.5B) and example/vllm-serve (mistral-7b) admit and serve inference on the MI355X node. Signed-off-by: Reza Jelveh --- internal/pkg/plugin/plugin.go | 71 +++++++++++++++++++++++++++-------- 1 file changed, 55 insertions(+), 16 deletions(-) diff --git a/internal/pkg/plugin/plugin.go b/internal/pkg/plugin/plugin.go index d5a4d62f..94238b92 100644 --- a/internal/pkg/plugin/plugin.go +++ b/internal/pkg/plugin/plugin.go @@ -64,6 +64,9 @@ type AMDGPUPlugin struct { // amdSMIUUIDToROCrUUID maps the scheduler-facing AMD SMI UUID to the UUID // spelling understood by ROCR_VISIBLE_DEVICES. amdSMIUUIDToROCrUUID map[string]string + // bdfToROCrUUID maps the topology BDF to its ROCr UUID for kubelet split + // ids ("#") resolved during Allocate. + bdfToROCrUUID map[string]string } type AMDGPUPluginOption func(*AMDGPUPlugin) @@ -173,6 +176,7 @@ func (p *AMDGPUPlugin) getAPIDevices() []*utils.DeviceInfo { } p.amdSMIUUIDToTopology = make(map[string]string, len(amdSMIUUIDs)) p.amdSMIUUIDToROCrUUID = make(map[string]string, len(amdSMIUUIDs)) + p.bdfToROCrUUID = make(map[string]string, len(p.AMDGPUs)) rocrUUIDs := amdgpu.GetROCrUUIDsFromTopology() amdSMIProductNames, err := amdgpu.GetAMDSMIProductNames(bdfs) if err != nil { @@ -222,6 +226,7 @@ func (p *AMDGPUPlugin) getAPIDevices() []*utils.DeviceInfo { // device node. Allocate uses the in-memory UUID -> topology map below. p.amdSMIUUIDToTopology[uuid] = key p.amdSMIUUIDToROCrUUID[uuid] = rocrUUID + p.bdfToROCrUUID[key] = rocrUUID // key is the standard PCI BDF spelling (domain:bus:device.function). // The KFD topology bdf above uses a fourth colon-separated component. customInfo := map[string]any{"pciBDF": strings.ToLower(key)} @@ -575,7 +580,12 @@ func (p *AMDGPUPlugin) deviceDataFromAllocationUUID(uuid, _ string) (map[string] } return nil, fmt.Errorf("AMD SMI UUID %q resolves to unavailable topology key %q", uuid, topoKey) } - + // kubelet split ids are "#"; resolve the bare BDF directly. + if bdf := strings.SplitN(uuid, "#", 2)[0]; bdf != uuid { + if d, found := p.AMDGPUs[bdf]; found { + return d, nil + } + } return nil, fmt.Errorf("no local GPU topology entry for AMD SMI UUID %q", uuid) } @@ -583,6 +593,11 @@ func (p *AMDGPUPlugin) rocrUUIDFromAllocationUUID(uuid string) (string, error) { if rocrUUID, ok := p.amdSMIUUIDToROCrUUID[uuid]; ok && rocrUUID != "" { return rocrUUID, nil } + if bdf := strings.SplitN(uuid, "#", 2)[0]; bdf != uuid { + if rocrUUID, ok := p.bdfToROCrUUID[bdf]; ok && rocrUUID != "" { + return rocrUUID, nil + } + } return "", fmt.Errorf("no ROCr UUID for AMD SMI UUID %q", uuid) } @@ -619,16 +634,26 @@ func (p *AMDGPUPlugin) Allocate(ctx context.Context, r *pluginapi.AllocateReques hostHookPath := os.Getenv("HOST_HOOK_PATH") glog.Infof("Allocate pod name is %s/%s, annotation is %+v", current.Namespace, current.Name, current.Annotations) + // True when any container requests sliced (cores > 0) devices; the CU + // persistence and node lock guard shared occupancy only. + slicedCU := false for idx := range r.ContainerRequests { currentCtr, devreq, err := utils.GetNextDeviceRequest("amd", *current) glog.Infof("deviceAllocateFromAnnotation(container=%s)=%+v", currentCtr.Name, devreq) - if err != nil { - utils.PodAllocationFailed(nodename, current, NodeLockName) - return &pluginapi.AllocateResponse{}, err - } - if len(devreq) != len(r.ContainerRequests[idx].DevicesIDs) { - utils.PodAllocationFailed(nodename, current, NodeLockName) - return &pluginapi.AllocateResponse{}, fmt.Errorf("device number not matched") + if err != nil || len(devreq) != len(r.ContainerRequests[idx].DevicesIDs) { + // The fork scheduler's annotation key is not the one this plugin + // reads; fall back to kubelet's own allocation (whole GPU). + glog.Warningf("annotation allocation unavailable (%v); using kubelet device ids %v", err, r.ContainerRequests[idx].DevicesIDs) + devreq = utils.ContainerDevices{} + for _, id := range r.ContainerRequests[idx].DevicesIDs { + devreq = append(devreq, utils.ContainerDevice{UUID: id, Type: "AMDGPU", Usedcores: 0}) + } + } else { + err = utils.EraseNextDeviceTypeFromAnnotation("amd", *current) + if err != nil { + utils.PodAllocationFailed(nodename, current, NodeLockName) + return &pluginapi.AllocateResponse{}, err + } } car := pluginapi.ContainerAllocateResponse{ @@ -677,12 +702,6 @@ func (p *AMDGPUPlugin) Allocate(ctx context.Context, r *pluginapi.AllocateReques } } - err = utils.EraseNextDeviceTypeFromAnnotation("amd", *current) - if err != nil { - utils.PodAllocationFailed(nodename, current, NodeLockName) - return &pluginapi.AllocateResponse{}, err - } - if len(devreq) > 0 { hsaCuSets := make([]string, 0, len(devreq)) for _, d := range devreq { @@ -707,7 +726,12 @@ func (p *AMDGPUPlugin) Allocate(ctx context.Context, r *pluginapi.AllocateReques } cuAllocationSnapshot[d.UUID] = baseAllocation } - _, deltaAllocation, err := cuallocation.AllocateN(baseAllocation, totalCUs, int(d.Usedcores)) + cores := int(d.Usedcores) + if cores == 0 { + // Whole GPU: mask every CU. + cores = totalCUs + } + _, deltaAllocation, err := cuallocation.AllocateN(baseAllocation, totalCUs, cores) if err != nil { utils.PodAllocationFailed(nodename, current, NodeLockName) return &pluginapi.AllocateResponse{}, fmt.Errorf("allocate cu for %s: %w", d.UUID, err) @@ -717,6 +741,7 @@ func (p *AMDGPUPlugin) Allocate(ctx context.Context, r *pluginapi.AllocateReques // Use container-local device index as GPU_list and ID_List as CU_list. hsaCuSets = append(hsaCuSets, fmt.Sprintf("%d:%s", i, cuList)) + slicedCU = slicedCU || d.Usedcores > 0 if oldList, ok := podCuAllocList[d.UUID]; ok && strings.TrimSpace(oldList) != "" { oldAllocation, err := idListToAllocation(oldList, totalCUs) if err != nil { @@ -752,7 +777,9 @@ func (p *AMDGPUPlugin) Allocate(ctx context.Context, r *pluginapi.AllocateReques response.ContainerResponses = append(response.ContainerResponses, &car) } - if len(podCuAllocList) > 0 { + // Whole-GPU allocations own the full card; their CU occupancy is not + // shared, so there is nothing to persist and no node lock to check. + if len(podCuAllocList) > 0 && slicedCU { b, err := json.Marshal(podCuAllocList) if err != nil { utils.PodAllocationFailed(nodename, current, NodeLockName) @@ -887,6 +914,18 @@ func (p *AMDGPUPlugin) getDeviceTotalCUs(uuid string) (int, error) { } return int(d.Devcore), nil } + // kubelet split ids ("#") do not match DeviceInfo.ID; resolve + // the bare BDF against the registered pciBDF. + if bdf := strings.SplitN(uuid, "#", 2)[0]; bdf != uuid { + for _, d := range p.deviceCache { + if d == nil || d.Devcore <= 0 { + continue + } + if pciBDF, ok := d.CustomInfo["pciBDF"].(string); ok && pciBDF == bdf { + return int(d.Devcore), nil + } + } + } return 0, fmt.Errorf("device %s not found in device cache", uuid) } From 10d63c25336996556071638b5bb845a8829367f5 Mon Sep 17 00:00:00 2001 From: Reza Jelveh Date: Wed, 19 Aug 2026 13:56:34 +0800 Subject: [PATCH 2/5] fix: tolerate unknown device ids in GPA policy GetPreferredAllocation can receive device ids that are not in the policy's device map (another plugin's registration on the same node). The previous code appended nil entries and panicked in the sort comparator, killing the plugin process and leaving kubelet with no healthy devices. Skip unknown ids instead. Signed-off-by: Reza Jelveh --- internal/pkg/allocator/besteffort_policy.go | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/internal/pkg/allocator/besteffort_policy.go b/internal/pkg/allocator/besteffort_policy.go index 38828659..1b536d1c 100644 --- a/internal/pkg/allocator/besteffort_policy.go +++ b/internal/pkg/allocator/besteffort_policy.go @@ -61,7 +61,11 @@ func NewBestEffortPolicy() *BestEffortPolicy { func (b *BestEffortPolicy) getDevicesFromIds(ids []string) []*Device { var res []*Device for _, id := range ids { - res = append(res, b.devicesMap[id]) + // Kubelet may report ids not in this plugin's device map (stale + // checkpoint entries); skip them instead of panicking downstream. + if d, ok := b.devicesMap[id]; ok { + res = append(res, d) + } } return res } From e34db40263097ecfbab8bebec31a64e3e69d4277 Mon Sep 17 00:00:00 2001 From: Reza Jelveh Date: Wed, 19 Aug 2026 13:56:43 +0800 Subject: [PATCH 3/5] fix(helm): drop allowPrivilegeEscalation from `securityContext` Pairing it with privileged is rejected by apply. Signed-off-by: Reza Jelveh --- helm/amd-gpu/values.yaml | 1 - 1 file changed, 1 deletion(-) diff --git a/helm/amd-gpu/values.yaml b/helm/amd-gpu/values.yaml index 737bf2b1..ea485a28 100644 --- a/helm/amd-gpu/values.yaml +++ b/helm/amd-gpu/values.yaml @@ -11,7 +11,6 @@ dp: # The plugin queries host GPU devices and passes them to workloads. securityContext: privileged: true - allowPrivilegeEscalation: false capabilities: drop: - ALL From 1a3b96d89d891bda1b8c7b75002f8035b2791bc4 Mon Sep 17 00:00:00 2001 From: Reza Jelveh Date: Wed, 19 Aug 2026 13:56:43 +0800 Subject: [PATCH 4/5] example(vllm-serve): use gfx950 vLLM image for MI355X rocm6.2_mi300 (gfx942) has no gfx950 support: torch reports "No HIP GPUs are available" at startup. Use the rocm7.13 gfx950 build that serves on the MI355X. Signed-off-by: Reza Jelveh --- example/vllm-serve/deployment.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/example/vllm-serve/deployment.yaml b/example/vllm-serve/deployment.yaml index 0b8246ee..9441447e 100644 --- a/example/vllm-serve/deployment.yaml +++ b/example/vllm-serve/deployment.yaml @@ -25,7 +25,7 @@ spec: hostIPC: true containers: - name: mistral-7b - image: rocm/vllm:rocm6.2_mi300_ubuntu20.04_py3.9_vllm_0.6.4 + image: rocm/vllm:rocm7.13.0_gfx950-dcgpu_ubuntu24.04_py3.13_pytorch_2.10.0_vllm_0.19.1 securityContext: seccompProfile: type: Unconfined From ab8b91322677c48ab50060539ee8396bec4cd40a Mon Sep 17 00:00:00 2001 From: Reza Jelveh Date: Wed, 19 Aug 2026 17:26:06 +0800 Subject: [PATCH 5/5] allocator: warn prominently when GPA sees unknown device ids Skipping unknown ids prevents the panic, but a silent fix hides the cause: another device plugin running on the same node merged its devices into kubelet's list. Log a warning naming the likely cause and the remedy (disable the other plugin, restart this one). Signed-off-by: Reza Jelveh --- internal/pkg/allocator/besteffort_policy.go | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/internal/pkg/allocator/besteffort_policy.go b/internal/pkg/allocator/besteffort_policy.go index 1b536d1c..685d945b 100644 --- a/internal/pkg/allocator/besteffort_policy.go +++ b/internal/pkg/allocator/besteffort_policy.go @@ -61,11 +61,12 @@ func NewBestEffortPolicy() *BestEffortPolicy { func (b *BestEffortPolicy) getDevicesFromIds(ids []string) []*Device { var res []*Device for _, id := range ids { - // Kubelet may report ids not in this plugin's device map (stale - // checkpoint entries); skip them instead of panicking downstream. - if d, ok := b.devicesMap[id]; ok { - res = append(res, d) + d, ok := b.devicesMap[id] + if !ok { + glog.Warningf("GPA policy: device id %q is not registered by this plugin; a second device plugin is likely running on this node and kubelet merged its devices. Disable the other plugin (or drain the node) and restart this plugin", id) + continue } + res = append(res, d) } return res }