From 646b6cddbeb6f7f5d6f4eb88740622c6e02a4edf Mon Sep 17 00:00:00 2001 From: houyuxi Date: Fri, 26 Jun 2026 19:35:46 +0800 Subject: [PATCH 1/6] fix(ascend): proportionally split card-level metrics for vnpu container metrics Signed-off-by: houyuxi --- server/internal/exporter/exporter.go | 52 +++++++++++++++++++++++++--- 1 file changed, 48 insertions(+), 4 deletions(-) diff --git a/server/internal/exporter/exporter.go b/server/internal/exporter/exporter.go index 636831d..50ad67e 100644 --- a/server/internal/exporter/exporter.go +++ b/server/internal/exporter/exporter.go @@ -363,6 +363,33 @@ func (s *MetricsGenerator) GenerateContainerMetrics(ctx context.Context) error { s.generateMetricsForMetaxGPU(containers) for _, device := range deviceInfos { + // === Ascend pre-calculation: card-level metrics + total allocation === + // ponytail: proportional split — npu-exporter doesn't support vnpu for 910B/A3 yet. + // Card-level metrics are divided across containers by Usedmem ratio. + var ascendCardUtil float32 + var ascendCardMemUsedMB float32 + var ascendTotalMemoryOnCard int32 + var ascendCardQueriesOK bool + if strings.HasPrefix(device.Provider, biz.AscendGPUDevice) { + var cdMemBytes float32 + var ascendCardUtilErr, ascendCardMemErr error + ascendCardUtil, ascendCardUtilErr = s.deviceCoreUtil(ctx, device.Provider, device.Id) + cdMemBytes, ascendCardMemErr = s.deviceMemUsed(ctx, device.Provider, device.Id) + ascendCardQueriesOK = ascendCardUtilErr == nil && ascendCardMemErr == nil + if ascendCardQueriesOK && cdMemBytes > 0 { + ascendCardMemUsedMB = cdMemBytes / 1024 / 1024 + } + for _, c := range containers { + for _, cd := range c.ContainerDevices { + if device.AliasId != "" && !device.MatchAlias(cd.UUID) { + continue + } + if strings.HasPrefix(cd.Type, biz.AscendGPUDevice) { + ascendTotalMemoryOnCard += cd.Usedmem + } + } + } + } for _, c := range containers { var vGPU int32 = 0 var core int32 = 0 @@ -385,8 +412,15 @@ func (s *MetricsGenerator) GenerateContainerMetrics(ctx context.Context) error { s.set(HamiContainerVmemoryAllocated, float64(memory), device.NodeName, provider, device.Type, device.Id, c.PodName, c.Name, c.Namespace, podUIDLabel) s.set(HamiContainerVcoreAllocated, float64(core), device.NodeName, provider, device.Type, device.Id, c.PodName, c.Name, c.Namespace, podUIDLabel) // 查询任务在当前设备下的算力利用率 - taskCoreUsed, err := s.taskCoreUsed(ctx, provider, c.Namespace, c.PodName, c.Name, c.PodUID, device.Id, device.NodeName, device.Index) - if err == nil { + var taskCoreUsed float32 + var taskCoreUsedErr error + if provider == biz.AscendGPUDevice && ascendCardQueriesOK && ascendTotalMemoryOnCard > 0 { + ratio := float32(memory) / float32(ascendTotalMemoryOnCard) + taskCoreUsed = ascendCardUtil * ratio + } else { + taskCoreUsed, taskCoreUsedErr = s.taskCoreUsed(ctx, provider, c.Namespace, c.PodName, c.Name, c.PodUID, device.Id, device.NodeName, device.Index) + } + if taskCoreUsedErr == nil { used := float64(0) util := float64(0) switch provider { @@ -396,6 +430,9 @@ func (s *MetricsGenerator) GenerateContainerMetrics(ctx context.Context) error { case biz.CambriconGPUDevice: used = float64(taskCoreUsed) / 100 * float64(core) util = float64(taskCoreUsed) + case biz.AscendGPUDevice: + used = float64(taskCoreUsed) + util = roundToOneDecimal(100 * float64(taskCoreUsed) / float64(core)) case biz.HygonGPUDevice: used = float64(taskCoreUsed) util = roundToOneDecimal(100 * float64(taskCoreUsed) / float64(core)) @@ -412,8 +449,15 @@ func (s *MetricsGenerator) GenerateContainerMetrics(ctx context.Context) error { s.set(HamiContainerCoreUsed, used, device.NodeName, provider, device.Type, device.Id, c.PodName, c.Name, c.Namespace) s.set(HamiContainerCoreUtil, util, device.NodeName, provider, device.Type, device.Id, c.PodName, c.Name, c.Namespace) } - taskMemoryUsed, err := s.taskMemoryUsed(ctx, provider, c.Namespace, c.PodName, c.Name, c.PodUID, device.Id, device.NodeName, device.Index) - if err == nil { + var taskMemoryUsed float32 + var taskMemoryUsedErr error + if provider == biz.AscendGPUDevice && ascendCardQueriesOK && ascendTotalMemoryOnCard > 0 { + ratio := float32(memory) / float32(ascendTotalMemoryOnCard) + taskMemoryUsed = ascendCardMemUsedMB * ratio + } else { + taskMemoryUsed, taskMemoryUsedErr = s.taskMemoryUsed(ctx, provider, c.Namespace, c.PodName, c.Name, c.PodUID, device.Id, device.NodeName, device.Index) + } + if taskMemoryUsedErr == nil { switch provider { case biz.CambriconGPUDevice: taskMemoryUsed = float32((taskMemoryUsed/100)*float32(memory)) * 1024 * 1024 From 1a3b61e18faac6ea5055a0e6888a5ee4161f7c3f Mon Sep 17 00:00:00 2001 From: houyuxi Date: Mon, 29 Jun 2026 12:05:50 +0800 Subject: [PATCH 2/6] fix(ascend): fix compile error, add query warn logs, and keep memory in bytes MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Replace undefined MatchAlias with strings.HasPrefix for pre-calculation loop, matching the metrics loop predicate - Add Warnf logs when Ascend card util/mem queries fail - Keep ascendCardMemUsedBytes in bytes instead of converting to MB early, so the common export path /1024/1024 conversion is correct ponytail: proportional split — npu-exporter doesn't support vnpu yet Signed-off-by: houyuxi --- server/internal/exporter/exporter.go | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/server/internal/exporter/exporter.go b/server/internal/exporter/exporter.go index 50ad67e..9fb581b 100644 --- a/server/internal/exporter/exporter.go +++ b/server/internal/exporter/exporter.go @@ -367,7 +367,7 @@ func (s *MetricsGenerator) GenerateContainerMetrics(ctx context.Context) error { // ponytail: proportional split — npu-exporter doesn't support vnpu for 910B/A3 yet. // Card-level metrics are divided across containers by Usedmem ratio. var ascendCardUtil float32 - var ascendCardMemUsedMB float32 + var ascendCardMemUsedBytes float32 var ascendTotalMemoryOnCard int32 var ascendCardQueriesOK bool if strings.HasPrefix(device.Provider, biz.AscendGPUDevice) { @@ -375,13 +375,19 @@ func (s *MetricsGenerator) GenerateContainerMetrics(ctx context.Context) error { var ascendCardUtilErr, ascendCardMemErr error ascendCardUtil, ascendCardUtilErr = s.deviceCoreUtil(ctx, device.Provider, device.Id) cdMemBytes, ascendCardMemErr = s.deviceMemUsed(ctx, device.Provider, device.Id) + if ascendCardUtilErr != nil { + s.log.Warnf("failed to query Ascend card util for device %s: %v", device.Id, ascendCardUtilErr) + } + if ascendCardMemErr != nil { + s.log.Warnf("failed to query Ascend card mem for device %s: %v", device.Id, ascendCardMemErr) + } ascendCardQueriesOK = ascendCardUtilErr == nil && ascendCardMemErr == nil if ascendCardQueriesOK && cdMemBytes > 0 { - ascendCardMemUsedMB = cdMemBytes / 1024 / 1024 + ascendCardMemUsedBytes = cdMemBytes } for _, c := range containers { for _, cd := range c.ContainerDevices { - if device.AliasId != "" && !device.MatchAlias(cd.UUID) { + if device.AliasId != "" && !strings.HasPrefix(cd.UUID, device.AliasId) { continue } if strings.HasPrefix(cd.Type, biz.AscendGPUDevice) { @@ -453,7 +459,7 @@ func (s *MetricsGenerator) GenerateContainerMetrics(ctx context.Context) error { var taskMemoryUsedErr error if provider == biz.AscendGPUDevice && ascendCardQueriesOK && ascendTotalMemoryOnCard > 0 { ratio := float32(memory) / float32(ascendTotalMemoryOnCard) - taskMemoryUsed = ascendCardMemUsedMB * ratio + taskMemoryUsed = ascendCardMemUsedBytes * ratio } else { taskMemoryUsed, taskMemoryUsedErr = s.taskMemoryUsed(ctx, provider, c.Namespace, c.PodName, c.Name, c.PodUID, device.Id, device.NodeName, device.Index) } From a79368917e932db59fa8d690af5dce2b6cf1b897 Mon Sep 17 00:00:00 2001 From: houyuxi Date: Mon, 29 Jun 2026 14:43:01 +0800 Subject: [PATCH 3/6] ci: clean comments Signed-off-by: houyuxi --- server/internal/exporter/exporter.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/server/internal/exporter/exporter.go b/server/internal/exporter/exporter.go index 9fb581b..fd4a1df 100644 --- a/server/internal/exporter/exporter.go +++ b/server/internal/exporter/exporter.go @@ -364,7 +364,7 @@ func (s *MetricsGenerator) GenerateContainerMetrics(ctx context.Context) error { s.generateMetricsForMetaxGPU(containers) for _, device := range deviceInfos { // === Ascend pre-calculation: card-level metrics + total allocation === - // ponytail: proportional split — npu-exporter doesn't support vnpu for 910B/A3 yet. + // npu-exporter doesn't support vnpu for 910B/A3 yet. // Card-level metrics are divided across containers by Usedmem ratio. var ascendCardUtil float32 var ascendCardMemUsedBytes float32 From f71748f6a5162b616d24567720275fdf63d50466 Mon Sep 17 00:00:00 2001 From: houyuxi Date: Fri, 3 Jul 2026 11:18:30 +0800 Subject: [PATCH 4/6] fix: add ascend switch case for taskMemory Signed-off-by: houyuxi --- server/internal/exporter/exporter.go | 12 +++++------- 1 file changed, 5 insertions(+), 7 deletions(-) diff --git a/server/internal/exporter/exporter.go b/server/internal/exporter/exporter.go index fd4a1df..e581e42 100644 --- a/server/internal/exporter/exporter.go +++ b/server/internal/exporter/exporter.go @@ -367,14 +367,13 @@ func (s *MetricsGenerator) GenerateContainerMetrics(ctx context.Context) error { // npu-exporter doesn't support vnpu for 910B/A3 yet. // Card-level metrics are divided across containers by Usedmem ratio. var ascendCardUtil float32 - var ascendCardMemUsedBytes float32 + var ascendCardMemUsedMB float32 var ascendTotalMemoryOnCard int32 var ascendCardQueriesOK bool if strings.HasPrefix(device.Provider, biz.AscendGPUDevice) { - var cdMemBytes float32 var ascendCardUtilErr, ascendCardMemErr error ascendCardUtil, ascendCardUtilErr = s.deviceCoreUtil(ctx, device.Provider, device.Id) - cdMemBytes, ascendCardMemErr = s.deviceMemUsed(ctx, device.Provider, device.Id) + ascendCardMemUsedMB, ascendCardMemErr = s.deviceMemUsed(ctx, device.Provider, device.Id) if ascendCardUtilErr != nil { s.log.Warnf("failed to query Ascend card util for device %s: %v", device.Id, ascendCardUtilErr) } @@ -382,9 +381,6 @@ func (s *MetricsGenerator) GenerateContainerMetrics(ctx context.Context) error { s.log.Warnf("failed to query Ascend card mem for device %s: %v", device.Id, ascendCardMemErr) } ascendCardQueriesOK = ascendCardUtilErr == nil && ascendCardMemErr == nil - if ascendCardQueriesOK && cdMemBytes > 0 { - ascendCardMemUsedBytes = cdMemBytes - } for _, c := range containers { for _, cd := range c.ContainerDevices { if device.AliasId != "" && !strings.HasPrefix(cd.UUID, device.AliasId) { @@ -459,7 +455,7 @@ func (s *MetricsGenerator) GenerateContainerMetrics(ctx context.Context) error { var taskMemoryUsedErr error if provider == biz.AscendGPUDevice && ascendCardQueriesOK && ascendTotalMemoryOnCard > 0 { ratio := float32(memory) / float32(ascendTotalMemoryOnCard) - taskMemoryUsed = ascendCardMemUsedBytes * ratio + taskMemoryUsed = ascendCardMemUsedMB * ratio } else { taskMemoryUsed, taskMemoryUsedErr = s.taskMemoryUsed(ctx, provider, c.Namespace, c.PodName, c.Name, c.PodUID, device.Id, device.NodeName, device.Index) } @@ -467,6 +463,8 @@ func (s *MetricsGenerator) GenerateContainerMetrics(ctx context.Context) error { switch provider { case biz.CambriconGPUDevice: taskMemoryUsed = float32((taskMemoryUsed/100)*float32(memory)) * 1024 * 1024 + case biz.AscendGPUDevice: + taskMemoryUsed = taskMemoryUsed * 1024 * 1024 case metax.MetaxSGPUDevice: taskMemoryUsed = float32(taskMemoryUsed) * 1024 // KB->Byte default: From 45d39935bddfeb2cb504003297b76c1734dee2eb Mon Sep 17 00:00:00 2001 From: houyuxi Date: Fri, 3 Jul 2026 16:41:21 +0800 Subject: [PATCH 5/6] fix(ascend): normalize provider from cd.Type to canonical Ascend value Container devices may report sub-types like "Ascend910C" or "Ascend310P" which do not match `case biz.AscendGPUDevice` in subsequent switch/case. Normalizing to "Ascend" at assignment eliminates the need to convert every comparison site to HasPrefix. Signed-off-by: houyuxi --- server/internal/exporter/exporter.go | 3 +++ 1 file changed, 3 insertions(+) diff --git a/server/internal/exporter/exporter.go b/server/internal/exporter/exporter.go index e581e42..8f3a65a 100644 --- a/server/internal/exporter/exporter.go +++ b/server/internal/exporter/exporter.go @@ -405,6 +405,9 @@ func (s *MetricsGenerator) GenerateContainerMetrics(ctx context.Context) error { core = core + cd.Usedcores memory = memory + cd.Usedmem provider = cd.Type + if strings.HasPrefix(provider, biz.AscendGPUDevice) { + provider = biz.AscendGPUDevice + } } if provider == "" || provider == metax.MetaxGPUDevice { continue From 994264fab0f01c4d5ee27995c61c83466b5e5865 Mon Sep 17 00:00:00 2001 From: houyuxi Date: Fri, 3 Jul 2026 15:49:12 +0800 Subject: [PATCH 6/6] fix: node device provider use == Signed-off-by: houyuxi --- server/internal/exporter/exporter.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/server/internal/exporter/exporter.go b/server/internal/exporter/exporter.go index 8f3a65a..8d1afab 100644 --- a/server/internal/exporter/exporter.go +++ b/server/internal/exporter/exporter.go @@ -370,7 +370,7 @@ func (s *MetricsGenerator) GenerateContainerMetrics(ctx context.Context) error { var ascendCardMemUsedMB float32 var ascendTotalMemoryOnCard int32 var ascendCardQueriesOK bool - if strings.HasPrefix(device.Provider, biz.AscendGPUDevice) { + if device.Provider == biz.AscendGPUDevice { var ascendCardUtilErr, ascendCardMemErr error ascendCardUtil, ascendCardUtilErr = s.deviceCoreUtil(ctx, device.Provider, device.Id) ascendCardMemUsedMB, ascendCardMemErr = s.deviceMemUsed(ctx, device.Provider, device.Id)