From ef1240b940f137dd78773b7cc039e3d25f22bfbd Mon Sep 17 00:00:00 2001 From: pingxin403 Date: Sun, 5 Jul 2026 16:04:34 +0800 Subject: [PATCH 1/2] fix: enable Ascend NPU container-level metrics in exporter and chart display Replace hardcoded return 0, nil with real PromQL queries (npu_chip_info_utilization, npu_chip_info_hbm_used_memory) for Ascend taskCoreUsed/taskMemoryUsed. Add AscendGPUDevice case to GenerateContainerMetrics switch for core unit conversion. Remove NVIDIA/MXC type guards in task detail view so all vendors show trend charts. Closes #86 Signed-off-by: pingxin403 --- .../projects/vgpu/views/task/admin/Detail.vue | 17 ++- server/internal/exporter/exporter.go | 15 +-- server/internal/exporter/exporter_test.go | 100 ++++++++++++++++++ 3 files changed, 115 insertions(+), 17 deletions(-) create mode 100644 server/internal/exporter/exporter_test.go diff --git a/packages/web/projects/vgpu/views/task/admin/Detail.vue b/packages/web/projects/vgpu/views/task/admin/Detail.vue index 4d43f8a9..15785fb7 100644 --- a/packages/web/projects/vgpu/views/task/admin/Detail.vue +++ b/packages/web/projects/vgpu/views/task/admin/Detail.vue @@ -150,23 +150,18 @@
- - +
diff --git a/server/internal/exporter/exporter.go b/server/internal/exporter/exporter.go index 636831d4..2639714d 100644 --- a/server/internal/exporter/exporter.go +++ b/server/internal/exporter/exporter.go @@ -399,10 +399,13 @@ func (s *MetricsGenerator) GenerateContainerMetrics(ctx context.Context) error { case biz.HygonGPUDevice: used = float64(taskCoreUsed) util = roundToOneDecimal(100 * float64(taskCoreUsed) / float64(core)) - case metax.MetaxSGPUDevice: - used = float64(taskCoreUsed) - util = roundToOneDecimal(100 * float64(taskCoreUsed) / float64(core)) - default: + case biz.AscendGPUDevice: + used = float64(taskCoreUsed) / 100 * float64(core) + util = float64(taskCoreUsed) + case metax.MetaxSGPUDevice: + used = float64(taskCoreUsed) + util = roundToOneDecimal(100 * float64(taskCoreUsed) / float64(core)) + default: } cardCoreUtil, err := s.deviceCoreUtil(ctx, provider, device.Id) if err == nil && used != 0 && cardCoreUtil > 95 { @@ -540,7 +543,7 @@ func (s *MetricsGenerator) taskCoreUsed(ctx context.Context, provider, namespace case biz.CambriconGPUDevice: query = fmt.Sprintf("avg(mlu_utilization * on(uuid) group_right mlu_container{namespace=\"%s\",pod=\"%s\",container=\"%s\",type=\"mlu370.smlu.vcore\"})", namespace, pod, container) case biz.AscendGPUDevice: - return 0, nil + query = fmt.Sprintf("avg(npu_chip_info_utilization{vdie_id=\"%s\"})", deviceUUID) case biz.HygonGPUDevice: query = fmt.Sprintf("avg(vdcu_percent{pod_uuid=\"%s\", container_name=\"%s\"})", podUUID, container) case biz.MetaxGPUDevice, metax.MetaxGPUDevice: @@ -563,7 +566,7 @@ func (s *MetricsGenerator) taskMemoryUsed(ctx context.Context, provider, namespa case biz.CambriconGPUDevice: query = fmt.Sprintf("avg(mlu_memory_utilization * on(uuid) group_right mlu_container{namespace=\"%s\",pod=\"%s\",container=\"%s\",type=\"mlu370.smlu.vmemory\"})", namespace, pod, container) case biz.AscendGPUDevice: - return 0, nil + query = fmt.Sprintf("avg(npu_chip_info_hbm_used_memory{vdie_id=\"%s\"})", deviceUUID) case biz.HygonGPUDevice: query = fmt.Sprintf("avg(vdcu_usage_memory_size{pod_uuid=\"%s\", container_name=\"%s\"})", podUUID, container) case metax.MetaxGPUDevice: diff --git a/server/internal/exporter/exporter_test.go b/server/internal/exporter/exporter_test.go new file mode 100644 index 00000000..86b2fc51 --- /dev/null +++ b/server/internal/exporter/exporter_test.go @@ -0,0 +1,100 @@ +package exporter + +import ( + "context" + "fmt" + "io" + "net/http" + "net/http/httptest" + "strings" + "testing" + "time" + + "vgpu/internal/biz" + "vgpu/internal/data/prom" + "vgpu/internal/service" + + "github.com/go-kratos/kratos/v2/log" +) + +// testPromHandler returns a handler that acts as a fake Prometheus /api/v1/query +// endpoint. When value is non-empty, it returns a single-sample vector with that +// value. When value is empty, it returns an empty result vector. +func testPromHandler(t *testing.T, value string) http.HandlerFunc { + return func(w http.ResponseWriter, r *http.Request) { + if r.Method != http.MethodPost && r.Method != http.MethodGet { + t.Errorf("unexpected method: %s", r.Method) + } + if !strings.HasSuffix(r.URL.Path, "/api/v1/query") { + t.Errorf("unexpected path: %s", r.URL.Path) + } + w.Header().Set("Content-Type", "application/json") + if value == "" { + fmt.Fprint(w, `{"status":"success","data":{"resultType":"vector","result":[]}}`) + return + } + fmt.Fprintf(w, `{"status":"success","data":{"resultType":"vector","result":[{"metric":{},"value":[1719500000,%q]}]}}`, value) + } +} + +// newTestGenerator starts an httptest server with the given handler, creates a +// prom.Client pointed at it, wraps it in a MonitorService, and returns a bare +// MetricsGenerator (other fields left nil). The returned close func shuts down +// the test server. +func newTestGenerator(t *testing.T, handler http.HandlerFunc) (*MetricsGenerator, func()) { + srv := httptest.NewServer(handler) + + promClient, err := prom.NewClient(srv.URL, time.Second, "") + if err != nil { + srv.Close() + t.Fatalf("prom.NewClient: %v", err) + } + + monitorSvc := service.NewMonitorService(promClient, nil, nil) + + gen := &MetricsGenerator{ + monitorService: monitorSvc, + log: log.NewHelper(log.NewStdLogger(io.Discard)), + } + + return gen, srv.Close +} + +func TestMetricsGenerator_taskCoreUsed_Ascend(t *testing.T) { + gen, cleanup := newTestGenerator(t, testPromHandler(t, "42.5")) + defer cleanup() + + result, err := gen.taskCoreUsed(context.Background(), biz.AscendGPUDevice, "ns", "pod", "ctr", "pod-uuid", "dev-uuid", "host", 0) + if err != nil { + t.Fatalf("taskCoreUsed failed: %v", err) + } + if result != 42.5 { + t.Errorf("expected 42.5, got %v", result) + } +} + +func TestMetricsGenerator_taskMemoryUsed_Ascend(t *testing.T) { + gen, cleanup := newTestGenerator(t, testPromHandler(t, "8589934592")) + defer cleanup() + + result, err := gen.taskMemoryUsed(context.Background(), biz.AscendGPUDevice, "ns", "pod", "ctr", "pod-uuid", "dev-uuid", "host", 0) + if err != nil { + t.Fatalf("taskMemoryUsed failed: %v", err) + } + if result != 8589934592 { + t.Errorf("expected 8589934592, got %v", result) + } +} + +func TestMetricsGenerator_taskCoreUsed_Ascend_EmptyResult(t *testing.T) { + gen, cleanup := newTestGenerator(t, testPromHandler(t, "")) + defer cleanup() + + result, err := gen.taskCoreUsed(context.Background(), biz.AscendGPUDevice, "ns", "pod", "ctr", "pod-uuid", "dev-uuid", "host", 0) + if err != nil { + t.Fatalf("taskCoreUsed failed: %v", err) + } + if result != 0 { + t.Errorf("expected 0, got %v", result) + } +} From 2d2366ea257fe7b70601f205c874a0cc0b9ecb0e Mon Sep 17 00:00:00 2001 From: pingxin403 Date: Sun, 5 Jul 2026 16:50:32 +0800 Subject: [PATCH 2/2] fix: convert Ascend HBM memory from MB to bytes for correct container metrics npu_chip_info_hbm_used_memory returns MB (megabytes, 10^6 bytes), not bytes. The downstream /1024/1024 conversion expects bytes input for MiB output. Fix: multiply by 1000*1000 to convert MB to bytes before the division. Address CodeRabbit review comment on MB vs byte unit mismatch. Signed-off-by: pingxin403 --- server/internal/exporter/exporter.go | 3 +++ server/internal/exporter/exporter_test.go | 7 ++++--- 2 files changed, 7 insertions(+), 3 deletions(-) diff --git a/server/internal/exporter/exporter.go b/server/internal/exporter/exporter.go index 2639714d..3e2ad2b8 100644 --- a/server/internal/exporter/exporter.go +++ b/server/internal/exporter/exporter.go @@ -418,6 +418,9 @@ func (s *MetricsGenerator) GenerateContainerMetrics(ctx context.Context) error { taskMemoryUsed, err := s.taskMemoryUsed(ctx, provider, c.Namespace, c.PodName, c.Name, c.PodUID, device.Id, device.NodeName, device.Index) if err == nil { switch provider { + case biz.AscendGPUDevice: + // npu_chip_info_hbm_used_memory returns MB (10^6 bytes), convert to bytes for downstream /1024/1024 → MiB + taskMemoryUsed = float32(float64(taskMemoryUsed) * 1000 * 1000) case biz.CambriconGPUDevice: taskMemoryUsed = float32((taskMemoryUsed/100)*float32(memory)) * 1024 * 1024 case metax.MetaxSGPUDevice: diff --git a/server/internal/exporter/exporter_test.go b/server/internal/exporter/exporter_test.go index 86b2fc51..9a704888 100644 --- a/server/internal/exporter/exporter_test.go +++ b/server/internal/exporter/exporter_test.go @@ -74,15 +74,16 @@ func TestMetricsGenerator_taskCoreUsed_Ascend(t *testing.T) { } func TestMetricsGenerator_taskMemoryUsed_Ascend(t *testing.T) { - gen, cleanup := newTestGenerator(t, testPromHandler(t, "8589934592")) + // npu_chip_info_hbm_used_memory returns MB, e.g., 8192 for 8 GB HBM + gen, cleanup := newTestGenerator(t, testPromHandler(t, "8192")) defer cleanup() result, err := gen.taskMemoryUsed(context.Background(), biz.AscendGPUDevice, "ns", "pod", "ctr", "pod-uuid", "dev-uuid", "host", 0) if err != nil { t.Fatalf("taskMemoryUsed failed: %v", err) } - if result != 8589934592 { - t.Errorf("expected 8589934592, got %v", result) + if result != 8192 { + t.Errorf("expected 8192 (MB), got %v", result) } }