From 906c8a7f3273d9c923250d8632f94dd72fcdbef2 Mon Sep 17 00:00:00 2001 From: Zhuoran-Bao Date: Thu, 25 Jun 2026 21:55:57 -0400 Subject: [PATCH] feat(collector): add NUMA topology collector --- README.md | 1 + collector/numa_topology_linux.go | 202 ++++++++++++++++++++++++ collector/numa_topology_linux_test.go | 193 +++++++++++++++++++++++ collector/numatopology/cpulist.go | 43 +++++ collector/numatopology/cpulist_test.go | 36 +++++ collector/numatopology/model.go | 23 +++ collector/numatopology/sysfs.go | 37 +++++ collector/numatopology/sysfs_test.go | 52 +++++++ collector/numatopology/virsh.go | 166 ++++++++++++++++++++ collector/numatopology/virsh_test.go | 208 +++++++++++++++++++++++++ 10 files changed, 961 insertions(+) create mode 100644 collector/numa_topology_linux.go create mode 100644 collector/numa_topology_linux_test.go create mode 100644 collector/numatopology/cpulist.go create mode 100644 collector/numatopology/cpulist_test.go create mode 100644 collector/numatopology/model.go create mode 100644 collector/numatopology/sysfs.go create mode 100644 collector/numatopology/sysfs_test.go create mode 100644 collector/numatopology/virsh.go create mode 100644 collector/numatopology/virsh_test.go diff --git a/README.md b/README.md index 8abf2cdf0f..7371b1eee1 100644 --- a/README.md +++ b/README.md @@ -208,6 +208,7 @@ logind | Exposes session counts from [logind](http://www.freedesktop.org/wiki/So meminfo\_numa | Exposes memory statistics from `/sys/devices/system/node/node[0-9]*/meminfo`, `/sys/devices/system/node/node[0-9]*/numastat`. | Linux mountstats | Exposes filesystem statistics from `/proc/self/mountstats`. Exposes detailed NFS client statistics. | Linux network_route | Exposes the routing table as metrics | Linux +numatopology | Exposes per-NUMA-node CPU and memory capacity from `/sys/devices/system/node/node[0-9]*` and per-VM NUMA assignment parsed from libvirt domain XML files in `/run/libvirt/qemu` (configurable via `--collector.numatopology.libvirt-xml-dir`). Per-VM metrics scale with VM count per host and can be disabled via `--no-collector.numatopology.vm-metrics`. | Linux nvmesubsystem | Exposes NVMe over Fabrics subsystem path health metrics from `/sys/class/nvme-subsystem/`. | Linux pcidevice | Exposes pci devices' information including their link status and parent devices. | Linux perf | Exposes perf based metrics (Warning: Metrics are dependent on kernel configuration and settings). | Linux diff --git a/collector/numa_topology_linux.go b/collector/numa_topology_linux.go new file mode 100644 index 0000000000..f471902c8e --- /dev/null +++ b/collector/numa_topology_linux.go @@ -0,0 +1,202 @@ +// Copyright 2024 The Prometheus Authors +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//go:build !nonumatopology + +package collector + +import ( + "fmt" + "log/slog" + "os" + "path/filepath" + "regexp" + "strconv" + "strings" + + "github.com/alecthomas/kingpin/v2" + "github.com/prometheus/client_golang/prometheus" + "github.com/prometheus/node_exporter/collector/numatopology" +) + +const numaTopologySubsystem = "numatopology" + +var ( + numaTopologyLibvirtXMLDir = kingpin.Flag( + "collector.numatopology.libvirt-xml-dir", + "Directory containing libvirt status XML files.", + ).Default("/run/libvirt/qemu").String() + + numaTopologyVMMetrics = kingpin.Flag( + "collector.numatopology.vm-metrics", + "Enable per-VM NUMA assignment metrics (node_numatopology_vm_cpu, node_numatopology_vm_memory_bytes). Cardinality scales with VM count per host.", + ).Default("true").Bool() + + numaNodeDirRE = regexp.MustCompile(`node(\d+)$`) +) + +type numaTopologyCollector struct { + logger *slog.Logger + libvirtXMLDir string + emitVMMetrics bool + + cpuCapacity *prometheus.Desc + memCapacity *prometheus.Desc + cpuUsed *prometheus.Desc + memUsed *prometheus.Desc + vmCPU *prometheus.Desc + vmMemBytes *prometheus.Desc +} + +func init() { + registerCollector(numaTopologySubsystem, defaultDisabled, NewNumaTopologyCollector) +} + +// NewNumaTopologyCollector returns a new Collector exposing NUMA topology metrics. +func NewNumaTopologyCollector(logger *slog.Logger) (Collector, error) { + return &numaTopologyCollector{ + logger: logger, + libvirtXMLDir: *numaTopologyLibvirtXMLDir, + emitVMMetrics: *numaTopologyVMMetrics, + cpuCapacity: prometheus.NewDesc( + prometheus.BuildFQName(namespace, numaTopologySubsystem, "cpu_capacity"), + "Number of physical CPUs on this NUMA node.", + []string{"node"}, nil, + ), + memCapacity: prometheus.NewDesc( + prometheus.BuildFQName(namespace, numaTopologySubsystem, "memory_capacity_bytes"), + "Total memory on this NUMA node in bytes.", + []string{"node"}, nil, + ), + cpuUsed: prometheus.NewDesc( + prometheus.BuildFQName(namespace, numaTopologySubsystem, "cpu_used"), + "vCPUs assigned to NUMA-pinned VMs on this node.", + []string{"node"}, nil, + ), + memUsed: prometheus.NewDesc( + prometheus.BuildFQName(namespace, numaTopologySubsystem, "memory_used_bytes"), + "Memory assigned to NUMA-pinned VMs on this node in bytes.", + []string{"node"}, nil, + ), + vmCPU: prometheus.NewDesc( + prometheus.BuildFQName(namespace, numaTopologySubsystem, "vm_cpu"), + "vCPUs assigned to a specific VM on this NUMA node. Cardinality scales with VM count per host.", + []string{"node", "vm"}, nil, + ), + vmMemBytes: prometheus.NewDesc( + prometheus.BuildFQName(namespace, numaTopologySubsystem, "vm_memory_bytes"), + "Memory assigned to a specific VM on this NUMA node in bytes. Cardinality scales with VM count per host.", + []string{"node", "vm"}, nil, + ), + }, nil +} + +// Update implements Collector. +func (c *numaTopologyCollector) Update(ch chan<- prometheus.Metric) error { + nodeDirs, err := filepath.Glob(sysFilePath("devices/system/node/node[0-9]*")) + if err != nil { + return fmt.Errorf("globbing NUMA node sysfs: %w", err) + } + if len(nodeDirs) == 0 { + return fmt.Errorf("no NUMA nodes found under %s", sysFilePath("devices/system/node")) + } + + type nodeCapacity struct { + cpus int + mem int64 + } + capacities := make(map[string]nodeCapacity, len(nodeDirs)) + + for _, dir := range nodeDirs { + m := numaNodeDirRE.FindStringSubmatch(dir) + if m == nil { + continue + } + nodeID := m[1] + + cpulistBytes, err := os.ReadFile(filepath.Join(dir, "cpulist")) + if err != nil { + c.logger.Warn("reading cpulist", "node", nodeID, "err", err) + continue + } + + meminfoBytes, err := os.ReadFile(filepath.Join(dir, "meminfo")) + if err != nil { + c.logger.Warn("reading meminfo", "node", nodeID, "err", err) + continue + } + + cpuCount := numatopology.CountCPUList(strings.TrimSpace(string(cpulistBytes))) + memBytes, err := numatopology.ParseMeminfo(string(meminfoBytes)) + if err != nil { + c.logger.Warn("parsing meminfo", "node", nodeID, "err", err) + continue + } + + capacities[nodeID] = nodeCapacity{cpus: cpuCount, mem: memBytes} + } + + if len(capacities) == 0 { + return fmt.Errorf("no NUMA nodes with readable sysfs data") + } + + // vmUsage: node ID string → vm name → [vCPUs, memBytes] + vmUsage := make(map[string]map[string][2]int64) + + xmlFiles, err := filepath.Glob(filepath.Join(c.libvirtXMLDir, "*.xml")) + if err != nil { + c.logger.Warn("globbing libvirt XML dir", "dir", c.libvirtXMLDir, "err", err) + } + for _, xmlFile := range xmlFiles { + data, err := os.ReadFile(xmlFile) + if err != nil { + c.logger.Warn("reading libvirt XML", "file", xmlFile, "err", err) + continue + } + res, err := numatopology.ParseVirshXML(string(data)) + if err != nil { + c.logger.Warn("parsing libvirt XML", "file", xmlFile, "err", err) + continue + } + if res == nil { + continue // not NUMA-pinned + } + for hostNode, usage := range res.HostNUMAUsage { + nodeStr := strconv.Itoa(hostNode) + if vmUsage[nodeStr] == nil { + vmUsage[nodeStr] = make(map[string][2]int64) + } + prev := vmUsage[nodeStr][res.VMName] + vmUsage[nodeStr][res.VMName] = [2]int64{prev[0] + usage[0], prev[1] + usage[1]} + } + } + + for nodeID, cap := range capacities { + ch <- prometheus.MustNewConstMetric(c.cpuCapacity, prometheus.GaugeValue, float64(cap.cpus), nodeID) + ch <- prometheus.MustNewConstMetric(c.memCapacity, prometheus.GaugeValue, float64(cap.mem), nodeID) + + var totalCPU, totalMem int64 + for vmName, usage := range vmUsage[nodeID] { + totalCPU += usage[0] + totalMem += usage[1] + if c.emitVMMetrics { + ch <- prometheus.MustNewConstMetric(c.vmCPU, prometheus.GaugeValue, float64(usage[0]), nodeID, vmName) + ch <- prometheus.MustNewConstMetric(c.vmMemBytes, prometheus.GaugeValue, float64(usage[1]), nodeID, vmName) + } + } + ch <- prometheus.MustNewConstMetric(c.cpuUsed, prometheus.GaugeValue, float64(totalCPU), nodeID) + ch <- prometheus.MustNewConstMetric(c.memUsed, prometheus.GaugeValue, float64(totalMem), nodeID) + } + + return nil +} diff --git a/collector/numa_topology_linux_test.go b/collector/numa_topology_linux_test.go new file mode 100644 index 0000000000..6769b72492 --- /dev/null +++ b/collector/numa_topology_linux_test.go @@ -0,0 +1,193 @@ +// Copyright 2024 The Prometheus Authors +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//go:build linux && !nonumatopology + +package collector + +import ( + "io" + "log/slog" + "os" + "path/filepath" + "strings" + "testing" + + "github.com/prometheus/client_golang/prometheus" + "github.com/prometheus/client_golang/prometheus/testutil" +) + +// testNumaTopologyCollector wraps the collector to satisfy prometheus.Collector. +type testNumaTopologyCollector struct { + c Collector +} + +func (tc testNumaTopologyCollector) Collect(ch chan<- prometheus.Metric) { + tc.c.Update(ch) +} + +func (tc testNumaTopologyCollector) Describe(ch chan<- *prometheus.Desc) { + prometheus.DescribeByCollect(tc, ch) +} + +// pinnedDomainXML is a file with VM "test-vm" pinned to both NUMA nodes. +// Cell 0 → host node 0: 2 vCPUs (0-1), 2097152 KiB = 2147483648 bytes +// Cell 1 → host node 1: 2 vCPUs (2-3), 2097152 KiB = 2147483648 bytes +const pinnedDomainXML = ` + + instance-0000001a + + test-vm + + + + + + + + + + + + +` + +// unpinnedDomainXML has no elements — must be skipped by the collector. +const unpinnedDomainXML = ` + + instance-0000002b + + +` + +func TestNumaTopologyCollector(t *testing.T) { + // Point sysfs at the existing test fixtures. + *sysPath = "fixtures/sys" + *numaTopologyVMMetrics = true + + // Create a temp dir with libvirt XML files. + libvirtDir := t.TempDir() + if err := os.WriteFile(filepath.Join(libvirtDir, "instance-pinned.xml"), []byte(pinnedDomainXML), 0644); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(filepath.Join(libvirtDir, "instance-unpinned.xml"), []byte(unpinnedDomainXML), 0644); err != nil { + t.Fatal(err) + } + *numaTopologyLibvirtXMLDir = libvirtDir + + logger := slog.New(slog.NewTextHandler(io.Discard, nil)) + c, err := NewNumaTopologyCollector(logger) + if err != nil { + t.Fatal(err) + } + + reg := prometheus.NewRegistry() + reg.MustRegister(&testNumaTopologyCollector{c: c}) + + // Metric values derived from sys.ttar fixtures and pinnedDomainXML above. + // node0 MemTotal: 134182340 kB * 1024 = 137402716160 bytes + // node1 MemTotal: 134217728 kB * 1024 = 137438953472 bytes + // node2 MemTotal: 134217728 kB * 1024 = 137438953472 bytes (cpulist empty → 0 CPUs) + // VM mem per node: 2097152 KiB * 1024 = 2147483648 bytes + expected := ` + # HELP node_numatopology_cpu_capacity Number of physical CPUs on this NUMA node. + # TYPE node_numatopology_cpu_capacity gauge + node_numatopology_cpu_capacity{node="0"} 2 + node_numatopology_cpu_capacity{node="1"} 2 + node_numatopology_cpu_capacity{node="2"} 0 + # HELP node_numatopology_cpu_used vCPUs assigned to NUMA-pinned VMs on this node. + # TYPE node_numatopology_cpu_used gauge + node_numatopology_cpu_used{node="0"} 2 + node_numatopology_cpu_used{node="1"} 2 + node_numatopology_cpu_used{node="2"} 0 + # HELP node_numatopology_memory_capacity_bytes Total memory on this NUMA node in bytes. + # TYPE node_numatopology_memory_capacity_bytes gauge + node_numatopology_memory_capacity_bytes{node="0"} 1.3740271616e+11 + node_numatopology_memory_capacity_bytes{node="1"} 1.37438953472e+11 + node_numatopology_memory_capacity_bytes{node="2"} 1.37438953472e+11 + # HELP node_numatopology_memory_used_bytes Memory assigned to NUMA-pinned VMs on this node in bytes. + # TYPE node_numatopology_memory_used_bytes gauge + node_numatopology_memory_used_bytes{node="0"} 2.147483648e+09 + node_numatopology_memory_used_bytes{node="1"} 2.147483648e+09 + node_numatopology_memory_used_bytes{node="2"} 0 + # HELP node_numatopology_vm_cpu vCPUs assigned to a specific VM on this NUMA node. Cardinality scales with VM count per host. + # TYPE node_numatopology_vm_cpu gauge + node_numatopology_vm_cpu{node="0",vm="test-vm"} 2 + node_numatopology_vm_cpu{node="1",vm="test-vm"} 2 + # HELP node_numatopology_vm_memory_bytes Memory assigned to a specific VM on this NUMA node in bytes. Cardinality scales with VM count per host. + # TYPE node_numatopology_vm_memory_bytes gauge + node_numatopology_vm_memory_bytes{node="0",vm="test-vm"} 2.147483648e+09 + node_numatopology_vm_memory_bytes{node="1",vm="test-vm"} 2.147483648e+09 + ` + + if err := testutil.GatherAndCompare(reg, strings.NewReader(expected)); err != nil { + t.Fatal(err) + } +} + +func TestNumaTopologyCollectorVMMetricsDisabled(t *testing.T) { + *sysPath = "fixtures/sys" + + libvirtDir := t.TempDir() + if err := os.WriteFile(filepath.Join(libvirtDir, "instance-pinned.xml"), []byte(pinnedDomainXML), 0644); err != nil { + t.Fatal(err) + } + *numaTopologyLibvirtXMLDir = libvirtDir + + *numaTopologyVMMetrics = false + t.Cleanup(func() { *numaTopologyVMMetrics = true }) + + logger := slog.New(slog.NewTextHandler(io.Discard, nil)) + c, err := NewNumaTopologyCollector(logger) + if err != nil { + t.Fatal(err) + } + + reg := prometheus.NewRegistry() + reg.MustRegister(&testNumaTopologyCollector{c: c}) + + // Per-node aggregates still emit; per-VM metrics must be absent. + expected := ` + # HELP node_numatopology_cpu_capacity Number of physical CPUs on this NUMA node. + # TYPE node_numatopology_cpu_capacity gauge + node_numatopology_cpu_capacity{node="0"} 2 + node_numatopology_cpu_capacity{node="1"} 2 + node_numatopology_cpu_capacity{node="2"} 0 + # HELP node_numatopology_cpu_used vCPUs assigned to NUMA-pinned VMs on this node. + # TYPE node_numatopology_cpu_used gauge + node_numatopology_cpu_used{node="0"} 2 + node_numatopology_cpu_used{node="1"} 2 + node_numatopology_cpu_used{node="2"} 0 + # HELP node_numatopology_memory_capacity_bytes Total memory on this NUMA node in bytes. + # TYPE node_numatopology_memory_capacity_bytes gauge + node_numatopology_memory_capacity_bytes{node="0"} 1.3740271616e+11 + node_numatopology_memory_capacity_bytes{node="1"} 1.37438953472e+11 + node_numatopology_memory_capacity_bytes{node="2"} 1.37438953472e+11 + # HELP node_numatopology_memory_used_bytes Memory assigned to NUMA-pinned VMs on this node in bytes. + # TYPE node_numatopology_memory_used_bytes gauge + node_numatopology_memory_used_bytes{node="0"} 2.147483648e+09 + node_numatopology_memory_used_bytes{node="1"} 2.147483648e+09 + node_numatopology_memory_used_bytes{node="2"} 0 + ` + + if err := testutil.GatherAndCompare(reg, strings.NewReader(expected), + "node_numatopology_cpu_capacity", + "node_numatopology_cpu_used", + "node_numatopology_memory_capacity_bytes", + "node_numatopology_memory_used_bytes", + "node_numatopology_vm_cpu", + "node_numatopology_vm_memory_bytes", + ); err != nil { + t.Fatal(err) + } +} diff --git a/collector/numatopology/cpulist.go b/collector/numatopology/cpulist.go new file mode 100644 index 0000000000..65d509d0d7 --- /dev/null +++ b/collector/numatopology/cpulist.go @@ -0,0 +1,43 @@ +// Copyright 2024 The Prometheus Authors +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package numatopology + +import ( + "strconv" + "strings" +) + +// CountCPUList counts the CPUs described by a sysfs cpulist string. +// Examples: "0" → 1, "0-3" → 4, "0-3,8-11" → 8. +func CountCPUList(cpulist string) int { + total := 0 + for _, part := range strings.Split(cpulist, ",") { + part = strings.TrimSpace(part) + if part == "" { + continue + } + if idx := strings.IndexByte(part, '-'); idx != -1 { + lo, err1 := strconv.Atoi(part[:idx]) + hi, err2 := strconv.Atoi(part[idx+1:]) + if err1 == nil && err2 == nil && hi >= lo { + total += hi - lo + 1 + } + } else { + if _, err := strconv.Atoi(part); err == nil { + total++ + } + } + } + return total +} diff --git a/collector/numatopology/cpulist_test.go b/collector/numatopology/cpulist_test.go new file mode 100644 index 0000000000..e4c7d1ff23 --- /dev/null +++ b/collector/numatopology/cpulist_test.go @@ -0,0 +1,36 @@ +// Copyright 2024 The Prometheus Authors +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package numatopology + +import "testing" + +func TestCountCPUList(t *testing.T) { + tests := []struct { + input string + want int + }{ + {"", 0}, + {"0", 1}, + {"0-3", 4}, + {"0-3,8-11", 8}, + {"0,2,4", 3}, + {"0-1,4-7", 6}, + } + for _, tt := range tests { + got := CountCPUList(tt.input) + if got != tt.want { + t.Errorf("CountCPUList(%q) = %d, want %d", tt.input, got, tt.want) + } + } +} diff --git a/collector/numatopology/model.go b/collector/numatopology/model.go new file mode 100644 index 0000000000..7e625a24c1 --- /dev/null +++ b/collector/numatopology/model.go @@ -0,0 +1,23 @@ +// Copyright 2024 The Prometheus Authors +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Package numatopology provides parsing utilities for NUMA topology discovery. +package numatopology + +// VirshResult holds the VM name and per-host-NUMA-node resource usage +// extracted from a libvirt domain XML. +type VirshResult struct { + VMName string + // HostNUMAUsage maps host NUMA node ID → [vCPUs, memBytes]. + HostNUMAUsage map[int][2]int64 +} diff --git a/collector/numatopology/sysfs.go b/collector/numatopology/sysfs.go new file mode 100644 index 0000000000..43fe153b06 --- /dev/null +++ b/collector/numatopology/sysfs.go @@ -0,0 +1,37 @@ +// Copyright 2024 The Prometheus Authors +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package numatopology + +import ( + "fmt" + "regexp" + "strconv" +) + +var reMemTotal = regexp.MustCompile(`MemTotal:\s+(\d+)\s+kB`) + +// ParseMeminfo extracts MemTotal from sysfs node meminfo content and returns bytes. +// The sysfs format is "Node N MemTotal: NNNNNN kB"; the regex matches the +// "MemTotal: N kB" suffix regardless of any leading "Node N" prefix. +func ParseMeminfo(content string) (int64, error) { + m := reMemTotal.FindStringSubmatch(content) + if m == nil { + return 0, fmt.Errorf("MemTotal not found in meminfo") + } + kib, err := strconv.ParseInt(m[1], 10, 64) + if err != nil { + return 0, fmt.Errorf("parsing MemTotal: %w", err) + } + return kib * 1024, nil +} diff --git a/collector/numatopology/sysfs_test.go b/collector/numatopology/sysfs_test.go new file mode 100644 index 0000000000..deb65e9624 --- /dev/null +++ b/collector/numatopology/sysfs_test.go @@ -0,0 +1,52 @@ +// Copyright 2024 The Prometheus Authors +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package numatopology + +import "testing" + +func TestParseMeminfo(t *testing.T) { + tests := []struct { + name string + input string + want int64 + wantErr bool + }{ + { + name: "standard sysfs meminfo format", + input: "Node 0 MemTotal: 16384000 kB\nNode 0 MemFree: 4096 kB\n", + want: 16384000 * 1024, + }, + { + name: "inline MemTotal without node prefix", + input: "MemTotal: 8192000 kB\n", + want: 8192000 * 1024, + }, + { + name: "missing MemTotal", + input: "Node 0 MemFree: 4096 kB\n", + wantErr: true, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got, err := ParseMeminfo(tt.input) + if (err != nil) != tt.wantErr { + t.Fatalf("ParseMeminfo() error = %v, wantErr %v", err, tt.wantErr) + } + if !tt.wantErr && got != tt.want { + t.Errorf("ParseMeminfo() = %d, want %d", got, tt.want) + } + }) + } +} diff --git a/collector/numatopology/virsh.go b/collector/numatopology/virsh.go new file mode 100644 index 0000000000..4808b5f68a --- /dev/null +++ b/collector/numatopology/virsh.go @@ -0,0 +1,166 @@ +// Copyright 2024 The Prometheus Authors +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package numatopology + +import ( + "encoding/xml" + "fmt" + "math" + "strconv" + "strings" +) + +type domain struct { + XMLName xml.Name `xml:"domain"` + Name string `xml:"name"` + NovaInstance novaInstance `xml:"http://openstack.org/xmlns/libvirt/nova/1.0 instance"` + CPU domainCPU `xml:"cpu"` + NumaTune numaTune `xml:"numatune"` +} + +type novaInstance struct { + Name string `xml:"http://openstack.org/xmlns/libvirt/nova/1.0 name"` +} + +type domainCPU struct { + NUMA domainNUMA `xml:"numa"` +} + +type domainNUMA struct { + Cells []numaCell `xml:"cell"` +} + +type numaCell struct { + ID string `xml:"id,attr"` + CPUs string `xml:"cpus,attr"` + Memory string `xml:"memory,attr"` + Unit string `xml:"unit,attr"` +} + +type numaTune struct { + MemNodes []memNode `xml:"memnode"` +} + +type memNode struct { + CellID string `xml:"cellid,attr"` + Mode string `xml:"mode,attr"` + NodeSet string `xml:"nodeset,attr"` +} + +type domstatusWrapper struct { + XMLName xml.Name `xml:"domstatus"` + Domain domain `xml:"domain"` +} + +// ParseVirshXML parses libvirt domain XML in either format: +// - root (output of "virsh dumpxml") +// - root (files in /run/libvirt/qemu/*.xml) +// +// Returns (nil, nil) when the domain has no explicit NUMA pinning +// (no elements in ). +// Returns (nil, err) on XML parse errors. +func ParseVirshXML(xmlStr string) (*VirshResult, error) { + var d domain + if err := xml.Unmarshal([]byte(xmlStr), &d); err != nil { + var ds domstatusWrapper + if err2 := xml.Unmarshal([]byte(xmlStr), &ds); err2 != nil { + return nil, fmt.Errorf("parsing domain XML: %w", err2) + } + d = ds.Domain + } + + if len(d.NumaTune.MemNodes) == 0 { + return nil, nil + } + + vmName := strings.TrimSpace(d.NovaInstance.Name) + if vmName == "" { + vmName = strings.TrimSpace(d.Name) + } + + cellToHostNode := make(map[int]int, len(d.NumaTune.MemNodes)) + for _, mn := range d.NumaTune.MemNodes { + cellID, err := strconv.Atoi(strings.TrimSpace(mn.CellID)) + if err != nil { + continue + } + hostNode, err := strconv.Atoi(strings.TrimSpace(mn.NodeSet)) + if err != nil { + // Multiple-node sets cannot be represented by VirshResult without + // incorrectly attributing all of the cell's resources to one node. + continue + } + cellToHostNode[cellID] = hostNode + } + + hostNUMAUsage := make(map[int][2]int64) + for _, cell := range d.CPU.NUMA.Cells { + cellID, err := strconv.Atoi(strings.TrimSpace(cell.ID)) + if err != nil { + continue + } + hostNode, ok := cellToHostNode[cellID] + if !ok { + continue + } + cpuCount := int64(CountCPUList(cell.CPUs)) + memBytes, err := memoryBytes(cell.Memory, cell.Unit) + if err != nil { + return nil, fmt.Errorf("parsing memory for NUMA cell %q: %w", cell.ID, err) + } + + prev := hostNUMAUsage[hostNode] + hostNUMAUsage[hostNode] = [2]int64{prev[0] + cpuCount, prev[1] + memBytes} + } + + return &VirshResult{ + VMName: vmName, + HostNUMAUsage: hostNUMAUsage, + }, nil +} + +func memoryBytes(value, unit string) (int64, error) { + memory, err := strconv.ParseInt(strings.TrimSpace(value), 10, 64) + if err != nil { + return 0, fmt.Errorf("invalid value %q: %w", value, err) + } + if memory < 0 { + return 0, fmt.Errorf("invalid negative value %q", value) + } + + var multiplier int64 + switch strings.TrimSpace(unit) { + case "bytes": + multiplier = 1 + case "KB": + multiplier = 1000 + case "", "KiB": + multiplier = 1024 + case "MB": + multiplier = 1000 * 1000 + case "MiB": + multiplier = 1024 * 1024 + case "GB": + multiplier = 1000 * 1000 * 1000 + case "GiB": + multiplier = 1024 * 1024 * 1024 + default: + return 0, fmt.Errorf("unsupported unit %q", unit) + } + + if memory > math.MaxInt64/multiplier { + return 0, fmt.Errorf("value %q %s overflows bytes", value, unit) + } + return memory * multiplier, nil +} diff --git a/collector/numatopology/virsh_test.go b/collector/numatopology/virsh_test.go new file mode 100644 index 0000000000..51f0188daf --- /dev/null +++ b/collector/numatopology/virsh_test.go @@ -0,0 +1,208 @@ +// Copyright 2024 The Prometheus Authors +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package numatopology + +import ( + "strings" + "testing" +) + +// virshXMLNUMAPinned is root format (from "virsh dumpxml"). +const virshXMLNUMAPinned = ` + instance-0000001a + + my-vm-01 + + + + + + + + + + + +` + +// virshXMLDomstatus mirrors the format of /run/libvirt/qemu/*.xml files. +const virshXMLDomstatus = ` + + instance-0000001a + + my-vm-01 + + + + + + + + + + +` + +// virshXMLNotPinned has no — not NUMA-pinned. +const virshXMLNotPinned = ` + instance-0000002b + +` + +// virshXMLFallbackName tests that libvirt domain name is used when nova:name is absent. +const virshXMLFallbackName = ` + instance-0000003c + + + + + + + + +` + +func TestParseVirshXMLNUMAPinned(t *testing.T) { + res, err := ParseVirshXML(virshXMLNUMAPinned) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if res == nil { + t.Fatal("expected non-nil result for NUMA-pinned VM") + } + if res.VMName != "my-vm-01" { + t.Errorf("VMName = %q, want %q", res.VMName, "my-vm-01") + } + + n0, ok := res.HostNUMAUsage[0] + if !ok { + t.Fatal("host NUMA node 0 not found") + } + if n0[0] != 4 { // CountCPUList("0-3") = 4 + t.Errorf("node 0 vCPUs = %d, want 4", n0[0]) + } + if n0[1] != 4194304*1024 { // 4194304 KiB in bytes + t.Errorf("node 0 memBytes = %d, want %d", n0[1], int64(4194304*1024)) + } + + n1, ok := res.HostNUMAUsage[1] + if !ok { + t.Fatal("host NUMA node 1 not found") + } + if n1[0] != 4 { + t.Errorf("node 1 vCPUs = %d, want 4", n1[0]) + } + if n1[1] != 4194304*1024 { + t.Errorf("node 1 memBytes = %d, want %d", n1[1], int64(4194304*1024)) + } +} + +func TestParseVirshXMLDomstatus(t *testing.T) { + res, err := ParseVirshXML(virshXMLDomstatus) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if res == nil { + t.Fatal("expected non-nil result for domstatus NUMA-pinned VM") + } + if res.VMName != "my-vm-01" { + t.Errorf("VMName = %q, want %q", res.VMName, "my-vm-01") + } + n0, ok := res.HostNUMAUsage[0] + if !ok { + t.Fatal("host NUMA node 0 not found") + } + if n0[0] != 4 { + t.Errorf("node 0 vCPUs = %d, want 4", n0[0]) + } +} + +func TestParseVirshXMLNotPinned(t *testing.T) { + res, err := ParseVirshXML(virshXMLNotPinned) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if res != nil { + t.Errorf("expected nil for non-NUMA-pinned VM, got %+v", res) + } +} + +func TestParseVirshXMLFallbackName(t *testing.T) { + res, err := ParseVirshXML(virshXMLFallbackName) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if res == nil { + t.Fatal("expected non-nil result") + } + if res.VMName != "instance-0000003c" { + t.Errorf("VMName = %q, want %q", res.VMName, "instance-0000003c") + } +} + +func TestParseVirshXMLInvalid(t *testing.T) { + _, err := ParseVirshXML("vm` + res, err := ParseVirshXML(xml) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if got := res.HostNUMAUsage[0][1]; got != tt.want { + t.Errorf("memory = %d bytes, want %d", got, tt.want) + } + }) + } +} + +func TestParseVirshXMLSkipsMultipleNodeSet(t *testing.T) { + xml := `vm` + res, err := ParseVirshXML(xml) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if len(res.HostNUMAUsage) != 0 { + t.Errorf("expected no host attribution for multiple-node set, got %+v", res.HostNUMAUsage) + } +} + +func TestParseVirshXMLReturnsDomstatusError(t *testing.T) { + _, err := ParseVirshXML("") + if err == nil { + t.Fatal("expected error for unexpected root element") + } + if !strings.Contains(err.Error(), "domstatus") { + t.Errorf("error = %q, want domstatus unmarshal error", err) + } +}