From 0b94860e5942e64e9b715a10196909207541946f Mon Sep 17 00:00:00 2001 From: Kyle Mason Date: Wed, 5 Aug 2026 23:13:56 -0400 Subject: [PATCH] collector/slabinfo: index duplicate slab entries /proc/slabinfo can list the same slab name more than once, for example one cache per device instance. Labelling only by slab name therefore produces duplicate label sets, causing the registry to reject the scrape. Add an index label carrying each entry's position among entries sharing the same slab name. This preserves one series per kernel cache and keeps each entry's own counts and geometry. The label name matches the existing ordinal index convention used by other collectors. The ordinal follows /proc/slabinfo ordering and is not a stable identity. It is assigned after filtering so excluded slabs do not consume an index. Fixes #3506 Signed-off-by: Kyle Mason --- collector/fixtures/e2e-64k-page-output.txt | 40 ++--- collector/fixtures/e2e-output.txt | 40 ++--- collector/slabinfo_linux.go | 46 ++++-- collector/slabinfo_linux_test.go | 169 +++++++++++++++++++++ 4 files changed, 239 insertions(+), 56 deletions(-) create mode 100644 collector/slabinfo_linux_test.go diff --git a/collector/fixtures/e2e-64k-page-output.txt b/collector/fixtures/e2e-64k-page-output.txt index f0ac7ebac4..8bf31ff4c0 100644 --- a/collector/fixtures/e2e-64k-page-output.txt +++ b/collector/fixtures/e2e-64k-page-output.txt @@ -3774,34 +3774,34 @@ node_scrape_collector_success{collector="zfs"} 1 node_scrape_collector_success{collector="zoneinfo"} 1 # HELP node_slabinfo_active_objects The number of objects that are currently active (i.e., in use). # TYPE node_slabinfo_active_objects gauge -node_slabinfo_active_objects{slab="dmaengine-unmap-128"} 1206 -node_slabinfo_active_objects{slab="kmalloc-8192"} 132 -node_slabinfo_active_objects{slab="kmem_cache"} 320 -node_slabinfo_active_objects{slab="tw_sock_TCP"} 704 +node_slabinfo_active_objects{index="0",slab="dmaengine-unmap-128"} 1206 +node_slabinfo_active_objects{index="0",slab="kmalloc-8192"} 132 +node_slabinfo_active_objects{index="0",slab="kmem_cache"} 320 +node_slabinfo_active_objects{index="0",slab="tw_sock_TCP"} 704 # HELP node_slabinfo_object_size_bytes The size of objects in this slab, in bytes. # TYPE node_slabinfo_object_size_bytes gauge -node_slabinfo_object_size_bytes{slab="dmaengine-unmap-128"} 1088 -node_slabinfo_object_size_bytes{slab="kmalloc-8192"} 8192 -node_slabinfo_object_size_bytes{slab="kmem_cache"} 256 -node_slabinfo_object_size_bytes{slab="tw_sock_TCP"} 256 +node_slabinfo_object_size_bytes{index="0",slab="dmaengine-unmap-128"} 1088 +node_slabinfo_object_size_bytes{index="0",slab="kmalloc-8192"} 8192 +node_slabinfo_object_size_bytes{index="0",slab="kmem_cache"} 256 +node_slabinfo_object_size_bytes{index="0",slab="tw_sock_TCP"} 256 # HELP node_slabinfo_objects The total number of allocated objects (i.e., objects that are both in use and not in use). # TYPE node_slabinfo_objects gauge -node_slabinfo_objects{slab="dmaengine-unmap-128"} 1320 -node_slabinfo_objects{slab="kmalloc-8192"} 148 -node_slabinfo_objects{slab="kmem_cache"} 320 -node_slabinfo_objects{slab="tw_sock_TCP"} 864 +node_slabinfo_objects{index="0",slab="dmaengine-unmap-128"} 1320 +node_slabinfo_objects{index="0",slab="kmalloc-8192"} 148 +node_slabinfo_objects{index="0",slab="kmem_cache"} 320 +node_slabinfo_objects{index="0",slab="tw_sock_TCP"} 864 # HELP node_slabinfo_objects_per_slab The number of objects stored in each slab. # TYPE node_slabinfo_objects_per_slab gauge -node_slabinfo_objects_per_slab{slab="dmaengine-unmap-128"} 30 -node_slabinfo_objects_per_slab{slab="kmalloc-8192"} 4 -node_slabinfo_objects_per_slab{slab="kmem_cache"} 32 -node_slabinfo_objects_per_slab{slab="tw_sock_TCP"} 32 +node_slabinfo_objects_per_slab{index="0",slab="dmaengine-unmap-128"} 30 +node_slabinfo_objects_per_slab{index="0",slab="kmalloc-8192"} 4 +node_slabinfo_objects_per_slab{index="0",slab="kmem_cache"} 32 +node_slabinfo_objects_per_slab{index="0",slab="tw_sock_TCP"} 32 # HELP node_slabinfo_pages_per_slab The number of pages allocated for each slab. # TYPE node_slabinfo_pages_per_slab gauge -node_slabinfo_pages_per_slab{slab="dmaengine-unmap-128"} 8 -node_slabinfo_pages_per_slab{slab="kmalloc-8192"} 8 -node_slabinfo_pages_per_slab{slab="kmem_cache"} 2 -node_slabinfo_pages_per_slab{slab="tw_sock_TCP"} 2 +node_slabinfo_pages_per_slab{index="0",slab="dmaengine-unmap-128"} 8 +node_slabinfo_pages_per_slab{index="0",slab="kmalloc-8192"} 8 +node_slabinfo_pages_per_slab{index="0",slab="kmem_cache"} 2 +node_slabinfo_pages_per_slab{index="0",slab="tw_sock_TCP"} 2 # HELP node_sockstat_FRAG6_inuse Number of FRAG6 sockets in state inuse. # TYPE node_sockstat_FRAG6_inuse gauge node_sockstat_FRAG6_inuse 0 diff --git a/collector/fixtures/e2e-output.txt b/collector/fixtures/e2e-output.txt index 9afd903d6f..f803e2b183 100644 --- a/collector/fixtures/e2e-output.txt +++ b/collector/fixtures/e2e-output.txt @@ -3806,34 +3806,34 @@ node_scrape_collector_success{collector="zfs"} 1 node_scrape_collector_success{collector="zoneinfo"} 1 # HELP node_slabinfo_active_objects The number of objects that are currently active (i.e., in use). # TYPE node_slabinfo_active_objects gauge -node_slabinfo_active_objects{slab="dmaengine-unmap-128"} 1206 -node_slabinfo_active_objects{slab="kmalloc-8192"} 132 -node_slabinfo_active_objects{slab="kmem_cache"} 320 -node_slabinfo_active_objects{slab="tw_sock_TCP"} 704 +node_slabinfo_active_objects{index="0",slab="dmaengine-unmap-128"} 1206 +node_slabinfo_active_objects{index="0",slab="kmalloc-8192"} 132 +node_slabinfo_active_objects{index="0",slab="kmem_cache"} 320 +node_slabinfo_active_objects{index="0",slab="tw_sock_TCP"} 704 # HELP node_slabinfo_object_size_bytes The size of objects in this slab, in bytes. # TYPE node_slabinfo_object_size_bytes gauge -node_slabinfo_object_size_bytes{slab="dmaengine-unmap-128"} 1088 -node_slabinfo_object_size_bytes{slab="kmalloc-8192"} 8192 -node_slabinfo_object_size_bytes{slab="kmem_cache"} 256 -node_slabinfo_object_size_bytes{slab="tw_sock_TCP"} 256 +node_slabinfo_object_size_bytes{index="0",slab="dmaengine-unmap-128"} 1088 +node_slabinfo_object_size_bytes{index="0",slab="kmalloc-8192"} 8192 +node_slabinfo_object_size_bytes{index="0",slab="kmem_cache"} 256 +node_slabinfo_object_size_bytes{index="0",slab="tw_sock_TCP"} 256 # HELP node_slabinfo_objects The total number of allocated objects (i.e., objects that are both in use and not in use). # TYPE node_slabinfo_objects gauge -node_slabinfo_objects{slab="dmaengine-unmap-128"} 1320 -node_slabinfo_objects{slab="kmalloc-8192"} 148 -node_slabinfo_objects{slab="kmem_cache"} 320 -node_slabinfo_objects{slab="tw_sock_TCP"} 864 +node_slabinfo_objects{index="0",slab="dmaengine-unmap-128"} 1320 +node_slabinfo_objects{index="0",slab="kmalloc-8192"} 148 +node_slabinfo_objects{index="0",slab="kmem_cache"} 320 +node_slabinfo_objects{index="0",slab="tw_sock_TCP"} 864 # HELP node_slabinfo_objects_per_slab The number of objects stored in each slab. # TYPE node_slabinfo_objects_per_slab gauge -node_slabinfo_objects_per_slab{slab="dmaengine-unmap-128"} 30 -node_slabinfo_objects_per_slab{slab="kmalloc-8192"} 4 -node_slabinfo_objects_per_slab{slab="kmem_cache"} 32 -node_slabinfo_objects_per_slab{slab="tw_sock_TCP"} 32 +node_slabinfo_objects_per_slab{index="0",slab="dmaengine-unmap-128"} 30 +node_slabinfo_objects_per_slab{index="0",slab="kmalloc-8192"} 4 +node_slabinfo_objects_per_slab{index="0",slab="kmem_cache"} 32 +node_slabinfo_objects_per_slab{index="0",slab="tw_sock_TCP"} 32 # HELP node_slabinfo_pages_per_slab The number of pages allocated for each slab. # TYPE node_slabinfo_pages_per_slab gauge -node_slabinfo_pages_per_slab{slab="dmaengine-unmap-128"} 8 -node_slabinfo_pages_per_slab{slab="kmalloc-8192"} 8 -node_slabinfo_pages_per_slab{slab="kmem_cache"} 2 -node_slabinfo_pages_per_slab{slab="tw_sock_TCP"} 2 +node_slabinfo_pages_per_slab{index="0",slab="dmaengine-unmap-128"} 8 +node_slabinfo_pages_per_slab{index="0",slab="kmalloc-8192"} 8 +node_slabinfo_pages_per_slab{index="0",slab="kmem_cache"} 2 +node_slabinfo_pages_per_slab{index="0",slab="tw_sock_TCP"} 2 # HELP node_sockstat_FRAG6_inuse Number of FRAG6 sockets in state inuse. # TYPE node_sockstat_FRAG6_inuse gauge node_sockstat_FRAG6_inuse 0 diff --git a/collector/slabinfo_linux.go b/collector/slabinfo_linux.go index 48bbf2abfe..66867b8e76 100644 --- a/collector/slabinfo_linux.go +++ b/collector/slabinfo_linux.go @@ -18,6 +18,7 @@ package collector import ( "fmt" "log/slog" + "strconv" "github.com/alecthomas/kingpin/v2" "github.com/prometheus/client_golang/prometheus" @@ -50,7 +51,7 @@ func NewSlabinfoCollector(logger *slog.Logger) (Collector, error) { return &slabinfoCollector{logger: logger, fs: fs, subsystem: "slabinfo", - labels: []string{"slab"}, + labels: []string{"slab", "index"}, slabNameFilter: newDeviceFilter(*slabNameExclude, *slabNameInclude), }, nil } @@ -61,71 +62,84 @@ func (c *slabinfoCollector) Update(ch chan<- prometheus.Metric) error { return fmt.Errorf("couldn't get %s: %w", c.subsystem, err) } + // /proc/slabinfo can list the same slab name more than once, for example one + // cache per device instance. The kernel permits this: the duplicate-name check + // in kmem_cache_sanity_check() only WARNs, and is compiled out unless + // CONFIG_DEBUG_VM is set. Labelling by name alone therefore produces duplicate + // label sets, which fails the entire scrape. + // + // Each entry is given its position among the entries sharing its name, so every + // cache keeps its own series and its own geometry. The ordinal reflects the + // order of /proc/slabinfo and is not a stable identity: if a cache is created + // or destroyed, subsequent entries shift. + seen := make(map[string]int, len(slabinfo.Slabs)) for _, slab := range slabinfo.Slabs { if c.slabNameFilter.ignored(slab.Name) { continue } - ch <- c.activeObjects(slab.Name, slab.ObjActive) - ch <- c.objects(slab.Name, slab.ObjNum) - ch <- c.objectSizeBytes(slab.Name, slab.ObjSize) - ch <- c.objectsPerSlab(slab.Name, slab.ObjPerSlab) - ch <- c.pagesPerSlab(slab.Name, slab.PagesPerSlab) + index := strconv.Itoa(seen[slab.Name]) + seen[slab.Name]++ + ch <- c.activeObjects(slab.Name, index, slab.ObjActive) + ch <- c.objects(slab.Name, index, slab.ObjNum) + ch <- c.objectSizeBytes(slab.Name, index, slab.ObjSize) + ch <- c.objectsPerSlab(slab.Name, index, slab.ObjPerSlab) + ch <- c.pagesPerSlab(slab.Name, index, slab.PagesPerSlab) } return nil } -func (c *slabinfoCollector) activeObjects(label string, val int64) prometheus.Metric { +func (c *slabinfoCollector) activeObjects(label, index string, val int64) prometheus.Metric { desc := prometheus.NewDesc( prometheus.BuildFQName(namespace, c.subsystem, "active_objects"), "The number of objects that are currently active (i.e., in use).", c.labels, nil) return prometheus.MustNewConstMetric( - desc, prometheus.GaugeValue, float64(val), label, + desc, prometheus.GaugeValue, float64(val), label, index, ) } -func (c *slabinfoCollector) objects(label string, val int64) prometheus.Metric { +func (c *slabinfoCollector) objects(label, index string, val int64) prometheus.Metric { desc := prometheus.NewDesc( prometheus.BuildFQName(namespace, c.subsystem, "objects"), "The total number of allocated objects (i.e., objects that are both in use and not in use).", c.labels, nil) return prometheus.MustNewConstMetric( - desc, prometheus.GaugeValue, float64(val), label, + desc, prometheus.GaugeValue, float64(val), label, index, ) } -func (c *slabinfoCollector) objectSizeBytes(label string, val int64) prometheus.Metric { +func (c *slabinfoCollector) objectSizeBytes(label, index string, val int64) prometheus.Metric { desc := prometheus.NewDesc( prometheus.BuildFQName(namespace, c.subsystem, "object_size_bytes"), "The size of objects in this slab, in bytes.", c.labels, nil) return prometheus.MustNewConstMetric( - desc, prometheus.GaugeValue, float64(val), label, + desc, prometheus.GaugeValue, float64(val), label, index, ) } -func (c *slabinfoCollector) objectsPerSlab(label string, val int64) prometheus.Metric { +func (c *slabinfoCollector) objectsPerSlab(label, index string, val int64) prometheus.Metric { desc := prometheus.NewDesc( prometheus.BuildFQName(namespace, c.subsystem, "objects_per_slab"), "The number of objects stored in each slab.", c.labels, nil) return prometheus.MustNewConstMetric( - desc, prometheus.GaugeValue, float64(val), label, + desc, prometheus.GaugeValue, float64(val), label, index, ) } -func (c *slabinfoCollector) pagesPerSlab(label string, val int64) prometheus.Metric { +func (c *slabinfoCollector) pagesPerSlab(label, index string, val int64) prometheus.Metric { desc := prometheus.NewDesc( prometheus.BuildFQName(namespace, c.subsystem, "pages_per_slab"), "The number of pages allocated for each slab.", c.labels, nil) return prometheus.MustNewConstMetric( - desc, prometheus.GaugeValue, float64(val), label, + desc, prometheus.GaugeValue, float64(val), label, index, ) } diff --git a/collector/slabinfo_linux_test.go b/collector/slabinfo_linux_test.go new file mode 100644 index 0000000000..3b167f2830 --- /dev/null +++ b/collector/slabinfo_linux_test.go @@ -0,0 +1,169 @@ +// Copyright The Prometheus Authors +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//go:build !noslabinfo + +package collector + +import ( + "log/slog" + "os" + "path/filepath" + "testing" + + "github.com/prometheus/client_golang/prometheus" +) + +const slabinfoHeader = `slabinfo - version: 2.1 +# name : tunables : slabdata +` + +// writeSlabinfo writes body into a temporary procfs and points the collector at +// it for the duration of the test. procPath is a package-global flag value, so +// it is restored afterwards to keep these tests independent of execution order. +// These tests must not call t.Parallel() for the same reason. +func writeSlabinfo(t *testing.T, body string) { + t.Helper() + dir := t.TempDir() + if err := os.WriteFile(filepath.Join(dir, "slabinfo"), []byte(slabinfoHeader+body), 0o644); err != nil { + t.Fatal(err) + } + orig := *procPath + t.Cleanup(func() { *procPath = orig }) + *procPath = dir +} + +// gatherSlabinfo collects through a registry, which is what surfaces duplicate +// label sets, and keys the result by "metric/slab/index". +func gatherSlabinfo(t *testing.T) map[string]float64 { + t.Helper() + c, err := NewSlabinfoCollector(slog.New(slog.NewTextHandler(os.Stderr, nil))) + if err != nil { + t.Fatal(err) + } + reg := prometheus.NewPedanticRegistry() + if err := reg.Register(prometheus.CollectorFunc(func(ch chan<- prometheus.Metric) { + if err := c.Update(ch); err != nil { + t.Errorf("Update failed: %v", err) + } + })); err != nil { + t.Fatal(err) + } + mfs, err := reg.Gather() + if err != nil { + t.Fatalf("Gather failed: %v", err) + } + got := map[string]float64{} + for _, mf := range mfs { + for _, m := range mf.Metric { + var name, index string + for _, l := range m.Label { + switch l.GetName() { + case "slab": + name = l.GetValue() + case "index": + index = l.GetValue() + } + } + got[mf.GetName()+"/"+name+"/"+index] = m.GetGauge().GetValue() + } + } + return got +} + +func checkSlabinfo(t *testing.T, got map[string]float64, want map[string]float64) { + t.Helper() + for key, w := range want { + if g, ok := got[key]; !ok { + t.Errorf("%s: missing", key) + } else if g != w { + t.Errorf("%s: got %v, want %v", key, g, w) + } + } +} + +// A slab name may appear more than once, for example one cache per device +// instance. Each entry keeps its own series, distinguished by the index label. +func TestSlabinfoDuplicateSlabNames(t *testing.T) { + writeSlabinfo(t, `kmem_cache 320 320 256 32 2 : tunables 0 0 0 : slabdata 10 10 0 +mlx5_fs_ftes 44 44 736 44 8 : tunables 0 0 0 : slabdata 1 1 0 +mlx5_fs_fgs 42 42 776 42 8 : tunables 0 0 0 : slabdata 1 1 0 +mlx5_fs_ftes 396 396 736 44 8 : tunables 0 0 0 : slabdata 9 9 0 +mlx5_fs_fgs 84 84 776 42 8 : tunables 0 0 0 : slabdata 2 2 0 +`) + checkSlabinfo(t, gatherSlabinfo(t), map[string]float64{ + "node_slabinfo_active_objects/mlx5_fs_ftes/0": 44, + "node_slabinfo_active_objects/mlx5_fs_ftes/1": 396, + "node_slabinfo_objects/mlx5_fs_ftes/1": 396, + "node_slabinfo_active_objects/mlx5_fs_fgs/0": 42, + "node_slabinfo_active_objects/mlx5_fs_fgs/1": 84, + "node_slabinfo_object_size_bytes/mlx5_fs_ftes/1": 736, + "node_slabinfo_objects_per_slab/mlx5_fs_ftes/1": 44, + "node_slabinfo_pages_per_slab/mlx5_fs_ftes/1": 8, + "node_slabinfo_active_objects/kmem_cache/0": 320, + }) +} + +// Slabs with distinct names each get index 0. +func TestSlabinfoDistinctSlabNames(t *testing.T) { + writeSlabinfo(t, `tw_sock_TCP 704 864 256 32 2 : tunables 0 0 0 : slabdata 27 27 0 +kmem_cache 320 320 256 32 2 : tunables 0 0 0 : slabdata 10 10 0 +`) + checkSlabinfo(t, gatherSlabinfo(t), map[string]float64{ + "node_slabinfo_active_objects/tw_sock_TCP/0": 704, + "node_slabinfo_objects/tw_sock_TCP/0": 864, + "node_slabinfo_active_objects/kmem_cache/0": 320, + "node_slabinfo_objects/kmem_cache/0": 320, + }) +} + +// The kernel does not guarantee that same-named caches share geometry: the +// duplicate-name check in kmem_cache_sanity_check() only WARNs and is compiled +// out without CONFIG_DEBUG_VM, and object_size is per-cache. Indexing keeps each +// cache's own geometry, so no representative value has to be chosen. +func TestSlabinfoDuplicateSlabNamesDifferentGeometry(t *testing.T) { + writeSlabinfo(t, `weird_cache 10 10 128 32 2 : tunables 0 0 0 : slabdata 1 1 0 +weird_cache 20 20 256 16 4 : tunables 0 0 0 : slabdata 2 2 0 +`) + checkSlabinfo(t, gatherSlabinfo(t), map[string]float64{ + "node_slabinfo_active_objects/weird_cache/0": 10, + "node_slabinfo_object_size_bytes/weird_cache/0": 128, + "node_slabinfo_objects_per_slab/weird_cache/0": 32, + "node_slabinfo_pages_per_slab/weird_cache/0": 2, + "node_slabinfo_active_objects/weird_cache/1": 20, + "node_slabinfo_object_size_bytes/weird_cache/1": 256, + "node_slabinfo_objects_per_slab/weird_cache/1": 16, + "node_slabinfo_pages_per_slab/weird_cache/1": 4, + }) +} + +// Excluded slabs must not consume an index, so the ordinals of the remaining +// entries stay contiguous. +func TestSlabinfoIndexIgnoresFilteredSlabs(t *testing.T) { + orig := *slabNameExclude + t.Cleanup(func() { *slabNameExclude = orig }) + *slabNameExclude = "^drop_me$" + + writeSlabinfo(t, `dup_cache 10 10 128 32 2 : tunables 0 0 0 : slabdata 1 1 0 +drop_me 99 99 999 99 9 : tunables 0 0 0 : slabdata 9 9 0 +dup_cache 20 20 128 32 2 : tunables 0 0 0 : slabdata 2 2 0 +`) + got := gatherSlabinfo(t) + checkSlabinfo(t, got, map[string]float64{ + "node_slabinfo_active_objects/dup_cache/0": 10, + "node_slabinfo_active_objects/dup_cache/1": 20, + }) + if _, ok := got["node_slabinfo_active_objects/drop_me/0"]; ok { + t.Error("excluded slab was collected") + } +}