diff --git a/collector/fixtures/e2e-64k-page-output.txt b/collector/fixtures/e2e-64k-page-output.txt index f0ac7ebac4..8bf31ff4c0 100644 --- a/collector/fixtures/e2e-64k-page-output.txt +++ b/collector/fixtures/e2e-64k-page-output.txt @@ -3774,34 +3774,34 @@ node_scrape_collector_success{collector="zfs"} 1 node_scrape_collector_success{collector="zoneinfo"} 1 # HELP node_slabinfo_active_objects The number of objects that are currently active (i.e., in use). # TYPE node_slabinfo_active_objects gauge -node_slabinfo_active_objects{slab="dmaengine-unmap-128"} 1206 -node_slabinfo_active_objects{slab="kmalloc-8192"} 132 -node_slabinfo_active_objects{slab="kmem_cache"} 320 -node_slabinfo_active_objects{slab="tw_sock_TCP"} 704 +node_slabinfo_active_objects{index="0",slab="dmaengine-unmap-128"} 1206 +node_slabinfo_active_objects{index="0",slab="kmalloc-8192"} 132 +node_slabinfo_active_objects{index="0",slab="kmem_cache"} 320 +node_slabinfo_active_objects{index="0",slab="tw_sock_TCP"} 704 # HELP node_slabinfo_object_size_bytes The size of objects in this slab, in bytes. # TYPE node_slabinfo_object_size_bytes gauge -node_slabinfo_object_size_bytes{slab="dmaengine-unmap-128"} 1088 -node_slabinfo_object_size_bytes{slab="kmalloc-8192"} 8192 -node_slabinfo_object_size_bytes{slab="kmem_cache"} 256 -node_slabinfo_object_size_bytes{slab="tw_sock_TCP"} 256 +node_slabinfo_object_size_bytes{index="0",slab="dmaengine-unmap-128"} 1088 +node_slabinfo_object_size_bytes{index="0",slab="kmalloc-8192"} 8192 +node_slabinfo_object_size_bytes{index="0",slab="kmem_cache"} 256 +node_slabinfo_object_size_bytes{index="0",slab="tw_sock_TCP"} 256 # HELP node_slabinfo_objects The total number of allocated objects (i.e., objects that are both in use and not in use). # TYPE node_slabinfo_objects gauge -node_slabinfo_objects{slab="dmaengine-unmap-128"} 1320 -node_slabinfo_objects{slab="kmalloc-8192"} 148 -node_slabinfo_objects{slab="kmem_cache"} 320 -node_slabinfo_objects{slab="tw_sock_TCP"} 864 +node_slabinfo_objects{index="0",slab="dmaengine-unmap-128"} 1320 +node_slabinfo_objects{index="0",slab="kmalloc-8192"} 148 +node_slabinfo_objects{index="0",slab="kmem_cache"} 320 +node_slabinfo_objects{index="0",slab="tw_sock_TCP"} 864 # HELP node_slabinfo_objects_per_slab The number of objects stored in each slab. # TYPE node_slabinfo_objects_per_slab gauge -node_slabinfo_objects_per_slab{slab="dmaengine-unmap-128"} 30 -node_slabinfo_objects_per_slab{slab="kmalloc-8192"} 4 -node_slabinfo_objects_per_slab{slab="kmem_cache"} 32 -node_slabinfo_objects_per_slab{slab="tw_sock_TCP"} 32 +node_slabinfo_objects_per_slab{index="0",slab="dmaengine-unmap-128"} 30 +node_slabinfo_objects_per_slab{index="0",slab="kmalloc-8192"} 4 +node_slabinfo_objects_per_slab{index="0",slab="kmem_cache"} 32 +node_slabinfo_objects_per_slab{index="0",slab="tw_sock_TCP"} 32 # HELP node_slabinfo_pages_per_slab The number of pages allocated for each slab. # TYPE node_slabinfo_pages_per_slab gauge -node_slabinfo_pages_per_slab{slab="dmaengine-unmap-128"} 8 -node_slabinfo_pages_per_slab{slab="kmalloc-8192"} 8 -node_slabinfo_pages_per_slab{slab="kmem_cache"} 2 -node_slabinfo_pages_per_slab{slab="tw_sock_TCP"} 2 +node_slabinfo_pages_per_slab{index="0",slab="dmaengine-unmap-128"} 8 +node_slabinfo_pages_per_slab{index="0",slab="kmalloc-8192"} 8 +node_slabinfo_pages_per_slab{index="0",slab="kmem_cache"} 2 +node_slabinfo_pages_per_slab{index="0",slab="tw_sock_TCP"} 2 # HELP node_sockstat_FRAG6_inuse Number of FRAG6 sockets in state inuse. # TYPE node_sockstat_FRAG6_inuse gauge node_sockstat_FRAG6_inuse 0 diff --git a/collector/fixtures/e2e-output.txt b/collector/fixtures/e2e-output.txt index 9afd903d6f..f803e2b183 100644 --- a/collector/fixtures/e2e-output.txt +++ b/collector/fixtures/e2e-output.txt @@ -3806,34 +3806,34 @@ node_scrape_collector_success{collector="zfs"} 1 node_scrape_collector_success{collector="zoneinfo"} 1 # HELP node_slabinfo_active_objects The number of objects that are currently active (i.e., in use). # TYPE node_slabinfo_active_objects gauge -node_slabinfo_active_objects{slab="dmaengine-unmap-128"} 1206 -node_slabinfo_active_objects{slab="kmalloc-8192"} 132 -node_slabinfo_active_objects{slab="kmem_cache"} 320 -node_slabinfo_active_objects{slab="tw_sock_TCP"} 704 +node_slabinfo_active_objects{index="0",slab="dmaengine-unmap-128"} 1206 +node_slabinfo_active_objects{index="0",slab="kmalloc-8192"} 132 +node_slabinfo_active_objects{index="0",slab="kmem_cache"} 320 +node_slabinfo_active_objects{index="0",slab="tw_sock_TCP"} 704 # HELP node_slabinfo_object_size_bytes The size of objects in this slab, in bytes. # TYPE node_slabinfo_object_size_bytes gauge -node_slabinfo_object_size_bytes{slab="dmaengine-unmap-128"} 1088 -node_slabinfo_object_size_bytes{slab="kmalloc-8192"} 8192 -node_slabinfo_object_size_bytes{slab="kmem_cache"} 256 -node_slabinfo_object_size_bytes{slab="tw_sock_TCP"} 256 +node_slabinfo_object_size_bytes{index="0",slab="dmaengine-unmap-128"} 1088 +node_slabinfo_object_size_bytes{index="0",slab="kmalloc-8192"} 8192 +node_slabinfo_object_size_bytes{index="0",slab="kmem_cache"} 256 +node_slabinfo_object_size_bytes{index="0",slab="tw_sock_TCP"} 256 # HELP node_slabinfo_objects The total number of allocated objects (i.e., objects that are both in use and not in use). # TYPE node_slabinfo_objects gauge -node_slabinfo_objects{slab="dmaengine-unmap-128"} 1320 -node_slabinfo_objects{slab="kmalloc-8192"} 148 -node_slabinfo_objects{slab="kmem_cache"} 320 -node_slabinfo_objects{slab="tw_sock_TCP"} 864 +node_slabinfo_objects{index="0",slab="dmaengine-unmap-128"} 1320 +node_slabinfo_objects{index="0",slab="kmalloc-8192"} 148 +node_slabinfo_objects{index="0",slab="kmem_cache"} 320 +node_slabinfo_objects{index="0",slab="tw_sock_TCP"} 864 # HELP node_slabinfo_objects_per_slab The number of objects stored in each slab. # TYPE node_slabinfo_objects_per_slab gauge -node_slabinfo_objects_per_slab{slab="dmaengine-unmap-128"} 30 -node_slabinfo_objects_per_slab{slab="kmalloc-8192"} 4 -node_slabinfo_objects_per_slab{slab="kmem_cache"} 32 -node_slabinfo_objects_per_slab{slab="tw_sock_TCP"} 32 +node_slabinfo_objects_per_slab{index="0",slab="dmaengine-unmap-128"} 30 +node_slabinfo_objects_per_slab{index="0",slab="kmalloc-8192"} 4 +node_slabinfo_objects_per_slab{index="0",slab="kmem_cache"} 32 +node_slabinfo_objects_per_slab{index="0",slab="tw_sock_TCP"} 32 # HELP node_slabinfo_pages_per_slab The number of pages allocated for each slab. # TYPE node_slabinfo_pages_per_slab gauge -node_slabinfo_pages_per_slab{slab="dmaengine-unmap-128"} 8 -node_slabinfo_pages_per_slab{slab="kmalloc-8192"} 8 -node_slabinfo_pages_per_slab{slab="kmem_cache"} 2 -node_slabinfo_pages_per_slab{slab="tw_sock_TCP"} 2 +node_slabinfo_pages_per_slab{index="0",slab="dmaengine-unmap-128"} 8 +node_slabinfo_pages_per_slab{index="0",slab="kmalloc-8192"} 8 +node_slabinfo_pages_per_slab{index="0",slab="kmem_cache"} 2 +node_slabinfo_pages_per_slab{index="0",slab="tw_sock_TCP"} 2 # HELP node_sockstat_FRAG6_inuse Number of FRAG6 sockets in state inuse. # TYPE node_sockstat_FRAG6_inuse gauge node_sockstat_FRAG6_inuse 0 diff --git a/collector/slabinfo_linux.go b/collector/slabinfo_linux.go index 48bbf2abfe..66867b8e76 100644 --- a/collector/slabinfo_linux.go +++ b/collector/slabinfo_linux.go @@ -18,6 +18,7 @@ package collector import ( "fmt" "log/slog" + "strconv" "github.com/alecthomas/kingpin/v2" "github.com/prometheus/client_golang/prometheus" @@ -50,7 +51,7 @@ func NewSlabinfoCollector(logger *slog.Logger) (Collector, error) { return &slabinfoCollector{logger: logger, fs: fs, subsystem: "slabinfo", - labels: []string{"slab"}, + labels: []string{"slab", "index"}, slabNameFilter: newDeviceFilter(*slabNameExclude, *slabNameInclude), }, nil } @@ -61,71 +62,84 @@ func (c *slabinfoCollector) Update(ch chan<- prometheus.Metric) error { return fmt.Errorf("couldn't get %s: %w", c.subsystem, err) } + // /proc/slabinfo can list the same slab name more than once, for example one + // cache per device instance. The kernel permits this: the duplicate-name check + // in kmem_cache_sanity_check() only WARNs, and is compiled out unless + // CONFIG_DEBUG_VM is set. Labelling by name alone therefore produces duplicate + // label sets, which fails the entire scrape. + // + // Each entry is given its position among the entries sharing its name, so every + // cache keeps its own series and its own geometry. The ordinal reflects the + // order of /proc/slabinfo and is not a stable identity: if a cache is created + // or destroyed, subsequent entries shift. + seen := make(map[string]int, len(slabinfo.Slabs)) for _, slab := range slabinfo.Slabs { if c.slabNameFilter.ignored(slab.Name) { continue } - ch <- c.activeObjects(slab.Name, slab.ObjActive) - ch <- c.objects(slab.Name, slab.ObjNum) - ch <- c.objectSizeBytes(slab.Name, slab.ObjSize) - ch <- c.objectsPerSlab(slab.Name, slab.ObjPerSlab) - ch <- c.pagesPerSlab(slab.Name, slab.PagesPerSlab) + index := strconv.Itoa(seen[slab.Name]) + seen[slab.Name]++ + ch <- c.activeObjects(slab.Name, index, slab.ObjActive) + ch <- c.objects(slab.Name, index, slab.ObjNum) + ch <- c.objectSizeBytes(slab.Name, index, slab.ObjSize) + ch <- c.objectsPerSlab(slab.Name, index, slab.ObjPerSlab) + ch <- c.pagesPerSlab(slab.Name, index, slab.PagesPerSlab) } return nil } -func (c *slabinfoCollector) activeObjects(label string, val int64) prometheus.Metric { +func (c *slabinfoCollector) activeObjects(label, index string, val int64) prometheus.Metric { desc := prometheus.NewDesc( prometheus.BuildFQName(namespace, c.subsystem, "active_objects"), "The number of objects that are currently active (i.e., in use).", c.labels, nil) return prometheus.MustNewConstMetric( - desc, prometheus.GaugeValue, float64(val), label, + desc, prometheus.GaugeValue, float64(val), label, index, ) } -func (c *slabinfoCollector) objects(label string, val int64) prometheus.Metric { +func (c *slabinfoCollector) objects(label, index string, val int64) prometheus.Metric { desc := prometheus.NewDesc( prometheus.BuildFQName(namespace, c.subsystem, "objects"), "The total number of allocated objects (i.e., objects that are both in use and not in use).", c.labels, nil) return prometheus.MustNewConstMetric( - desc, prometheus.GaugeValue, float64(val), label, + desc, prometheus.GaugeValue, float64(val), label, index, ) } -func (c *slabinfoCollector) objectSizeBytes(label string, val int64) prometheus.Metric { +func (c *slabinfoCollector) objectSizeBytes(label, index string, val int64) prometheus.Metric { desc := prometheus.NewDesc( prometheus.BuildFQName(namespace, c.subsystem, "object_size_bytes"), "The size of objects in this slab, in bytes.", c.labels, nil) return prometheus.MustNewConstMetric( - desc, prometheus.GaugeValue, float64(val), label, + desc, prometheus.GaugeValue, float64(val), label, index, ) } -func (c *slabinfoCollector) objectsPerSlab(label string, val int64) prometheus.Metric { +func (c *slabinfoCollector) objectsPerSlab(label, index string, val int64) prometheus.Metric { desc := prometheus.NewDesc( prometheus.BuildFQName(namespace, c.subsystem, "objects_per_slab"), "The number of objects stored in each slab.", c.labels, nil) return prometheus.MustNewConstMetric( - desc, prometheus.GaugeValue, float64(val), label, + desc, prometheus.GaugeValue, float64(val), label, index, ) } -func (c *slabinfoCollector) pagesPerSlab(label string, val int64) prometheus.Metric { +func (c *slabinfoCollector) pagesPerSlab(label, index string, val int64) prometheus.Metric { desc := prometheus.NewDesc( prometheus.BuildFQName(namespace, c.subsystem, "pages_per_slab"), "The number of pages allocated for each slab.", c.labels, nil) return prometheus.MustNewConstMetric( - desc, prometheus.GaugeValue, float64(val), label, + desc, prometheus.GaugeValue, float64(val), label, index, ) } diff --git a/collector/slabinfo_linux_test.go b/collector/slabinfo_linux_test.go new file mode 100644 index 0000000000..3b167f2830 --- /dev/null +++ b/collector/slabinfo_linux_test.go @@ -0,0 +1,169 @@ +// Copyright The Prometheus Authors +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//go:build !noslabinfo + +package collector + +import ( + "log/slog" + "os" + "path/filepath" + "testing" + + "github.com/prometheus/client_golang/prometheus" +) + +const slabinfoHeader = `slabinfo - version: 2.1 +# name : tunables : slabdata +` + +// writeSlabinfo writes body into a temporary procfs and points the collector at +// it for the duration of the test. procPath is a package-global flag value, so +// it is restored afterwards to keep these tests independent of execution order. +// These tests must not call t.Parallel() for the same reason. +func writeSlabinfo(t *testing.T, body string) { + t.Helper() + dir := t.TempDir() + if err := os.WriteFile(filepath.Join(dir, "slabinfo"), []byte(slabinfoHeader+body), 0o644); err != nil { + t.Fatal(err) + } + orig := *procPath + t.Cleanup(func() { *procPath = orig }) + *procPath = dir +} + +// gatherSlabinfo collects through a registry, which is what surfaces duplicate +// label sets, and keys the result by "metric/slab/index". +func gatherSlabinfo(t *testing.T) map[string]float64 { + t.Helper() + c, err := NewSlabinfoCollector(slog.New(slog.NewTextHandler(os.Stderr, nil))) + if err != nil { + t.Fatal(err) + } + reg := prometheus.NewPedanticRegistry() + if err := reg.Register(prometheus.CollectorFunc(func(ch chan<- prometheus.Metric) { + if err := c.Update(ch); err != nil { + t.Errorf("Update failed: %v", err) + } + })); err != nil { + t.Fatal(err) + } + mfs, err := reg.Gather() + if err != nil { + t.Fatalf("Gather failed: %v", err) + } + got := map[string]float64{} + for _, mf := range mfs { + for _, m := range mf.Metric { + var name, index string + for _, l := range m.Label { + switch l.GetName() { + case "slab": + name = l.GetValue() + case "index": + index = l.GetValue() + } + } + got[mf.GetName()+"/"+name+"/"+index] = m.GetGauge().GetValue() + } + } + return got +} + +func checkSlabinfo(t *testing.T, got map[string]float64, want map[string]float64) { + t.Helper() + for key, w := range want { + if g, ok := got[key]; !ok { + t.Errorf("%s: missing", key) + } else if g != w { + t.Errorf("%s: got %v, want %v", key, g, w) + } + } +} + +// A slab name may appear more than once, for example one cache per device +// instance. Each entry keeps its own series, distinguished by the index label. +func TestSlabinfoDuplicateSlabNames(t *testing.T) { + writeSlabinfo(t, `kmem_cache 320 320 256 32 2 : tunables 0 0 0 : slabdata 10 10 0 +mlx5_fs_ftes 44 44 736 44 8 : tunables 0 0 0 : slabdata 1 1 0 +mlx5_fs_fgs 42 42 776 42 8 : tunables 0 0 0 : slabdata 1 1 0 +mlx5_fs_ftes 396 396 736 44 8 : tunables 0 0 0 : slabdata 9 9 0 +mlx5_fs_fgs 84 84 776 42 8 : tunables 0 0 0 : slabdata 2 2 0 +`) + checkSlabinfo(t, gatherSlabinfo(t), map[string]float64{ + "node_slabinfo_active_objects/mlx5_fs_ftes/0": 44, + "node_slabinfo_active_objects/mlx5_fs_ftes/1": 396, + "node_slabinfo_objects/mlx5_fs_ftes/1": 396, + "node_slabinfo_active_objects/mlx5_fs_fgs/0": 42, + "node_slabinfo_active_objects/mlx5_fs_fgs/1": 84, + "node_slabinfo_object_size_bytes/mlx5_fs_ftes/1": 736, + "node_slabinfo_objects_per_slab/mlx5_fs_ftes/1": 44, + "node_slabinfo_pages_per_slab/mlx5_fs_ftes/1": 8, + "node_slabinfo_active_objects/kmem_cache/0": 320, + }) +} + +// Slabs with distinct names each get index 0. +func TestSlabinfoDistinctSlabNames(t *testing.T) { + writeSlabinfo(t, `tw_sock_TCP 704 864 256 32 2 : tunables 0 0 0 : slabdata 27 27 0 +kmem_cache 320 320 256 32 2 : tunables 0 0 0 : slabdata 10 10 0 +`) + checkSlabinfo(t, gatherSlabinfo(t), map[string]float64{ + "node_slabinfo_active_objects/tw_sock_TCP/0": 704, + "node_slabinfo_objects/tw_sock_TCP/0": 864, + "node_slabinfo_active_objects/kmem_cache/0": 320, + "node_slabinfo_objects/kmem_cache/0": 320, + }) +} + +// The kernel does not guarantee that same-named caches share geometry: the +// duplicate-name check in kmem_cache_sanity_check() only WARNs and is compiled +// out without CONFIG_DEBUG_VM, and object_size is per-cache. Indexing keeps each +// cache's own geometry, so no representative value has to be chosen. +func TestSlabinfoDuplicateSlabNamesDifferentGeometry(t *testing.T) { + writeSlabinfo(t, `weird_cache 10 10 128 32 2 : tunables 0 0 0 : slabdata 1 1 0 +weird_cache 20 20 256 16 4 : tunables 0 0 0 : slabdata 2 2 0 +`) + checkSlabinfo(t, gatherSlabinfo(t), map[string]float64{ + "node_slabinfo_active_objects/weird_cache/0": 10, + "node_slabinfo_object_size_bytes/weird_cache/0": 128, + "node_slabinfo_objects_per_slab/weird_cache/0": 32, + "node_slabinfo_pages_per_slab/weird_cache/0": 2, + "node_slabinfo_active_objects/weird_cache/1": 20, + "node_slabinfo_object_size_bytes/weird_cache/1": 256, + "node_slabinfo_objects_per_slab/weird_cache/1": 16, + "node_slabinfo_pages_per_slab/weird_cache/1": 4, + }) +} + +// Excluded slabs must not consume an index, so the ordinals of the remaining +// entries stay contiguous. +func TestSlabinfoIndexIgnoresFilteredSlabs(t *testing.T) { + orig := *slabNameExclude + t.Cleanup(func() { *slabNameExclude = orig }) + *slabNameExclude = "^drop_me$" + + writeSlabinfo(t, `dup_cache 10 10 128 32 2 : tunables 0 0 0 : slabdata 1 1 0 +drop_me 99 99 999 99 9 : tunables 0 0 0 : slabdata 9 9 0 +dup_cache 20 20 128 32 2 : tunables 0 0 0 : slabdata 2 2 0 +`) + got := gatherSlabinfo(t) + checkSlabinfo(t, got, map[string]float64{ + "node_slabinfo_active_objects/dup_cache/0": 10, + "node_slabinfo_active_objects/dup_cache/1": 20, + }) + if _, ok := got["node_slabinfo_active_objects/drop_me/0"]; ok { + t.Error("excluded slab was collected") + } +}