Skip to content

collector/slabinfo: index duplicate slab entries #3784

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
machismo0311 wants to merge 1 commit into
base: master
Choose a base branch
from
+239 −56
Open

collector/slabinfo: index duplicate slab entries #3784

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
40 changes: 20 additions & 20 deletions collector/fixtures/e2e-64k-page-output.txt
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -3774,34 +3774,34 @@ node_scrape_collector_success{collector="zfs"} 1
node_scrape_collector_success{collector="zoneinfo"} 1
# HELP node_slabinfo_active_objects The number of objects that are currently active (i.e., in use).
# TYPE node_slabinfo_active_objects gauge
node_slabinfo_active_objects{slab="dmaengine-unmap-128"} 1206
node_slabinfo_active_objects{slab="kmalloc-8192"} 132
node_slabinfo_active_objects{slab="kmem_cache"} 320
node_slabinfo_active_objects{slab="tw_sock_TCP"} 704
node_slabinfo_active_objects{index="0",slab="dmaengine-unmap-128"} 1206
node_slabinfo_active_objects{index="0",slab="kmalloc-8192"} 132
node_slabinfo_active_objects{index="0",slab="kmem_cache"} 320
node_slabinfo_active_objects{index="0",slab="tw_sock_TCP"} 704
# HELP node_slabinfo_object_size_bytes The size of objects in this slab, in bytes.
# TYPE node_slabinfo_object_size_bytes gauge
node_slabinfo_object_size_bytes{slab="dmaengine-unmap-128"} 1088
node_slabinfo_object_size_bytes{slab="kmalloc-8192"} 8192
node_slabinfo_object_size_bytes{slab="kmem_cache"} 256
node_slabinfo_object_size_bytes{slab="tw_sock_TCP"} 256
node_slabinfo_object_size_bytes{index="0",slab="dmaengine-unmap-128"} 1088
node_slabinfo_object_size_bytes{index="0",slab="kmalloc-8192"} 8192
node_slabinfo_object_size_bytes{index="0",slab="kmem_cache"} 256
node_slabinfo_object_size_bytes{index="0",slab="tw_sock_TCP"} 256
# HELP node_slabinfo_objects The total number of allocated objects (i.e., objects that are both in use and not in use).
# TYPE node_slabinfo_objects gauge
node_slabinfo_objects{slab="dmaengine-unmap-128"} 1320
node_slabinfo_objects{slab="kmalloc-8192"} 148
node_slabinfo_objects{slab="kmem_cache"} 320
node_slabinfo_objects{slab="tw_sock_TCP"} 864
node_slabinfo_objects{index="0",slab="dmaengine-unmap-128"} 1320
node_slabinfo_objects{index="0",slab="kmalloc-8192"} 148
node_slabinfo_objects{index="0",slab="kmem_cache"} 320
node_slabinfo_objects{index="0",slab="tw_sock_TCP"} 864
# HELP node_slabinfo_objects_per_slab The number of objects stored in each slab.
# TYPE node_slabinfo_objects_per_slab gauge
node_slabinfo_objects_per_slab{slab="dmaengine-unmap-128"} 30
node_slabinfo_objects_per_slab{slab="kmalloc-8192"} 4
node_slabinfo_objects_per_slab{slab="kmem_cache"} 32
node_slabinfo_objects_per_slab{slab="tw_sock_TCP"} 32
node_slabinfo_objects_per_slab{index="0",slab="dmaengine-unmap-128"} 30
node_slabinfo_objects_per_slab{index="0",slab="kmalloc-8192"} 4
node_slabinfo_objects_per_slab{index="0",slab="kmem_cache"} 32
node_slabinfo_objects_per_slab{index="0",slab="tw_sock_TCP"} 32
# HELP node_slabinfo_pages_per_slab The number of pages allocated for each slab.
# TYPE node_slabinfo_pages_per_slab gauge
node_slabinfo_pages_per_slab{slab="dmaengine-unmap-128"} 8
node_slabinfo_pages_per_slab{slab="kmalloc-8192"} 8
node_slabinfo_pages_per_slab{slab="kmem_cache"} 2
node_slabinfo_pages_per_slab{slab="tw_sock_TCP"} 2
node_slabinfo_pages_per_slab{index="0",slab="dmaengine-unmap-128"} 8
node_slabinfo_pages_per_slab{index="0",slab="kmalloc-8192"} 8
node_slabinfo_pages_per_slab{index="0",slab="kmem_cache"} 2
node_slabinfo_pages_per_slab{index="0",slab="tw_sock_TCP"} 2
# HELP node_sockstat_FRAG6_inuse Number of FRAG6 sockets in state inuse.
# TYPE node_sockstat_FRAG6_inuse gauge
node_sockstat_FRAG6_inuse 0
Expand Down
40 changes: 20 additions & 20 deletions collector/fixtures/e2e-output.txt
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -3806,34 +3806,34 @@ node_scrape_collector_success{collector="zfs"} 1
node_scrape_collector_success{collector="zoneinfo"} 1
# HELP node_slabinfo_active_objects The number of objects that are currently active (i.e., in use).
# TYPE node_slabinfo_active_objects gauge
node_slabinfo_active_objects{slab="dmaengine-unmap-128"} 1206
node_slabinfo_active_objects{slab="kmalloc-8192"} 132
node_slabinfo_active_objects{slab="kmem_cache"} 320
node_slabinfo_active_objects{slab="tw_sock_TCP"} 704
node_slabinfo_active_objects{index="0",slab="dmaengine-unmap-128"} 1206
node_slabinfo_active_objects{index="0",slab="kmalloc-8192"} 132
node_slabinfo_active_objects{index="0",slab="kmem_cache"} 320
node_slabinfo_active_objects{index="0",slab="tw_sock_TCP"} 704
# HELP node_slabinfo_object_size_bytes The size of objects in this slab, in bytes.
# TYPE node_slabinfo_object_size_bytes gauge
node_slabinfo_object_size_bytes{slab="dmaengine-unmap-128"} 1088
node_slabinfo_object_size_bytes{slab="kmalloc-8192"} 8192
node_slabinfo_object_size_bytes{slab="kmem_cache"} 256
node_slabinfo_object_size_bytes{slab="tw_sock_TCP"} 256
node_slabinfo_object_size_bytes{index="0",slab="dmaengine-unmap-128"} 1088
node_slabinfo_object_size_bytes{index="0",slab="kmalloc-8192"} 8192
node_slabinfo_object_size_bytes{index="0",slab="kmem_cache"} 256
node_slabinfo_object_size_bytes{index="0",slab="tw_sock_TCP"} 256
# HELP node_slabinfo_objects The total number of allocated objects (i.e., objects that are both in use and not in use).
# TYPE node_slabinfo_objects gauge
node_slabinfo_objects{slab="dmaengine-unmap-128"} 1320
node_slabinfo_objects{slab="kmalloc-8192"} 148
node_slabinfo_objects{slab="kmem_cache"} 320
node_slabinfo_objects{slab="tw_sock_TCP"} 864
node_slabinfo_objects{index="0",slab="dmaengine-unmap-128"} 1320
node_slabinfo_objects{index="0",slab="kmalloc-8192"} 148
node_slabinfo_objects{index="0",slab="kmem_cache"} 320
node_slabinfo_objects{index="0",slab="tw_sock_TCP"} 864
# HELP node_slabinfo_objects_per_slab The number of objects stored in each slab.
# TYPE node_slabinfo_objects_per_slab gauge
node_slabinfo_objects_per_slab{slab="dmaengine-unmap-128"} 30
node_slabinfo_objects_per_slab{slab="kmalloc-8192"} 4
node_slabinfo_objects_per_slab{slab="kmem_cache"} 32
node_slabinfo_objects_per_slab{slab="tw_sock_TCP"} 32
node_slabinfo_objects_per_slab{index="0",slab="dmaengine-unmap-128"} 30
node_slabinfo_objects_per_slab{index="0",slab="kmalloc-8192"} 4
node_slabinfo_objects_per_slab{index="0",slab="kmem_cache"} 32
node_slabinfo_objects_per_slab{index="0",slab="tw_sock_TCP"} 32
# HELP node_slabinfo_pages_per_slab The number of pages allocated for each slab.
# TYPE node_slabinfo_pages_per_slab gauge
node_slabinfo_pages_per_slab{slab="dmaengine-unmap-128"} 8
node_slabinfo_pages_per_slab{slab="kmalloc-8192"} 8
node_slabinfo_pages_per_slab{slab="kmem_cache"} 2
node_slabinfo_pages_per_slab{slab="tw_sock_TCP"} 2
node_slabinfo_pages_per_slab{index="0",slab="dmaengine-unmap-128"} 8
node_slabinfo_pages_per_slab{index="0",slab="kmalloc-8192"} 8
node_slabinfo_pages_per_slab{index="0",slab="kmem_cache"} 2
node_slabinfo_pages_per_slab{index="0",slab="tw_sock_TCP"} 2
# HELP node_sockstat_FRAG6_inuse Number of FRAG6 sockets in state inuse.
# TYPE node_sockstat_FRAG6_inuse gauge
node_sockstat_FRAG6_inuse 0
Expand Down
46 changes: 30 additions & 16 deletions collector/slabinfo_linux.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 +18,7 @@ package collector
import (
"fmt"
"log/slog"
"strconv"

"github.com/alecthomas/kingpin/v2"
"github.com/prometheus/client_golang/prometheus"
Expand Down Expand Up @@ -50,7 +51,7 @@ func NewSlabinfoCollector(logger *slog.Logger) (Collector, error) {
return &slabinfoCollector{logger: logger,
fs: fs,
subsystem: "slabinfo",
labels: []string{"slab"},
labels: []string{"slab", "index"},
slabNameFilter: newDeviceFilter(*slabNameExclude, *slabNameInclude),
}, nil
}
Expand All @@ -61,71 +62,84 @@ func (c *slabinfoCollector) Update(ch chan<- prometheus.Metric) error {
return fmt.Errorf("couldn't get %s: %w", c.subsystem, err)
}

// /proc/slabinfo can list the same slab name more than once, for example one
// cache per device instance. The kernel permits this: the duplicate-name check
// in kmem_cache_sanity_check() only WARNs, and is compiled out unless
// CONFIG_DEBUG_VM is set. Labelling by name alone therefore produces duplicate
// label sets, which fails the entire scrape.
//
// Each entry is given its position among the entries sharing its name, so every
// cache keeps its own series and its own geometry. The ordinal reflects the
// order of /proc/slabinfo and is not a stable identity: if a cache is created
// or destroyed, subsequent entries shift.
seen := make(map[string]int, len(slabinfo.Slabs))
for _, slab := range slabinfo.Slabs {
if c.slabNameFilter.ignored(slab.Name) {
continue
}
ch <- c.activeObjects(slab.Name, slab.ObjActive)
ch <- c.objects(slab.Name, slab.ObjNum)
ch <- c.objectSizeBytes(slab.Name, slab.ObjSize)
ch <- c.objectsPerSlab(slab.Name, slab.ObjPerSlab)
ch <- c.pagesPerSlab(slab.Name, slab.PagesPerSlab)
index := strconv.Itoa(seen[slab.Name])
seen[slab.Name]++
ch <- c.activeObjects(slab.Name, index, slab.ObjActive)
ch <- c.objects(slab.Name, index, slab.ObjNum)
ch <- c.objectSizeBytes(slab.Name, index, slab.ObjSize)
ch <- c.objectsPerSlab(slab.Name, index, slab.ObjPerSlab)
ch <- c.pagesPerSlab(slab.Name, index, slab.PagesPerSlab)
}

return nil
}

func (c *slabinfoCollector) activeObjects(label string, val int64) prometheus.Metric {
func (c *slabinfoCollector) activeObjects(label, index string, val int64) prometheus.Metric {
desc := prometheus.NewDesc(
prometheus.BuildFQName(namespace, c.subsystem, "active_objects"),
"The number of objects that are currently active (i.e., in use).",
c.labels, nil)

return prometheus.MustNewConstMetric(
desc, prometheus.GaugeValue, float64(val), label,
desc, prometheus.GaugeValue, float64(val), label, index,
)
}

func (c *slabinfoCollector) objects(label string, val int64) prometheus.Metric {
func (c *slabinfoCollector) objects(label, index string, val int64) prometheus.Metric {
desc := prometheus.NewDesc(
prometheus.BuildFQName(namespace, c.subsystem, "objects"),
"The total number of allocated objects (i.e., objects that are both in use and not in use).",
c.labels, nil)

return prometheus.MustNewConstMetric(
desc, prometheus.GaugeValue, float64(val), label,
desc, prometheus.GaugeValue, float64(val), label, index,
)
}

func (c *slabinfoCollector) objectSizeBytes(label string, val int64) prometheus.Metric {
func (c *slabinfoCollector) objectSizeBytes(label, index string, val int64) prometheus.Metric {
desc := prometheus.NewDesc(
prometheus.BuildFQName(namespace, c.subsystem, "object_size_bytes"),
"The size of objects in this slab, in bytes.",
c.labels, nil)

return prometheus.MustNewConstMetric(
desc, prometheus.GaugeValue, float64(val), label,
desc, prometheus.GaugeValue, float64(val), label, index,
)
}

func (c *slabinfoCollector) objectsPerSlab(label string, val int64) prometheus.Metric {
func (c *slabinfoCollector) objectsPerSlab(label, index string, val int64) prometheus.Metric {
desc := prometheus.NewDesc(
prometheus.BuildFQName(namespace, c.subsystem, "objects_per_slab"),
"The number of objects stored in each slab.",
c.labels, nil)

return prometheus.MustNewConstMetric(
desc, prometheus.GaugeValue, float64(val), label,
desc, prometheus.GaugeValue, float64(val), label, index,
)
}

func (c *slabinfoCollector) pagesPerSlab(label string, val int64) prometheus.Metric {
func (c *slabinfoCollector) pagesPerSlab(label, index string, val int64) prometheus.Metric {
desc := prometheus.NewDesc(
prometheus.BuildFQName(namespace, c.subsystem, "pages_per_slab"),
"The number of pages allocated for each slab.",
c.labels, nil)

return prometheus.MustNewConstMetric(
desc, prometheus.GaugeValue, float64(val), label,
desc, prometheus.GaugeValue, float64(val), label, index,
)
}
169 changes: 169 additions & 0 deletions collector/slabinfo_linux_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,169 @@
// Copyright The Prometheus Authors
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

//go:build !noslabinfo

package collector

import (
"log/slog"
"os"
"path/filepath"
"testing"

"github.com/prometheus/client_golang/prometheus"
)

const slabinfoHeader = `slabinfo - version: 2.1
# name <active_objs> <num_objs> <objsize> <objperslab> <pagesperslab> : tunables <limit> <batchcount> <sharedfactor> : slabdata <active_slabs> <num_slabs> <sharedavail>
`

// writeSlabinfo writes body into a temporary procfs and points the collector at
// it for the duration of the test. procPath is a package-global flag value, so
// it is restored afterwards to keep these tests independent of execution order.
// These tests must not call t.Parallel() for the same reason.
func writeSlabinfo(t *testing.T, body string) {
t.Helper()
dir := t.TempDir()
if err := os.WriteFile(filepath.Join(dir, "slabinfo"), []byte(slabinfoHeader+body), 0o644); err != nil {
t.Fatal(err)
}
orig := *procPath
t.Cleanup(func() { *procPath = orig })
*procPath = dir
}

// gatherSlabinfo collects through a registry, which is what surfaces duplicate
// label sets, and keys the result by "metric/slab/index".
func gatherSlabinfo(t *testing.T) map[string]float64 {
t.Helper()
c, err := NewSlabinfoCollector(slog.New(slog.NewTextHandler(os.Stderr, nil)))
if err != nil {
t.Fatal(err)
}
reg := prometheus.NewPedanticRegistry()
if err := reg.Register(prometheus.CollectorFunc(func(ch chan<- prometheus.Metric) {
if err := c.Update(ch); err != nil {
t.Errorf("Update failed: %v", err)
}
})); err != nil {
t.Fatal(err)
}
mfs, err := reg.Gather()
if err != nil {
t.Fatalf("Gather failed: %v", err)
}
got := map[string]float64{}
for _, mf := range mfs {
for _, m := range mf.Metric {
var name, index string
for _, l := range m.Label {
switch l.GetName() {
case "slab":
name = l.GetValue()
case "index":
index = l.GetValue()
}
}
got[mf.GetName()+"/"+name+"/"+index] = m.GetGauge().GetValue()
}
}
return got
}

func checkSlabinfo(t *testing.T, got map[string]float64, want map[string]float64) {
t.Helper()
for key, w := range want {
if g, ok := got[key]; !ok {
t.Errorf("%s: missing", key)
} else if g != w {
t.Errorf("%s: got %v, want %v", key, g, w)
}
}
}

// A slab name may appear more than once, for example one cache per device
// instance. Each entry keeps its own series, distinguished by the index label.
func TestSlabinfoDuplicateSlabNames(t *testing.T) {
writeSlabinfo(t, `kmem_cache 320 320 256 32 2 : tunables 0 0 0 : slabdata 10 10 0
mlx5_fs_ftes 44 44 736 44 8 : tunables 0 0 0 : slabdata 1 1 0
mlx5_fs_fgs 42 42 776 42 8 : tunables 0 0 0 : slabdata 1 1 0
mlx5_fs_ftes 396 396 736 44 8 : tunables 0 0 0 : slabdata 9 9 0
mlx5_fs_fgs 84 84 776 42 8 : tunables 0 0 0 : slabdata 2 2 0
`)
checkSlabinfo(t, gatherSlabinfo(t), map[string]float64{
"node_slabinfo_active_objects/mlx5_fs_ftes/0": 44,
"node_slabinfo_active_objects/mlx5_fs_ftes/1": 396,
"node_slabinfo_objects/mlx5_fs_ftes/1": 396,
"node_slabinfo_active_objects/mlx5_fs_fgs/0": 42,
"node_slabinfo_active_objects/mlx5_fs_fgs/1": 84,
"node_slabinfo_object_size_bytes/mlx5_fs_ftes/1": 736,
"node_slabinfo_objects_per_slab/mlx5_fs_ftes/1": 44,
"node_slabinfo_pages_per_slab/mlx5_fs_ftes/1": 8,
"node_slabinfo_active_objects/kmem_cache/0": 320,
})
}

// Slabs with distinct names each get index 0.
func TestSlabinfoDistinctSlabNames(t *testing.T) {
writeSlabinfo(t, `tw_sock_TCP 704 864 256 32 2 : tunables 0 0 0 : slabdata 27 27 0
kmem_cache 320 320 256 32 2 : tunables 0 0 0 : slabdata 10 10 0
`)
checkSlabinfo(t, gatherSlabinfo(t), map[string]float64{
"node_slabinfo_active_objects/tw_sock_TCP/0": 704,
"node_slabinfo_objects/tw_sock_TCP/0": 864,
"node_slabinfo_active_objects/kmem_cache/0": 320,
"node_slabinfo_objects/kmem_cache/0": 320,
})
}

// The kernel does not guarantee that same-named caches share geometry: the
// duplicate-name check in kmem_cache_sanity_check() only WARNs and is compiled
// out without CONFIG_DEBUG_VM, and object_size is per-cache. Indexing keeps each
// cache's own geometry, so no representative value has to be chosen.
func TestSlabinfoDuplicateSlabNamesDifferentGeometry(t *testing.T) {
writeSlabinfo(t, `weird_cache 10 10 128 32 2 : tunables 0 0 0 : slabdata 1 1 0
weird_cache 20 20 256 16 4 : tunables 0 0 0 : slabdata 2 2 0
`)
checkSlabinfo(t, gatherSlabinfo(t), map[string]float64{
"node_slabinfo_active_objects/weird_cache/0": 10,
"node_slabinfo_object_size_bytes/weird_cache/0": 128,
"node_slabinfo_objects_per_slab/weird_cache/0": 32,
"node_slabinfo_pages_per_slab/weird_cache/0": 2,
"node_slabinfo_active_objects/weird_cache/1": 20,
"node_slabinfo_object_size_bytes/weird_cache/1": 256,
"node_slabinfo_objects_per_slab/weird_cache/1": 16,
"node_slabinfo_pages_per_slab/weird_cache/1": 4,
})
}

// Excluded slabs must not consume an index, so the ordinals of the remaining
// entries stay contiguous.
func TestSlabinfoIndexIgnoresFilteredSlabs(t *testing.T) {
orig := *slabNameExclude
t.Cleanup(func() { *slabNameExclude = orig })
*slabNameExclude = "^drop_me$"

writeSlabinfo(t, `dup_cache 10 10 128 32 2 : tunables 0 0 0 : slabdata 1 1 0
drop_me 99 99 999 99 9 : tunables 0 0 0 : slabdata 9 9 0
dup_cache 20 20 128 32 2 : tunables 0 0 0 : slabdata 2 2 0
`)
got := gatherSlabinfo(t)
checkSlabinfo(t, got, map[string]float64{
"node_slabinfo_active_objects/dup_cache/0": 10,
"node_slabinfo_active_objects/dup_cache/1": 20,
})
if _, ok := got["node_slabinfo_active_objects/drop_me/0"]; ok {
t.Error("excluded slab was collected")
}
}