diff --git a/internal/dashboard/defaults.go b/internal/dashboard/defaults.go index bd8517710..60a470711 100644 --- a/internal/dashboard/defaults.go +++ b/internal/dashboard/defaults.go @@ -30,7 +30,7 @@ func DefaultSpec() panel.Dashboard { {ID: "service_health", Title: "Is anything failing or slow?", Description: receiverDescription("otlp", "Service health and error trend with deploy markers."), Viz: "health", Width: 12, Query: &panel.Query{From: "spans", Where: []string{service}}}, {ID: "service_map", Title: "Services and external dependencies", Description: receiverDescription("otlp", "Failing or slow services and external dependencies observed in client spans."), Viz: "service_map", Width: 12, Height: "l", Query: &panel.Query{From: "spans", Where: []string{service}}}, capacityPanel("hosts", "Is anything running out? Hosts", "hostmetrics", "CPU busy, memory used and the fullest filesystem; worst first. Not reporting after two intervals is bad.", hostCapacitySQL()), - capacityPanel("containers", "Containers", "docker_stats", "CPU and memory usage against the container limit; worst first. Not reporting after two intervals is bad.", containerCapacitySQL()), + capacityPanel("containers", "Containers", "docker_stats", "CPU: 100% is one core. Status uses limits; memory only if no CPU limit. Silence after two intervals is bad.", containerCapacitySQL()), {ID: "versions", Title: "What changed?", Description: receiverDescription("otlp", "Service versions, first seen in the selected range, runtime and SDK."), Viz: "table", Width: 12, SQL: `WITH observations AS (` + versionObservationsSQL + `) SELECT namespace,service,version,min(seen) AS first_seen,runtime_name,runtime_version,sdk_language,sdk_version FROM observations WHERE version IS NOT NULL AND version<>'' GROUP BY namespace,service,version,runtime_name,runtime_version,sdk_language,sdk_version ORDER BY first_seen DESC,namespace,service,version`}, }, } diff --git a/internal/dashboard/defaults_sql.go b/internal/dashboard/defaults_sql.go index 4f5e29972..e787f9c94 100644 --- a/internal/dashboard/defaults_sql.go +++ b/internal/dashboard/defaults_sql.go @@ -53,11 +53,11 @@ func capacityCTE(containers bool) string { priority := "CASE name WHEN 'system.cpu.time' THEN 1 WHEN 'system.memory.usage' THEN 2 ELSE 3 END" if containers { identity += " AND container_id IS NOT NULL AND container_id<>''" - names = "'container.cpu.utilization','container.memory.usage.total','container.memory.usage.limit'" + names = "'container.cpu.utilization','container.cpu.limit','container.memory.usage.total','container.memory.usage.limit'" entity = "to_json([host,container_id])::VARCHAR" - priority = "CASE name WHEN 'container.cpu.utilization' THEN 1 WHEN 'container.memory.usage.total' THEN 2 ELSE 3 END" + priority = "CASE name WHEN 'container.cpu.utilization' THEN 1 WHEN 'container.memory.usage.total' THEN 2 WHEN 'container.cpu.limit' THEN 4 ELSE 3 END" } - return `WITH raw AS MATERIALIZED (SELECT series_id,ingested_unix_nano,namespace,time,value,type,name,start_time,temporality,unit, + return `WITH raw AS MATERIALIZED (SELECT series_id,ingested_unix_nano,namespace,time,value,type,name,start_time,temporality,unit,scope_name, resource['host.name']::VARCHAR AS host,resource['container.id']::VARCHAR AS container_id, resource['container.name']::VARCHAR AS container_name,attributes['state']::VARCHAR AS state, attributes['mountpoint']::VARCHAR AS mountpoint,attributes['device']::VARCHAR AS device @@ -98,12 +98,16 @@ func hostCapacitySQL() string { FROM results ORDER BY (reporting<>'reporting') DESC,status DESC NULLS LAST,namespace,host` } func containerCapacitySQL() string { + // docker_stats reports percent of one core despite declaring unit "1". + // Only a reported CPU limit makes that usage a capacity signal. return capacityCTE(true) + `, values_by_container AS (SELECT namespace,entity, - sum(value*CASE unit WHEN '%' THEN 1 WHEN '1' THEN 100 END) FILTER (WHERE name='container.cpu.utilization') AS cpu_percent, + sum(value*CASE WHEN scope_name='github.com/open-telemetry/opentelemetry-collector-contrib/receiver/dockerstatsreceiver' THEN 1 ELSE CASE unit WHEN '%' THEN 1 WHEN '1' THEN 100 END END) FILTER (WHERE name='container.cpu.utilization') AS cpu_percent, + sum(value) FILTER (WHERE name='container.cpu.limit' AND scope_name='github.com/open-telemetry/opentelemetry-collector-contrib/receiver/dockerstatsreceiver' AND value>0) AS cpu_limit, sum(value) FILTER (WHERE name='container.memory.usage.total') AS total_bytes,sum(value) FILTER (WHERE name='container.memory.usage.limit') AS limit_bytes FROM latest GROUP BY namespace,entity), - results AS (SELECT i.namespace,i.host,coalesce(i.container,i.container_id) AS container,` + strings.ReplaceAll(reportingSQL, "/* identity */", "OR i.container_id IS NULL OR i.container_id=''") + ` AS reporting,v.cpu_percent,100.0*v.total_bytes/nullif(v.limit_bytes,0) AS memory_percent,v.total_bytes,v.limit_bytes,c.last_seen FROM inventory i LEFT JOIN cadence c USING(namespace,entity) LEFT JOIN values_by_container v USING(namespace,entity)) - SELECT namespace,host,container,CASE WHEN reporting<>'reporting' THEN 100 ELSE greatest(cpu_percent,memory_percent) END::DOUBLE AS status,reporting,cpu_percent,memory_percent,total_bytes,limit_bytes,last_seen, - CASE WHEN limit_bytes IS NULL OR limit_bytes=0 THEN 'memory limit not reported' WHEN cpu_percent IS NULL OR memory_percent IS NULL THEN 'measurement missing, unsupported or stale' END AS measurement + results AS (SELECT i.namespace,i.host,coalesce(i.container,i.container_id) AS container,` + strings.ReplaceAll(reportingSQL, "/* identity */", "OR i.container_id IS NULL OR i.container_id=''") + ` AS reporting,v.cpu_percent,v.cpu_limit,100.0*v.total_bytes/nullif(v.limit_bytes,0) AS memory_percent,v.total_bytes,v.limit_bytes,c.last_seen FROM inventory i LEFT JOIN cadence c USING(namespace,entity) LEFT JOIN values_by_container v USING(namespace,entity)) + SELECT namespace,host,container,CASE WHEN reporting<>'reporting' THEN 100 ELSE greatest(cpu_percent/nullif(cpu_limit,0),memory_percent) END::DOUBLE AS status,reporting,cpu_percent,memory_percent,total_bytes,limit_bytes,last_seen, + nullif(concat_ws('; ',CASE WHEN cpu_limit IS NULL THEN 'cpu limit not reported' END, + CASE WHEN limit_bytes IS NULL OR limit_bytes=0 THEN 'memory limit not reported' WHEN cpu_percent IS NULL OR memory_percent IS NULL THEN 'measurement missing, unsupported or stale' END),'') AS measurement FROM results ORDER BY (reporting<>'reporting') DESC,status DESC NULLS LAST,namespace,host,container` } diff --git a/internal/dashboard/defaults_test.go b/internal/dashboard/defaults_test.go index c5a77a8b8..220c6adbf 100644 --- a/internal/dashboard/defaults_test.go +++ b/internal/dashboard/defaults_test.go @@ -414,3 +414,106 @@ func TestDefaultCapacityPreservesRetriesCountersNamespacesAndFreshness(t *testin } } } + +func TestDefaultContainersUseScopeCPUUnitsAndReportedLimitsForStatus(t *testing.T) { + e, repo := defaultFixture(t) + start := time.Date(2026, 10, 1, 12, 0, 0, 0, time.UTC) + end := start.Add(2 * time.Minute) + const dockerScope = "github.com/open-telemetry/opentelemetry-collector-contrib/receiver/dockerstatsreceiver" + limit := func(value float64) *float64 { return &value } + tests := []struct { + name string + scope string + unit string + cpu float64 + cpuLimit *float64 + staleLimit bool + memoryLimit *float64 + wantCPU any + wantStatus any + measurement any + }{ + {name: "docker-small", scope: dockerScope, unit: "1", cpu: 2, memoryLimit: limit(100), wantCPU: 2.0, wantStatus: 20.0, measurement: "cpu limit not reported"}, + {name: "docker-multicore-unlimited", scope: dockerScope, unit: "1", cpu: 193.46, memoryLimit: limit(100), wantCPU: 193.46, wantStatus: 20.0, measurement: "cpu limit not reported"}, + {name: "other-ratio", scope: "another.receiver", unit: "1", cpu: 0.5, memoryLimit: limit(100), wantCPU: 50.0, wantStatus: 20.0, measurement: "cpu limit not reported"}, + {name: "other-percent", scope: "another.receiver", unit: "%", cpu: 50, memoryLimit: limit(100), wantCPU: 50.0, wantStatus: 20.0, measurement: "cpu limit not reported"}, + {name: "docker-percent", scope: dockerScope, unit: "%", cpu: 193.46, memoryLimit: limit(100), wantCPU: 193.46, wantStatus: 20.0, measurement: "cpu limit not reported"}, + {name: "eight-core-limit", scope: dockerScope, unit: "1", cpu: 193.46, cpuLimit: limit(8), memoryLimit: limit(100), wantCPU: 193.46, wantStatus: 193.46 / 8}, + {name: "two-core-limit", scope: dockerScope, unit: "1", cpu: 193.46, cpuLimit: limit(2), memoryLimit: limit(100), wantCPU: 193.46, wantStatus: 193.46 / 2}, + {name: "one-core-warning", scope: dockerScope, unit: "1", cpu: 85, cpuLimit: limit(1), memoryLimit: limit(100), wantCPU: 85.0, wantStatus: 85.0}, + {name: "memory-worse", scope: dockerScope, unit: "1", cpu: 193.46, cpuLimit: limit(8), memoryLimit: limit(25), wantCPU: 193.46, wantStatus: 80.0}, + {name: "memory-bad-unlimited", scope: dockerScope, unit: "1", cpu: 2, memoryLimit: limit(20), wantCPU: 2.0, wantStatus: 100.0, measurement: "cpu limit not reported"}, + {name: "zero-cpu-limit", scope: dockerScope, unit: "1", cpu: 193.46, cpuLimit: limit(0), memoryLimit: limit(100), wantCPU: 193.46, wantStatus: 20.0, measurement: "cpu limit not reported"}, + {name: "negative-cpu-limit", scope: dockerScope, unit: "1", cpu: 193.46, cpuLimit: limit(-1), memoryLimit: limit(100), wantCPU: 193.46, wantStatus: 20.0, measurement: "cpu limit not reported"}, + {name: "stale-cpu-limit", scope: dockerScope, unit: "1", cpu: 193.46, cpuLimit: limit(1), staleLimit: true, memoryLimit: limit(100), wantCPU: 193.46, wantStatus: 20.0, measurement: "cpu limit not reported"}, + {name: "other-scope-limit", scope: "another.receiver", unit: "1", cpu: 0.5, cpuLimit: limit(0.1), memoryLimit: limit(100), wantCPU: 50.0, wantStatus: 20.0, measurement: "cpu limit not reported"}, + {name: "cpu-only-limit", scope: dockerScope, unit: "1", cpu: 193.46, cpuLimit: limit(2), wantCPU: 193.46, wantStatus: 193.46 / 2, measurement: "memory limit not reported"}, + {name: "neither-limit", scope: dockerScope, unit: "1", cpu: 193.46, wantCPU: 193.46, measurement: "cpu limit not reported; memory limit not reported"}, + {name: "unsupported-other-unit", scope: "another.receiver", unit: "unknown", cpu: 0.5, memoryLimit: limit(100), wantStatus: 20.0, measurement: "cpu limit not reported; measurement missing, unsupported or stale"}, + } + var metrics []telemetry.Metric + for _, tt := range tests { + for _, at := range []time.Time{start, start.Add(time.Minute)} { + metric := func(name, unit string, value float64, at time.Time) telemetry.Metric { + m := telemetry.Metric{Namespace: "test", Name: name, Type: "gauge", Unit: unit, Value: value, ScopeName: tt.scope, Resource: map[string]any{"host.name": "host", "container.id": tt.name}, TimeUnixNanos: at.UnixNano(), EventUnixNanos: at.UnixNano()} + m.SeriesID = telemetry.MetricSeriesID(m) + return m + } + metrics = append(metrics, metric("container.cpu.utilization", tt.unit, tt.cpu, at), metric("container.memory.usage.total", "By", 20, at)) + if tt.memoryLimit != nil { + metrics = append(metrics, metric("container.memory.usage.limit", "By", *tt.memoryLimit, at)) + } + if tt.cpuLimit != nil { + limitTime := at + if tt.staleLimit { + limitTime = at.Add(-6 * time.Minute) + } + metrics = append(metrics, metric("container.cpu.limit", "{cpu}", *tt.cpuLimit, limitTime)) + } + } + } + if err := repo.Commit(t.Context(), telemetrystore.Batch{ID: "container-cpu-limits", Metrics: metrics}); err != nil { + t.Fatal(err) + } + spec := DefaultSpec() + for _, p := range spec.Panels { + if p.ID == "containers" && !strings.Contains(p.Description, "100% is one core") { + t.Fatal("container CPU percentage must explain its denominator") + } + } + spec.Time = panel.Time{From: &start, To: &end} + results, err := e.Run(t.Context(), panel.RunRequest{Dashboard: spec, Panels: []string{"containers"}}) + if err != nil { + t.Fatal(err) + } + result := results[0] + containers := defaultValues(t, result, "container") + if len(containers) != len(tests) { + t.Fatalf("containers: got %d, want %d", len(containers), len(tests)) + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + row := -1 + for i, name := range containers { + if name == tt.name { + row = i + break + } + } + if row < 0 { + t.Fatal("container missing") + } + for column, want := range map[string]any{"cpu_percent": tt.wantCPU, "status": tt.wantStatus, "measurement": tt.measurement, "reporting": "reporting"} { + got := defaultValues(t, result, column)[row] + if number, ok := want.(float64); ok { + if value, ok := got.(float64); ok && math.Abs(value-number) <= 1e-12 { + continue + } + } else if got == want { + continue + } + t.Errorf("%s: got %v, want %v", column, got, want) + } + }) + } +}