Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion internal/dashboard/defaults.go
Original file line number Diff line number Diff line change
Expand Up @@ -30,7 +30,7 @@ func DefaultSpec() panel.Dashboard {
{ID: "service_health", Title: "Is anything failing or slow?", Description: receiverDescription("otlp", "Service health and error trend with deploy markers."), Viz: "health", Width: 12, Query: &panel.Query{From: "spans", Where: []string{service}}},
{ID: "service_map", Title: "Services and external dependencies", Description: receiverDescription("otlp", "Failing or slow services and external dependencies observed in client spans."), Viz: "service_map", Width: 12, Height: "l", Query: &panel.Query{From: "spans", Where: []string{service}}},
capacityPanel("hosts", "Is anything running out? Hosts", "hostmetrics", "CPU busy, memory used and the fullest filesystem; worst first. Not reporting after two intervals is bad.", hostCapacitySQL()),
capacityPanel("containers", "Containers", "docker_stats", "CPU and memory usage against the container limit; worst first. Not reporting after two intervals is bad.", containerCapacitySQL()),
capacityPanel("containers", "Containers", "docker_stats", "CPU: 100% is one core. Status uses limits; memory only if no CPU limit. Silence after two intervals is bad.", containerCapacitySQL()),
{ID: "versions", Title: "What changed?", Description: receiverDescription("otlp", "Service versions, first seen in the selected range, runtime and SDK."), Viz: "table", Width: 12, SQL: `WITH observations AS (` + versionObservationsSQL + `) SELECT namespace,service,version,min(seen) AS first_seen,runtime_name,runtime_version,sdk_language,sdk_version FROM observations WHERE version IS NOT NULL AND version<>'' GROUP BY namespace,service,version,runtime_name,runtime_version,sdk_language,sdk_version ORDER BY first_seen DESC,namespace,service,version`},
},
}
Expand Down
18 changes: 11 additions & 7 deletions internal/dashboard/defaults_sql.go
Original file line number Diff line number Diff line change
Expand Up @@ -53,11 +53,11 @@ func capacityCTE(containers bool) string {
priority := "CASE name WHEN 'system.cpu.time' THEN 1 WHEN 'system.memory.usage' THEN 2 ELSE 3 END"
if containers {
identity += " AND container_id IS NOT NULL AND container_id<>''"
names = "'container.cpu.utilization','container.memory.usage.total','container.memory.usage.limit'"
names = "'container.cpu.utilization','container.cpu.limit','container.memory.usage.total','container.memory.usage.limit'"
entity = "to_json([host,container_id])::VARCHAR"
priority = "CASE name WHEN 'container.cpu.utilization' THEN 1 WHEN 'container.memory.usage.total' THEN 2 ELSE 3 END"
priority = "CASE name WHEN 'container.cpu.utilization' THEN 1 WHEN 'container.memory.usage.total' THEN 2 WHEN 'container.cpu.limit' THEN 4 ELSE 3 END"
}
return `WITH raw AS MATERIALIZED (SELECT series_id,ingested_unix_nano,namespace,time,value,type,name,start_time,temporality,unit,
return `WITH raw AS MATERIALIZED (SELECT series_id,ingested_unix_nano,namespace,time,value,type,name,start_time,temporality,unit,scope_name,
resource['host.name']::VARCHAR AS host,resource['container.id']::VARCHAR AS container_id,
resource['container.name']::VARCHAR AS container_name,attributes['state']::VARCHAR AS state,
attributes['mountpoint']::VARCHAR AS mountpoint,attributes['device']::VARCHAR AS device
Expand Down Expand Up @@ -98,12 +98,16 @@ func hostCapacitySQL() string {
FROM results ORDER BY (reporting<>'reporting') DESC,status DESC NULLS LAST,namespace,host`
}
func containerCapacitySQL() string {
// docker_stats reports percent of one core despite declaring unit "1".
// Only a reported CPU limit makes that usage a capacity signal.
return capacityCTE(true) + `, values_by_container AS (SELECT namespace,entity,
sum(value*CASE unit WHEN '%' THEN 1 WHEN '1' THEN 100 END) FILTER (WHERE name='container.cpu.utilization') AS cpu_percent,
sum(value*CASE WHEN scope_name='github.com/open-telemetry/opentelemetry-collector-contrib/receiver/dockerstatsreceiver' THEN 1 ELSE CASE unit WHEN '%' THEN 1 WHEN '1' THEN 100 END END) FILTER (WHERE name='container.cpu.utilization') AS cpu_percent,
sum(value) FILTER (WHERE name='container.cpu.limit' AND scope_name='github.com/open-telemetry/opentelemetry-collector-contrib/receiver/dockerstatsreceiver' AND value>0) AS cpu_limit,
sum(value) FILTER (WHERE name='container.memory.usage.total') AS total_bytes,sum(value) FILTER (WHERE name='container.memory.usage.limit') AS limit_bytes FROM latest GROUP BY namespace,entity),
results AS (SELECT i.namespace,i.host,coalesce(i.container,i.container_id) AS container,` + strings.ReplaceAll(reportingSQL, "/* identity */", "OR i.container_id IS NULL OR i.container_id=''") + ` AS reporting,v.cpu_percent,100.0*v.total_bytes/nullif(v.limit_bytes,0) AS memory_percent,v.total_bytes,v.limit_bytes,c.last_seen FROM inventory i LEFT JOIN cadence c USING(namespace,entity) LEFT JOIN values_by_container v USING(namespace,entity))
SELECT namespace,host,container,CASE WHEN reporting<>'reporting' THEN 100 ELSE greatest(cpu_percent,memory_percent) END::DOUBLE AS status,reporting,cpu_percent,memory_percent,total_bytes,limit_bytes,last_seen,
CASE WHEN limit_bytes IS NULL OR limit_bytes=0 THEN 'memory limit not reported' WHEN cpu_percent IS NULL OR memory_percent IS NULL THEN 'measurement missing, unsupported or stale' END AS measurement
results AS (SELECT i.namespace,i.host,coalesce(i.container,i.container_id) AS container,` + strings.ReplaceAll(reportingSQL, "/* identity */", "OR i.container_id IS NULL OR i.container_id=''") + ` AS reporting,v.cpu_percent,v.cpu_limit,100.0*v.total_bytes/nullif(v.limit_bytes,0) AS memory_percent,v.total_bytes,v.limit_bytes,c.last_seen FROM inventory i LEFT JOIN cadence c USING(namespace,entity) LEFT JOIN values_by_container v USING(namespace,entity))
SELECT namespace,host,container,CASE WHEN reporting<>'reporting' THEN 100 ELSE greatest(cpu_percent/nullif(cpu_limit,0),memory_percent) END::DOUBLE AS status,reporting,cpu_percent,memory_percent,total_bytes,limit_bytes,last_seen,
nullif(concat_ws('; ',CASE WHEN cpu_limit IS NULL THEN 'cpu limit not reported' END,
CASE WHEN limit_bytes IS NULL OR limit_bytes=0 THEN 'memory limit not reported' WHEN cpu_percent IS NULL OR memory_percent IS NULL THEN 'measurement missing, unsupported or stale' END),'') AS measurement
FROM results ORDER BY (reporting<>'reporting') DESC,status DESC NULLS LAST,namespace,host,container`
}

Expand Down
103 changes: 103 additions & 0 deletions internal/dashboard/defaults_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -414,3 +414,106 @@ func TestDefaultCapacityPreservesRetriesCountersNamespacesAndFreshness(t *testin
}
}
}

func TestDefaultContainersUseScopeCPUUnitsAndReportedLimitsForStatus(t *testing.T) {
e, repo := defaultFixture(t)
start := time.Date(2026, 10, 1, 12, 0, 0, 0, time.UTC)
end := start.Add(2 * time.Minute)
const dockerScope = "github.com/open-telemetry/opentelemetry-collector-contrib/receiver/dockerstatsreceiver"
limit := func(value float64) *float64 { return &value }
tests := []struct {
name string
scope string
unit string
cpu float64
cpuLimit *float64
staleLimit bool
memoryLimit *float64
wantCPU any
wantStatus any
measurement any
}{
{name: "docker-small", scope: dockerScope, unit: "1", cpu: 2, memoryLimit: limit(100), wantCPU: 2.0, wantStatus: 20.0, measurement: "cpu limit not reported"},
{name: "docker-multicore-unlimited", scope: dockerScope, unit: "1", cpu: 193.46, memoryLimit: limit(100), wantCPU: 193.46, wantStatus: 20.0, measurement: "cpu limit not reported"},
{name: "other-ratio", scope: "another.receiver", unit: "1", cpu: 0.5, memoryLimit: limit(100), wantCPU: 50.0, wantStatus: 20.0, measurement: "cpu limit not reported"},
{name: "other-percent", scope: "another.receiver", unit: "%", cpu: 50, memoryLimit: limit(100), wantCPU: 50.0, wantStatus: 20.0, measurement: "cpu limit not reported"},
{name: "docker-percent", scope: dockerScope, unit: "%", cpu: 193.46, memoryLimit: limit(100), wantCPU: 193.46, wantStatus: 20.0, measurement: "cpu limit not reported"},
{name: "eight-core-limit", scope: dockerScope, unit: "1", cpu: 193.46, cpuLimit: limit(8), memoryLimit: limit(100), wantCPU: 193.46, wantStatus: 193.46 / 8},
{name: "two-core-limit", scope: dockerScope, unit: "1", cpu: 193.46, cpuLimit: limit(2), memoryLimit: limit(100), wantCPU: 193.46, wantStatus: 193.46 / 2},
{name: "one-core-warning", scope: dockerScope, unit: "1", cpu: 85, cpuLimit: limit(1), memoryLimit: limit(100), wantCPU: 85.0, wantStatus: 85.0},
{name: "memory-worse", scope: dockerScope, unit: "1", cpu: 193.46, cpuLimit: limit(8), memoryLimit: limit(25), wantCPU: 193.46, wantStatus: 80.0},
{name: "memory-bad-unlimited", scope: dockerScope, unit: "1", cpu: 2, memoryLimit: limit(20), wantCPU: 2.0, wantStatus: 100.0, measurement: "cpu limit not reported"},
{name: "zero-cpu-limit", scope: dockerScope, unit: "1", cpu: 193.46, cpuLimit: limit(0), memoryLimit: limit(100), wantCPU: 193.46, wantStatus: 20.0, measurement: "cpu limit not reported"},
{name: "negative-cpu-limit", scope: dockerScope, unit: "1", cpu: 193.46, cpuLimit: limit(-1), memoryLimit: limit(100), wantCPU: 193.46, wantStatus: 20.0, measurement: "cpu limit not reported"},
{name: "stale-cpu-limit", scope: dockerScope, unit: "1", cpu: 193.46, cpuLimit: limit(1), staleLimit: true, memoryLimit: limit(100), wantCPU: 193.46, wantStatus: 20.0, measurement: "cpu limit not reported"},
{name: "other-scope-limit", scope: "another.receiver", unit: "1", cpu: 0.5, cpuLimit: limit(0.1), memoryLimit: limit(100), wantCPU: 50.0, wantStatus: 20.0, measurement: "cpu limit not reported"},
{name: "cpu-only-limit", scope: dockerScope, unit: "1", cpu: 193.46, cpuLimit: limit(2), wantCPU: 193.46, wantStatus: 193.46 / 2, measurement: "memory limit not reported"},
{name: "neither-limit", scope: dockerScope, unit: "1", cpu: 193.46, wantCPU: 193.46, measurement: "cpu limit not reported; memory limit not reported"},
{name: "unsupported-other-unit", scope: "another.receiver", unit: "unknown", cpu: 0.5, memoryLimit: limit(100), wantStatus: 20.0, measurement: "cpu limit not reported; measurement missing, unsupported or stale"},
}
var metrics []telemetry.Metric
for _, tt := range tests {
for _, at := range []time.Time{start, start.Add(time.Minute)} {
metric := func(name, unit string, value float64, at time.Time) telemetry.Metric {
m := telemetry.Metric{Namespace: "test", Name: name, Type: "gauge", Unit: unit, Value: value, ScopeName: tt.scope, Resource: map[string]any{"host.name": "host", "container.id": tt.name}, TimeUnixNanos: at.UnixNano(), EventUnixNanos: at.UnixNano()}
m.SeriesID = telemetry.MetricSeriesID(m)
return m
}
metrics = append(metrics, metric("container.cpu.utilization", tt.unit, tt.cpu, at), metric("container.memory.usage.total", "By", 20, at))
if tt.memoryLimit != nil {
metrics = append(metrics, metric("container.memory.usage.limit", "By", *tt.memoryLimit, at))
}
if tt.cpuLimit != nil {
limitTime := at
if tt.staleLimit {
limitTime = at.Add(-6 * time.Minute)
}
metrics = append(metrics, metric("container.cpu.limit", "{cpu}", *tt.cpuLimit, limitTime))
}
}
}
if err := repo.Commit(t.Context(), telemetrystore.Batch{ID: "container-cpu-limits", Metrics: metrics}); err != nil {
t.Fatal(err)
}
spec := DefaultSpec()
for _, p := range spec.Panels {
if p.ID == "containers" && !strings.Contains(p.Description, "100% is one core") {
t.Fatal("container CPU percentage must explain its denominator")
}
}
spec.Time = panel.Time{From: &start, To: &end}
results, err := e.Run(t.Context(), panel.RunRequest{Dashboard: spec, Panels: []string{"containers"}})
if err != nil {
t.Fatal(err)
}
result := results[0]
containers := defaultValues(t, result, "container")
if len(containers) != len(tests) {
t.Fatalf("containers: got %d, want %d", len(containers), len(tests))
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
row := -1
for i, name := range containers {
if name == tt.name {
row = i
break
}
}
if row < 0 {
t.Fatal("container missing")
}
for column, want := range map[string]any{"cpu_percent": tt.wantCPU, "status": tt.wantStatus, "measurement": tt.measurement, "reporting": "reporting"} {
got := defaultValues(t, result, column)[row]
if number, ok := want.(float64); ok {
if value, ok := got.(float64); ok && math.Abs(value-number) <= 1e-12 {
continue
}
} else if got == want {
continue
}
t.Errorf("%s: got %v, want %v", column, got, want)
}
})
}
}
Loading