From ec725ebd15126da57694cc8e492cdd9ddbcd02a5 Mon Sep 17 00:00:00 2001 From: Vishal Rana Date: Fri, 16 Oct 2026 10:37:10 -0700 Subject: [PATCH] perf(dashboard): scan capacity metrics once and extract versions per resource The hosts and containers panels expanded their shared raw CTE into four Parquet scans each, decoding resource identity for every matching metric row four times. Materialize the CTE once and project only the columns its consumers read. On a one-hour OpenTelemetry Demo snapshot the warm panel time fell from 1,062 to 243 ms (hosts) and from 1,003 to 240 ms (containers). The versions panel extracted five resource fields from every span, log record and metric point in the window. Group each signal by namespace, service and resource with its earliest event first, then extract the fields once per distinct resource. First-seen times and runtime/SDK distinctions are unchanged; warm time fell from 4,164 to 2,277 ms. Frames match the previous SQL on the snapshot within 1e-12. New tests cover first-seen and runtime/SDK tuples, capacity retries, cumulative resets, deltas, namespaces, missing identities and freshness boundaries. Claude-Session: https://claude.ai/code/session_01XA52nWyQ3R5f5kK8kjpAQW --- internal/dashboard/defaults_sql.go | 11 +- internal/dashboard/defaults_test.go | 213 ++++++++++++++++++++++++++++ 2 files changed, 220 insertions(+), 4 deletions(-) diff --git a/internal/dashboard/defaults_sql.go b/internal/dashboard/defaults_sql.go index 6b62899c7..4f5e29972 100644 --- a/internal/dashboard/defaults_sql.go +++ b/internal/dashboard/defaults_sql.go @@ -57,7 +57,8 @@ func capacityCTE(containers bool) string { entity = "to_json([host,container_id])::VARCHAR" priority = "CASE name WHEN 'container.cpu.utilization' THEN 1 WHEN 'container.memory.usage.total' THEN 2 ELSE 3 END" } - return `WITH raw AS (SELECT *,resource['host.name']::VARCHAR AS host,resource['container.id']::VARCHAR AS container_id, + return `WITH raw AS MATERIALIZED (SELECT series_id,ingested_unix_nano,namespace,time,value,type,name,start_time,temporality,unit, + resource['host.name']::VARCHAR AS host,resource['container.id']::VARCHAR AS container_id, resource['container.name']::VARCHAR AS container_name,attributes['state']::VARCHAR AS state, attributes['mountpoint']::VARCHAR AS mountpoint,attributes['device']::VARCHAR AS device FROM metrics WHERE time >= make_timestamp_ns(epoch_ns($__from::TIMESTAMP_NS)-600000000000)::TIMESTAMPTZ_NS AND time < $__to::TIMESTAMP_NS::TIMESTAMPTZ_NS AND name IN (` + names + `)), @@ -106,6 +107,8 @@ func containerCapacitySQL() string { FROM results ORDER BY (reporting<>'reporting') DESC,status DESC NULLS LAST,namespace,host,container` } -const versionObservationsSQL = `SELECT namespace,service,resource['service.version']::VARCHAR AS version,resource['process.runtime.name']::VARCHAR AS runtime_name,resource['process.runtime.version']::VARCHAR AS runtime_version,resource['telemetry.sdk.language']::VARCHAR AS sdk_language,resource['telemetry.sdk.version']::VARCHAR AS sdk_version,start_time AS seen FROM spans WHERE $__window(start_time) - UNION ALL SELECT namespace,service,resource['service.version']::VARCHAR,resource['process.runtime.name']::VARCHAR,resource['process.runtime.version']::VARCHAR,resource['telemetry.sdk.language']::VARCHAR,resource['telemetry.sdk.version']::VARCHAR,time FROM logs WHERE $__window(time) - UNION ALL SELECT namespace,service,resource['service.version']::VARCHAR,resource['process.runtime.name']::VARCHAR,resource['process.runtime.version']::VARCHAR,resource['telemetry.sdk.language']::VARCHAR,resource['telemetry.sdk.version']::VARCHAR,time FROM metrics WHERE $__window(time)` +const versionObservationsSQL = `WITH resources AS MATERIALIZED ( + SELECT namespace,service,resource,min(start_time) AS seen FROM spans WHERE $__window(start_time) GROUP BY namespace,service,resource + UNION ALL SELECT namespace,service,resource,min(time) AS seen FROM logs WHERE $__window(time) GROUP BY namespace,service,resource + UNION ALL SELECT namespace,service,resource,min(time) AS seen FROM metrics WHERE $__window(time) GROUP BY namespace,service,resource) + SELECT namespace,service,resource['service.version']::VARCHAR AS version,resource['process.runtime.name']::VARCHAR AS runtime_name,resource['process.runtime.version']::VARCHAR AS runtime_version,resource['telemetry.sdk.language']::VARCHAR AS sdk_language,resource['telemetry.sdk.version']::VARCHAR AS sdk_version,seen FROM resources` diff --git a/internal/dashboard/defaults_test.go b/internal/dashboard/defaults_test.go index 497eed123..c5a77a8b8 100644 --- a/internal/dashboard/defaults_test.go +++ b/internal/dashboard/defaults_test.go @@ -2,6 +2,7 @@ package dashboard import ( "fmt" + "math" "reflect" "strings" "testing" @@ -201,3 +202,215 @@ func TestDefaultSystemPanelsShowCapacitySilenceVantageAndVersions(t *testing.T) } } } + +func TestDefaultVersionsPreserveFirstSeenAndRuntimeSDKTuples(t *testing.T) { + e, repo := defaultFixture(t) + start := time.Date(2026, 10, 1, 12, 0, 0, 0, time.UTC) + end := start.Add(2 * time.Minute) + base := map[string]any{"service.version": "v1", "process.runtime.name": "Go", "process.runtime.version": "1.27", "telemetry.sdk.language": "go", "telemetry.sdk.version": "1.40"} + keys := []string{"process.runtime.name", "process.runtime.version", "telemetry.sdk.language", "telemetry.sdk.version"} + alternatives := []string{"Other", "1.28", "other", "1.41"} + var spans []telemetry.Span + var logs []telemetry.Log + var metrics []telemetry.Metric + var want [][]any + for i := 0; i < 8; i++ { + resource := make(map[string]any) + for k, v := range base { + resource[k] = v + } + if i >= 1 && i <= 4 { + resource[keys[i-1]] = alternatives[i-1] + } + if i == 5 { + for _, k := range keys { + delete(resource, k) + } + } + namespace, service := "test", "svc" + if i == 6 { + namespace = "other" + } + if i == 7 { + service = "other" + } + first := start.Add(time.Duration(i) * time.Second) + want = append([][]any{{namespace, service, "v1", first.UnixMilli(), resource[keys[0]], resource[keys[1]], resource[keys[2]], resource[keys[3]]}}, want...) + // The earliest signal rotates. Every signal repeats the same resource, + // and an unrelated resource field must not split the final version row. + for j, offset := range []time.Duration{30 * time.Second, 0, 20 * time.Second} { + for signal := 0; signal < 3; signal++ { + at := first.Add(offset + time.Duration((signal+i)%3)*time.Second) + r := resource + if j == 2 { + r = make(map[string]any) + for k, v := range resource { + r[k] = v + } + r["host.name"] = "another host" + } + switch signal { + case 0: + spans = append(spans, telemetry.Span{Namespace: namespace, ServiceName: service, TraceID: fmt.Sprintf("trace-%d-%d", i, j), SpanID: "span", StartUnixNanos: at.UnixNano(), EndUnixNanos: at.Add(time.Millisecond).UnixNano(), Resource: r}) + case 1: + logs = append(logs, telemetry.Log{Namespace: namespace, ServiceName: service, TimeUnixNanos: at.UnixNano(), EventUnixNanos: at.UnixNano(), Resource: r}) + case 2: + m := telemetry.Metric{Namespace: namespace, ServiceName: service, Name: "version", Type: "gauge", Value: 1, TimeUnixNanos: at.UnixNano(), EventUnixNanos: at.UnixNano(), Resource: r} + m.SeriesID = telemetry.MetricSeriesID(m) + metrics = append(metrics, m) + } + } + } + } + // Out-of-window observations must not lower first_seen or add a row. + for _, at := range []time.Time{start.Add(-time.Nanosecond), end} { + resource := base + if at.Equal(end) { + resource = map[string]any{"service.version": "outside"} + } + spans = append(spans, telemetry.Span{Namespace: "test", ServiceName: "svc", TraceID: at.String(), SpanID: "boundary", StartUnixNanos: at.UnixNano(), EndUnixNanos: at.Add(time.Millisecond).UnixNano(), Resource: resource}) + logs = append(logs, telemetry.Log{Namespace: "test", ServiceName: "svc", TimeUnixNanos: at.UnixNano(), EventUnixNanos: at.UnixNano(), Resource: resource}) + m := telemetry.Metric{Namespace: "test", ServiceName: "svc", Name: "version", Type: "gauge", Value: 1, TimeUnixNanos: at.UnixNano(), EventUnixNanos: at.UnixNano(), Resource: resource} + m.SeriesID = telemetry.MetricSeriesID(m) + metrics = append(metrics, m) + } + if err := repo.Commit(t.Context(), telemetrystore.Batch{ID: "version-tuples", Spans: spans, Logs: logs, Metrics: metrics}); err != nil { + t.Fatal(err) + } + spec := DefaultSpec() + spec.Time = panel.Time{From: &start, To: &end} + results, err := e.Run(t.Context(), panel.RunRequest{Dashboard: spec, Panels: []string{"versions"}}) + if err != nil { + t.Fatal(err) + } + result := results[0] + columns := []string{"namespace", "service", "version", "first_seen", "runtime_name", "runtime_version", "sdk_language", "sdk_version"} + for col, name := range columns { + var values []any + for _, row := range want { + values = append(values, row[col]) + } + if got := defaultValues(t, result, name); !reflect.DeepEqual(got, values) { + t.Fatalf("%s: got %v, want %v", name, got, values) + } + } +} + +func TestDefaultCapacityPreservesRetriesCountersNamespacesAndFreshness(t *testing.T) { + e, repo := defaultFixture(t) + start := time.Date(2026, 10, 1, 12, 0, 0, 0, time.UTC) + for i, offset := range []time.Duration{-time.Minute, 0, time.Minute} { + at := start.Add(offset) + var metrics []telemetry.Metric + for _, namespace := range []string{"alpha", "beta"} { + metric := func(name string, value float64, attrs map[string]any) telemetry.Metric { + return telemetry.Metric{Namespace: namespace, Name: name, Type: "gauge", Unit: "1", Value: value, Attributes: attrs, Resource: map[string]any{"host.name": "shared", "container.id": "shared-id"}, TimeUnixNanos: at.UnixNano(), EventUnixNanos: at.UnixNano(), IngestedAt: at.UnixNano()} + } + for _, state := range []string{"idle", "user"} { + // Alpha's cumulative counter resets at the last point. Beta + // reports deltas, so its last interval must use the point value. + value := []float64{100, 160, 10}[i] + if state == "user" { + value = []float64{100, 280, 30}[i] + } + if namespace == "beta" { + value = 30 + if state == "user" { + value = 10 + } + } + m := metric("system.cpu.time", value, map[string]any{"state": state}) + m.Type, m.Monotonic, m.Temporality, m.Unit = "sum", true, "cumulative", "s" + if namespace == "beta" { + m.Temporality, m.StartTimeUnixNanos = "delta", at.Add(-time.Minute).UnixNano() + } + metrics = append(metrics, m) + } + if offset < 0 { + continue + } + used := 80.0 + if namespace == "beta" { + used = 20 + } + metrics = append(metrics, + metric("system.memory.usage", used, map[string]any{"state": "used"}), + metric("system.memory.usage", 100-used, map[string]any{"state": "free"}), + metric("container.cpu.utilization", used/100, nil), + metric("container.memory.usage.total", used, nil), + metric("container.memory.usage.limit", 100, nil)) + // Missing host/container identities must not become extra entities. + missingHost := metric("system.memory.usage", 999, nil) + missingHost.Resource = map[string]any{"container.id": "orphan"} + missingContainer := metric("container.cpu.utilization", 999, nil) + missingContainer.Resource = map[string]any{"host.name": "shared"} + metrics = append(metrics, missingHost, missingContainer) + } + for j := range metrics { + metrics[j].SeriesID = telemetry.MetricSeriesID(metrics[j]) + } + // Retry copies live in another immutable file. Older ingestions lose + // even with a larger value; equal ingestions use descending value. + var retries, tiedPoints []telemetry.Metric + for _, m := range metrics { + older := m + older.Value = 9999 + tied := m + tied.Value = 0 + retries = append(retries, older) + tiedPoints = append(tiedPoints, tied) + } + // Commit assigns ingestion timestamps per batch, so publish old copies + // first and publish equal-ingestion copies with the real points. + for j, points := range [][]telemetry.Metric{retries, append(metrics, tiedPoints...)} { + if err := repo.Commit(t.Context(), telemetrystore.Batch{ID: fmt.Sprintf("capacity-%d-%d", i, j), Metrics: points}); err != nil { + t.Fatal(err) + } + } + } + for _, offset := range []time.Duration{2 * time.Minute, 3*time.Minute - time.Nanosecond, 3 * time.Minute, 6 * time.Minute, 6*time.Minute + time.Nanosecond} { + end := start.Add(offset) + spec := DefaultSpec() + spec.Time = panel.Time{From: &start, To: &end} + results, err := e.Run(t.Context(), panel.RunRequest{Dashboard: spec, Panels: []string{"hosts", "containers"}}) + if err != nil { + t.Fatal(err) + } + for _, result := range results { + if got := defaultValues(t, result, "namespace"); !reflect.DeepEqual(got, []any{"alpha", "beta"}) { + t.Fatalf("%s at %s: namespaces %v", result.ID, offset, got) + } + state := "reporting" + if offset >= 3*time.Minute { + state = "not reporting" + if got := defaultValues(t, result, "status"); !reflect.DeepEqual(got, []any{100.0, 100.0}) { + t.Fatalf("%s at %s: status %v", result.ID, offset, got) + } + } + if got := defaultValues(t, result, "reporting"); !reflect.DeepEqual(got, []any{state, state}) { + t.Fatalf("%s at %s: reporting %v", result.ID, offset, got) + } + cpu := []any{75.0, 25.0} + if result.ID == "containers" { + cpu = []any{80.0, 20.0} + } + memory := []any{80.0, 20.0} + if offset > 6*time.Minute { + cpu, memory = []any{nil, nil}, []any{nil, nil} + } + for column, want := range map[string][]any{"cpu_percent": cpu, "memory_percent": memory} { + got := defaultValues(t, result, column) + for i, expected := range want { + if number, ok := expected.(float64); ok { + if value, ok := got[i].(float64); ok && math.Abs(value-number) <= 1e-12 { + continue + } + } else if got[i] == expected { + continue + } + t.Fatalf("%s at %s: %s got %v, want %v", result.ID, offset, column, got, want) + } + } + } + } +}