From 6b66224f85aa73f9aff67a97a3c5e4ad4ccfb5ce Mon Sep 17 00:00:00 2001 From: Vishal Rana Date: Fri, 16 Oct 2026 20:58:23 -0700 Subject: [PATCH] perf(observability): scan client spans once for external service-map peers The service map's external-peer query reused a clients CTE in two anti-joins. DuckDB inlined it, so each run scanned spans eight times (366 MB on a one-hour OpenTelemetry Demo snapshot). On an 8-vCPU Demo host the panel took 6.5 s at the median and 9.9 s at p95 during dashboard refreshes, at the edge of the 10-second deadline. Materialize the client projection once. On the snapshot the plan has one Parquet scan reading 80 MB, and the warm panel time falls from about 450 to 140 ms. All 45 frame rows match in order. Claude-Session: https://claude.ai/code/session_01XA52nWyQ3R5f5kK8kjpAQW --- internal/observability/topology.go | 4 +- internal/observability/topology_test.go | 91 +++++++++++++++++++++++++ 2 files changed, 94 insertions(+), 1 deletion(-) diff --git a/internal/observability/topology.go b/internal/observability/topology.go index 785eface4..3d06d793d 100644 --- a/internal/observability/topology.go +++ b/internal/observability/topology.go @@ -23,7 +23,9 @@ LIMIT ?` // Uninstrumented peers are visible through client spans; they do not require // another rollup or an integration with the remote service. -const topologyExternalQuery = `WITH clients AS ( +// Materialize the explicit client projection once for the peer and edge +// aggregates, so their anti-joins do not repeat the underlying span scans. +const topologyExternalQuery = `WITH clients AS MATERIALIZED ( SELECT s.service AS caller,s.peer_service AS callee,s.duration_ms, CASE WHEN s.status IN ('STATUS_CODE_ERROR','ERROR') THEN 1.0 ELSE 0.0 END AS error FROM spans s WHERE s.start_time>=?::TIMESTAMP_NS::TIMESTAMPTZ_NS AND s.start_time 1e-12 || math.Abs(node.P95MS-want.P95MS) > 1e-12 { + t.Fatalf("node %d = %+v, want %+v", i, node, want) + } + } + }) + } +} + func TestTopologyExternalPeersIgnoreOutOfWindowParquetSpans(t *testing.T) { start := time.Date(2026, 10, 1, 12, 0, 0, 123, time.UTC) end := start.Add(time.Minute)