From 0500b42fcbb0bffc5354ef9256c49d4b652a8fd3 Mon Sep 17 00:00:00 2001 From: Vishal Rana Date: Thu, 5 Nov 2026 23:41:30 -0800 Subject: [PATCH 1/3] fix(query): use trace indexes for service exemplars The bounded candidate IDs still made root lookup scan retained spans. Use shared completed trace candidates and indexed root metadata for exact service and namespace selections. Rank errors before scoped trace duration and detect truncation with a 21st distinct trace. Keep other filters and selected buckets on generic SQL. --- internal/observability/service_test.go | 4 +- internal/observability/trace.go | 19 +-- internal/query/exemplar_bounds_test.go | 8 +- internal/query/exemplar_index_test.go | 179 +++++++++++++++++++++++++ internal/query/exemplars.go | 101 ++++++++++++++ internal/queryrows/trace_candidates.go | 21 +++ internal/telemetry/parquet.go | 25 +++- 7 files changed, 335 insertions(+), 22 deletions(-) create mode 100644 internal/query/exemplar_index_test.go create mode 100644 internal/queryrows/trace_candidates.go diff --git a/internal/observability/service_test.go b/internal/observability/service_test.go index a0d6c3bc2..322980e89 100644 --- a/internal/observability/service_test.go +++ b/internal/observability/service_test.go @@ -186,8 +186,8 @@ func TestTraceSelectsRecentErrorAndCorrelatesLogs(t *testing.T) { svc, mock, repository := newMockService(t) start := time.Date(2026, 7, 20, 11, 0, 0, 0, time.UTC) end := start.Add(time.Hour) - mock.ExpectQuery(regexp.QuoteMeta(completedTraceQuery)). - WithArgs(start, end, "prod", "prod", "checkout", "checkout"). + mock.ExpectQuery(regexp.QuoteMeta(queryrows.CompletedTraceQuery)). + WithArgs(start, end, "prod", "prod", "checkout", "checkout", 1). WillReturnRows(sqlmock.NewRows([]string{"trace_id"}).AddRow("trace-1")) if err := repository.Commit(context.Background(), telemetrystore.Batch{ID: "trace-fixture", Spans: []telemetry.Span{ {Namespace: "prod", TraceID: "trace-1", SpanID: "root", ServiceName: "checkout", Name: "POST /pay", Kind: "SERVER", StartUnixNanos: start.UnixNano(), DurationMS: 200, StatusCode: "STATUS_CODE_ERROR", StatusMsg: "declined"}, diff --git a/internal/observability/trace.go b/internal/observability/trace.go index efb691020..3a924df73 100644 --- a/internal/observability/trace.go +++ b/internal/observability/trace.go @@ -31,7 +31,7 @@ func (s *Service) Trace(ctx context.Context, scope Scope, traceID, service strin ctx = queryrows.WithWindow(ctx, queryrows.Window{Start: scope.Start, End: scope.End, Namespace: scope.Namespace, Service: service}) if traceID == "" { candidateCtx := queryrows.WithWindow(ctx, queryrows.Window{Start: scope.Start, End: scope.End, Namespace: scope.Namespace, Service: service, Kind: queryrows.TraceCandidateRead}) - rows, queryErr := s.db.QueryContext(candidateCtx, completedTraceQuery, scope.Start, scope.End, scope.Namespace, scope.Namespace, service, service) + rows, queryErr := s.db.QueryContext(candidateCtx, queryrows.CompletedTraceQuery, scope.Start, scope.End, scope.Namespace, scope.Namespace, service, service, 1) if queryErr != nil { return Result[TraceDetail]{}, fmt.Errorf("query recent trace: %w", queryErr) } @@ -126,20 +126,3 @@ func (s *Service) traceLogsFromParquet(ctx context.Context, scope Scope, traceID rows.Close() return logs, nil } - -const completedTraceQuery = `WITH tail AS ( - SELECT trace_id,min(start_unix_nano) AS min_start,max(end_unix_nano) AS max_end, - max(CASE WHEN status = 'STATUS_CODE_ERROR' THEN 1 ELSE 0 END) AS has_error - FROM trace_tail - WHERE start_time>=$1::TIMESTAMP_NS::TIMESTAMPTZ_NS AND start_time<$2::TIMESTAMP_NS::TIMESTAMPTZ_NS - AND ($3='' OR namespace=$4) AND ($5='' OR service=$6) AND trace_id<>'' GROUP BY trace_id -), untouched AS ( - SELECT trace_id,min_start,max_end,has_error FROM trace_candidates - WHERE trace_id NOT IN (SELECT trace_id FROM tail) - ORDER BY has_error DESC,max_end-min_start DESC,trace_id LIMIT 1 -), touched AS ( - SELECT t.trace_id, least(t.min_start,c.min_start) AS min_start,greatest(t.max_end,c.max_end) AS max_end, - greatest(t.has_error,c.has_error) AS has_error - FROM tail t LEFT JOIN trace_candidates c ON t.trace_id=c.trace_id -), candidates AS (SELECT * FROM untouched UNION ALL SELECT * FROM touched) -SELECT trace_id FROM candidates ORDER BY has_error DESC,max_end-min_start DESC,trace_id LIMIT 1` diff --git a/internal/query/exemplar_bounds_test.go b/internal/query/exemplar_bounds_test.go index a4c54caae..17569bcdc 100644 --- a/internal/query/exemplar_bounds_test.go +++ b/internal/query/exemplar_bounds_test.go @@ -22,6 +22,12 @@ func (e exemplarProbeEngine) QueryContext(ctx context.Context, text string, args return e.query(ctx, text, args...) } +func (e exemplarProbeEngine) Trace(ctx context.Context, q telemetry.TraceQuery) ([]telemetry.IndexedSpan, telemetry.TraceTotals, error) { + return e.Engine.(interface { + Trace(context.Context, telemetry.TraceQuery) ([]telemetry.IndexedSpan, telemetry.TraceTotals, error) + }).Trace(ctx, q) +} + func exemplarDashboard(from string) Dashboard { return Dashboard{Name: "Selection", Time: Time{Range: "1h"}, Panels: []Panel{{ID: "p", Title: "P", Viz: "bar", Query: &Spec{From: from, Measures: []string{"count()"}, By: []string{"service"}}}}} } @@ -378,7 +384,7 @@ func TestExemplarSampleKeepsRootOrderingAndTraceErrors(t *testing.T) { req.From, req.To = fixtureStart.Add(time.Minute), fixtureStart.Add(2*time.Minute) req.Dimensions = map[string]string{"service": "selected"} got, err := e.Exemplars(t.Context(), req) - if err != nil || len(got.Traces) != 1 || !got.Truncated { + if err != nil || len(got.Traces) != 1 || got.Truncated { t.Fatalf("sample: %+v %v", got, err) } root := got.Traces[0] diff --git a/internal/query/exemplar_index_test.go b/internal/query/exemplar_index_test.go new file mode 100644 index 000000000..d4c8674d9 --- /dev/null +++ b/internal/query/exemplar_index_test.go @@ -0,0 +1,179 @@ +package query + +import ( + "context" + "fmt" + "github.com/labstack/fanout/internal/queryrows" + "github.com/labstack/fanout/internal/telemetry" + telemetrystore "github.com/labstack/fanout/internal/telemetry/store" + "reflect" + "testing" + "time" +) + +func TestIndexedExemplarsRankTracesAndTruncate(t *testing.T) { + duck, repo := newTestEngine(t) + var spans []telemetry.Span + for i := range 21 { + n := fixtureStart.Add(time.Minute).UnixNano() + int64(i) + status := "STATUS_CODE_OK" + if i == 0 { + status = "STATUS_CODE_ERROR" + } + spans = append(spans, telemetry.Span{Namespace: "shop", ServiceName: "checkout", TraceID: fmt.Sprintf("trace-%02d", i), SpanID: "root", Name: "root", StartUnixNanos: n, EndUnixNanos: n + int64(i+1)*1000000, DurationMS: float64(i + 1), StatusCode: status}) + } + // Rank a long scoped trace ahead of a trace with one slower span. + late := fixtureStart.Add(45 * time.Minute).UnixNano() + spans = append(spans, telemetry.Span{Namespace: "shop", ServiceName: "checkout", TraceID: "trace-01", SpanID: "late", ParentSpanID: "root", StartUnixNanos: late, EndUnixNanos: late + 1000000, DurationMS: 1}) + // A much longer trace in another namespace and a different service must not win. + for _, s := range []struct{ ns, service string }{{"other", "checkout"}, {"shop", "other"}} { + id := s.ns + s.service + if s.ns == "other" { + id = "trace-20" + } + spans = append(spans, telemetry.Span{Namespace: s.ns, ServiceName: s.service, TraceID: id, SpanID: "root", StartUnixNanos: fixtureStart.UnixNano(), EndUnixNanos: fixtureStart.Add(time.Hour).UnixNano(), DurationMS: 3600000, StatusCode: "STATUS_CODE_ERROR"}) + } + commit(t, repo, spans, nil) + if _, err := duck.RefreshReadCaches(t.Context()); err != nil { + t.Fatal(err) + } + queries := 0 + probe := exemplarProbeEngine{Engine: duck, query: func(ctx context.Context, text string, args ...any) (queryrows.Rows, error) { + queries++ + w, ok := queryrows.ReadWindow(ctx) + if text != queryrows.CompletedTraceQuery || !ok || w.Kind != queryrows.TraceCandidateRead || w.Namespace != "shop" || w.Service != "checkout" || args[len(args)-1] != 21 { + t.Fatalf("unbounded candidate read: %s window=%+v args=%v", text, w, args) + } + return duck.QueryContext(ctx, text, args...) + }} + e := NewExecutor(probe, 30) + defer e.Close() + e.now = func() time.Time { return fixtureStart.Add(time.Hour) } + d := exemplarDashboard("spans") + d.Variables = []Variable{{Name: "service", Kind: "custom", Options: []string{"checkout", "other"}, IncludeAll: true, Default: AllValue}} + d.Panels[0].Query.Where = []string{"namespace = 'shop'", "service = $service"} + req := exemplarRequest(d) + req.Dimensions = map[string]string{"service": "checkout"} + req.Vars = map[string]Value{"service": {All: true}} + got, err := e.exemplars(t.Context(), req, e.now()) + if err != nil { + t.Fatal(err) + } + if queries != 1 { + t.Fatalf("candidate statements=%d", queries) + } + if len(got.Traces) != 20 || !got.Truncated || got.Traces[0].TraceID != "trace-00" || got.Traces[1].TraceID != "trace-01" || got.Traces[2].TraceID != "trace-20" { + t.Fatalf("ranking/cap: %+v", got) + } + for _, trace := range got.Traces { + if trace.Namespace != "shop" || trace.Service != "checkout" { + t.Fatalf("scope escaped: %+v", trace) + } + } + // Exactly 20 distinct traces is complete. + req.From = fixtureStart.Add(time.Minute + time.Nanosecond) + got, err = e.exemplars(t.Context(), req, e.now()) + if err != nil || len(got.Traces) != 20 || got.Truncated { + t.Fatalf("exact cap: %+v %v", got, err) + } + req.Dashboard.Panels[0].Query.Where = append(req.Dashboard.Panels[0].Query.Where, "service = 'other'") + got, err = e.exemplars(t.Context(), req, e.now()) + if err != nil || len(got.Traces) != 0 { + t.Fatalf("conflict escaped: %+v %v", got, err) + } +} + +func TestIndexedExemplarsSeeTailCompactionAndRetirement(t *testing.T) { + duck, repo := newTestEngine(t) + // Fresh ingest stamps let the compactor select these small batches. + at := time.Now().UTC().Add(-time.Minute) + start, end := at.Add(-time.Minute), at.Add(time.Minute) + for i := range 8 { + n := at.UnixNano() + int64(i)*1000000 + if err := repo.Commit(t.Context(), telemetrystore.Batch{ID: fmt.Sprintf("index-%d", i), Spans: []telemetry.Span{{Namespace: "shop", ServiceName: "checkout", TraceID: "shared", SpanID: fmt.Sprint(i), ParentSpanID: "missing", StartUnixNanos: n, EndUnixNanos: n + 1000000, DurationMS: 1, IngestedAt: n}}}); err != nil { + t.Fatal(err) + } + } + if _, err := duck.RefreshReadCaches(t.Context()); err != nil { + t.Fatal(err) + } + // The parentless root sorts after the children and is outside a Limit:1 page. + n := at.Add(time.Second).UnixNano() + if err := repo.Commit(t.Context(), telemetrystore.Batch{ID: "tail", Spans: []telemetry.Span{ + {Namespace: "shop", ServiceName: "frontend", TraceID: "shared", SpanID: "root", Name: "real root", StartUnixNanos: n, EndUnixNanos: n + 10000000, DurationMS: 10, IngestedAt: n}, + {Namespace: "shop", ServiceName: "checkout", TraceID: "new-error", SpanID: "root", StartUnixNanos: n, EndUnixNanos: n + 1000000, DurationMS: 1, StatusCode: "STATUS_CODE_ERROR", IngestedAt: n}, + }}); err != nil { + t.Fatal(err) + } + e := NewExecutor(duck, 30) + defer e.Close() + d := Dashboard{Name: "Indexed", Time: Time{From: &start, To: &end}, Panels: []Panel{{ID: "p", Title: "Services", Viz: "service_map", Query: &Spec{From: "spans", Where: []string{"namespace = 'shop'"}}}}} + req := ExemplarRequest{Dashboard: d, PanelID: "p", From: start, To: end, Dimensions: map[string]string{"service": "checkout"}} + read := func() ExemplarResponse { + t.Helper() + got, err := e.exemplars(t.Context(), req, end) + if err != nil { + t.Fatal(err) + } + return got + } + want := read() + if len(want.Traces) != 2 || want.Traces[0].TraceID != "new-error" || want.Traces[1].Operation != "real root" || want.Traces[1].Service != "frontend" || want.Truncated { + t.Fatalf("tail/root: %+v", want) + } + if _, err := duck.RefreshReadCaches(t.Context()); err != nil { + t.Fatal(err) + } + if got := read(); !reflect.DeepEqual(got, want) { + t.Fatalf("cache changed results: %+v want %+v", got, want) + } + if n, err := repo.CompactParquet(t.Context(), duck, 8); err != nil || n != 8 { + t.Fatalf("compact %d: %v", n, err) + } + if got := read(); !reflect.DeepEqual(got, want) { + t.Fatalf("compaction changed results: %+v want %+v", got, want) + } + if _, err := duck.RefreshReadCaches(t.Context()); err != nil { + t.Fatal(err) + } + if got := read(); !reflect.DeepEqual(got, want) { + t.Fatalf("retired cache changed results: %+v", got) + } + if n, err := repo.PruneParquet(t.Context(), duck, end.Add(time.Minute).UnixNano(), 8); err != nil || n == 0 { + t.Fatalf("prune %d: %v", n, err) + } + if got := read(); len(got.Traces) != 0 { + t.Fatalf("retired traces visible: %+v", got) + } +} + +func TestExemplarIndexEligibility(t *testing.T) { + e := newFixtureExecutor(t) + for _, tc := range []struct { + where []string + dimensions map[string]string + bucket *SelectionBucket + want bool + }{ + {nil, map[string]string{"service": "checkout"}, nil, true}, + {[]string{"service = 'checkout'"}, nil, nil, true}, + {[]string{"namespace = 'shop'"}, map[string]string{"service": "checkout"}, nil, true}, + {[]string{"status = 'STATUS_CODE_ERROR'"}, map[string]string{"service": "checkout"}, nil, false}, + {[]string{"lower(service) = 'checkout'"}, nil, nil, false}, + {nil, map[string]string{"service": "checkout"}, &SelectionBucket{Lower: 1}, false}, + {nil, map[string]string{"operation": "GET"}, nil, false}, + {nil, nil, nil, false}, + } { + d := exemplarDashboard("spans") + d.Panels[0].Query.Where = tc.where + Normalize(&d) + checked, err := e.check(t.Context(), &d) + if err != nil { + t.Fatal(err) + } + _, got, _, err := e.exemplarTraceScope(t.Context(), &d.Panels[0], checked.Filters["p"], Scope{}, tc.dimensions, tc.bucket) + if err != nil || got != tc.want { + t.Fatalf("where=%v dimensions=%v eligible=%v err=%v", tc.where, tc.dimensions, got, err) + } + } +} diff --git a/internal/query/exemplars.go b/internal/query/exemplars.go index 0e9036b27..7ed709f3c 100644 --- a/internal/query/exemplars.go +++ b/internal/query/exemplars.go @@ -11,6 +11,7 @@ import ( "time" "github.com/labstack/fanout/internal/queryrows" + "github.com/labstack/fanout/internal/telemetry" ) type SelectionBucket struct { @@ -214,6 +215,17 @@ func (e *Executor) exemplars(ctx context.Context, req ExemplarRequest, now time. return out, err } + indexedScope, indexed, empty, err := e.exemplarTraceScope(ctx, p, checked.Filters[p.ID], Scope{Start: lo, End: hi, Vars: vars}, req.Dimensions, req.Bucket) + if err != nil { + return out, err + } + if indexed { + if empty { + return out, nil + } + return e.indexedExemplars(ctx, indexedScope, start, end) + } + if p.Query.From == "logs" && p.Options != nil && p.Options.Highlight != "" { where += " AND contains(lower(body),lower(?))" args = append(args, p.Options.Highlight) @@ -333,3 +345,92 @@ func exemplarError(ctx context.Context, err error) error { func outsideSelection() error { return Problems{{Path: "from", Message: "selection does not overlap the panel window"}} } + +// Only exact namespace/service equalities can use trace candidates. Every +// remaining selection keeps its generic span/log predicates. +func (e *Executor) exemplarTraceScope(ctx context.Context, p *Panel, filters []Filter, scope Scope, dimensions map[string]string, bucket *SelectionBucket) (queryrows.Window, bool, bool, error) { + w := queryrows.Window{Start: scope.Start, End: scope.End, Kind: queryrows.TraceCandidateRead} + if p.Query.From != "spans" || bucket != nil { + return w, false, false, nil + } + values := map[string]string{} + empty := false + add := func(field, value string) { + if prior, ok := values[field]; ok && prior != value { + empty = true + } + values[field] = value + } + for field, value := range dimensions { + if (field != "namespace" && field != "service") || value == "" { + return w, false, false, nil + } + add(field, value) + } + for _, f := range filters { + if scope.dropped(f) { + continue + } + field, value, err := rollupFilterValue(ctx, e.engine, f, scope) + if err != nil { + if isOperational(err) { + return w, false, false, err + } + return w, false, false, nil + } + add(field, value) + } + w.Namespace, w.Service = values["namespace"], values["service"] + return w, w.Service != "", empty, nil +} + +func (e *Executor) indexedExemplars(ctx context.Context, w queryrows.Window, panelStart, panelEnd time.Time) (ExemplarResponse, error) { + out := ExemplarResponse{Traces: []Exemplar{}} + rows, err := e.engine.QueryContext(queryrows.WithWindow(ctx, w), queryrows.CompletedTraceQuery, w.Start, w.End, w.Namespace, w.Namespace, w.Service, w.Service, 21) + if err != nil { + return out, exemplarError(ctx, err) + } + var ids []string + for rows.Next() { + var id string + if err := rows.Scan(&id); err != nil { + rows.Close() + return out, exemplarError(ctx, err) + } + ids = append(ids, id) + } + err = rows.Err() + rows.Close() + if err != nil { + return out, exemplarError(ctx, err) + } + if len(ids) == 0 { + return out, nil + } + out.Truncated = len(ids) > 20 + if out.Truncated { + ids = ids[:20] + } + reader, ok := e.engine.(interface { + Trace(context.Context, telemetry.TraceQuery) ([]telemetry.IndexedSpan, telemetry.TraceTotals, error) + }) + if !ok { + return out, fmt.Errorf("indexed trace reader unavailable") + } + for _, id := range ids { + _, totals, err := reader.Trace(ctx, telemetry.TraceQuery{TraceID: id, Namespace: w.Namespace, StartNanos: panelStart.UnixNano(), EndNanos: panelEnd.UnixNano(), Limit: 1}) + if err != nil { + return out, exemplarError(ctx, err) + } + if totals.Root == nil { + continue + } + root := totals.Root + status := root.StatusCode + if totals.HasError { + status = "STATUS_CODE_ERROR" + } + out.Traces = append(out.Traces, Exemplar{TraceID: id, Namespace: root.Namespace, Service: root.ServiceName, Operation: root.Name, DurationMS: root.DurationMS, Status: status, Start: time.Unix(0, root.StartUnixNanos).UTC()}) + } + return out, nil +} diff --git a/internal/queryrows/trace_candidates.go b/internal/queryrows/trace_candidates.go new file mode 100644 index 000000000..99f94f8ee --- /dev/null +++ b/internal/queryrows/trace_candidates.go @@ -0,0 +1,21 @@ +package queryrows + +// CompletedTraceQuery ranks scoped traces from completed batch candidates and +// the uncached/boundary tail. Arguments are start, end, namespace twice, service +// twice, and limit. Both top-N stages must use the same limit. +const CompletedTraceQuery = `WITH tail AS ( + SELECT trace_id,min(start_unix_nano) AS min_start,max(end_unix_nano) AS max_end, + max(CASE WHEN status = 'STATUS_CODE_ERROR' THEN 1 ELSE 0 END) AS has_error + FROM trace_tail + WHERE start_time>=$1::TIMESTAMP_NS::TIMESTAMPTZ_NS AND start_time<$2::TIMESTAMP_NS::TIMESTAMPTZ_NS + AND ($3='' OR namespace=$4) AND ($5='' OR service=$6) AND trace_id<>'' GROUP BY trace_id +), untouched AS ( + SELECT trace_id,min_start,max_end,has_error FROM trace_candidates + WHERE trace_id NOT IN (SELECT trace_id FROM tail) + ORDER BY has_error DESC,max_end-min_start DESC,trace_id LIMIT $7 +), touched AS ( + SELECT t.trace_id, least(t.min_start,c.min_start) AS min_start,greatest(t.max_end,c.max_end) AS max_end, + greatest(t.has_error,c.has_error) AS has_error + FROM tail t LEFT JOIN trace_candidates c ON t.trace_id=c.trace_id +), candidates AS (SELECT * FROM untouched UNION ALL SELECT * FROM touched) +SELECT trace_id FROM candidates ORDER BY has_error DESC,max_end-min_start DESC,trace_id LIMIT $7` diff --git a/internal/telemetry/parquet.go b/internal/telemetry/parquet.go index a32775fc7..b67e7a182 100644 --- a/internal/telemetry/parquet.go +++ b/internal/telemetry/parquet.go @@ -506,6 +506,9 @@ func (p *ParquetStore) RestoreRetiredInputs(inputs []string, replacementID strin // bounded by Limit. Deriving them here removes a second pass over every batch // file -- the thing that made trace_detail pay for its own correctness. type TraceTotals struct { + // Root prefers a parentless span, then start time and span ID, across all + // matching indexed ranges, independently of the detail page limit. + Root *IndexedSpan Spans int Services int DurationMS float64 @@ -645,7 +648,14 @@ func readIndexedTrace(ctx context.Context, batch *storedBatch, match traceRange, if end := row.StartUnixNano + int64(row.DurationMS*float64(time.Millisecond)); end > totals.MaxEndNano { totals.MaxEndNano = end } - selected.Add(row.span(), query.Limit) + span := row.span() + if totals.Root == nil { + totals.Root = new(IndexedSpan) + } + if totals.Spans == 1 || indexedRootEarlier(span, *totals.Root) { + *totals.Root = span + } + selected.Add(span, query.Limit) } remaining -= uint64(n) } @@ -680,6 +690,19 @@ func (h *indexedSpanHeap) Add(span IndexedSpan, limit int) { } } +func indexedRootEarlier(left, right IndexedSpan) bool { + if (left.ParentSpanID == "") != (right.ParentSpanID == "") { + return left.ParentSpanID == "" + } + if left.StartUnixNanos != right.StartUnixNanos { + return left.StartUnixNanos < right.StartUnixNanos + } + if left.SpanID != right.SpanID { + return left.SpanID < right.SpanID + } + return left.Namespace < right.Namespace +} + func indexedSpanEarlier(left, right IndexedSpan) bool { if left.StartUnixNanos != right.StartUnixNanos { return left.StartUnixNanos < right.StartUnixNanos From 47683422a17cbfbb8f2604cdcbc602c7a144a8b3 Mon Sep 17 00:00:00 2001 From: Vishal Rana Date: Fri, 6 Nov 2026 00:24:51 -0800 Subject: [PATCH 2/3] fix(query): isolate indexed exemplars by namespace Key completed candidates and tail joins by namespace and trace ID, including cache updates and retention repair. Read exemplar roots in one indexed batch pass and reuse checked filter trees. Use selected-scope errors consistently and mark missing roots truncated. --- internal/engine/batch_cache.go | 39 +++-- internal/engine/duck.go | 7 + internal/observability/batch_reads_test.go | 4 +- internal/observability/service_test.go | 2 +- internal/observability/trace.go | 2 +- internal/observability/trace_test.go | 25 +++ internal/query/check.go | 2 +- internal/query/deploy_split.go | 2 +- internal/query/exemplar_bounds_test.go | 12 +- internal/query/exemplar_index_test.go | 116 +++++++++++++- internal/query/exemplars.go | 38 +++-- internal/query/filter.go | 2 + internal/query/rollup_panels.go | 14 +- internal/query/rollup_panels_test.go | 16 +- internal/queryrows/trace_candidates.go | 19 +-- internal/telemetry/parquet.go | 172 ++++++++++++++++----- internal/telemetry/trace_index.go | 4 + 17 files changed, 372 insertions(+), 104 deletions(-) diff --git a/internal/engine/batch_cache.go b/internal/engine/batch_cache.go index 8c7387e4a..3889722ea 100644 --- a/internal/engine/batch_cache.go +++ b/internal/engine/batch_cache.go @@ -25,7 +25,7 @@ import ( // anomaly_log is not one of them: it holds detector output that nothing can // rebuild, so the drop keeps it. Its table is created once; any change to its // schema must be a separate, explicit migration step. -const readCacheVersion = 17 +const readCacheVersion = 18 const ( readCacheBatchLimit = 64 @@ -280,7 +280,7 @@ func (d *Duck) cacheBatches(ctx context.Context, batches []telemetry.BatchMetada return err } if len(changed) > 0 { - if err := exec(`CREATE OR REPLACE TEMP TABLE cache_retired_traces AS SELECT DISTINCT trace_id FROM read_trace_parts WHERE batch_id IN (` + idsSQL(changed) + `)`); err != nil { + if err := exec(`CREATE OR REPLACE TEMP TABLE cache_retired_traces AS SELECT DISTINCT namespace,trace_id FROM read_trace_parts WHERE batch_id IN (` + idsSQL(changed) + `)`); err != nil { return 0, err } } @@ -294,10 +294,10 @@ func (d *Duck) cacheBatches(ctx context.Context, batches []telemetry.BatchMetada if len(changed) > 0 { // Retention can change a trace's extrema or error flag. Compaction whose // complete replacement is cached preserves them and bypasses this repair. - if err := exec(`DELETE FROM read_trace_candidates WHERE trace_id IN (SELECT trace_id FROM cache_retired_traces) AND trace_id NOT IN (SELECT trace_id FROM read_trace_parts)`); err != nil { + if err := exec(`DELETE FROM read_trace_candidates WHERE (namespace,trace_id) IN (SELECT namespace,trace_id FROM cache_retired_traces) AND (namespace,trace_id) NOT IN (SELECT namespace,trace_id FROM read_trace_parts)`); err != nil { return 0, err } - if err := exec(`MERGE INTO read_trace_candidates c USING (` + traceCandidateSelect + ` FROM read_trace_parts WHERE trace_id IN (SELECT trace_id FROM cache_retired_traces) GROUP BY trace_id) AS excluded ON c.trace_id=excluded.trace_id + if err := exec(`MERGE INTO read_trace_candidates c USING (` + traceCandidateSelect + ` FROM read_trace_parts WHERE (namespace,trace_id) IN (SELECT namespace,trace_id FROM cache_retired_traces) GROUP BY namespace,trace_id) AS excluded ON c.namespace=excluded.namespace AND c.trace_id=excluded.trace_id WHEN MATCHED THEN UPDATE SET namespace=excluded.namespace,service=excluded.service,min_start=excluded.min_start,max_start=excluded.max_start,max_end=excluded.max_end,has_error=excluded.has_error WHEN NOT MATCHED THEN INSERT ` + traceCandidateColumns); err != nil { return 0, err @@ -359,10 +359,9 @@ func (d *Duck) cacheBatches(ctx context.Context, batches []telemetry.BatchMetada if err := exec(`INSERT INTO read_trace_parts SELECT * FROM cache_new_trace_parts`); err != nil { return 0, err } - candidates := traceCandidateSelect + ` FROM cache_new_trace_parts GROUP BY trace_id` - if err := exec(`MERGE INTO read_trace_candidates c USING (` + candidates + `) AS excluded ON c.trace_id=excluded.trace_id + candidates := traceCandidateSelect + ` FROM cache_new_trace_parts GROUP BY namespace,trace_id` + if err := exec(`MERGE INTO read_trace_candidates c USING (` + candidates + `) AS excluded ON c.namespace=excluded.namespace AND c.trace_id=excluded.trace_id WHEN MATCHED THEN UPDATE SET - namespace=CASE WHEN c.namespace=excluded.namespace THEN excluded.namespace END, service=CASE WHEN c.service=excluded.service THEN excluded.service END, min_start=least(c.min_start,excluded.min_start), max_start=greatest(c.max_start,excluded.max_start), @@ -466,18 +465,18 @@ func (d *Duck) aggregateSources(ctx context.Context, db snapshotSQL, batches []t } interior := fmt.Sprintf("min_start>=%d AND max_start<%d", w.Start.UnixNano(), w.End.UnixNano()) if len(partialFiles) > 0 { - tail += " UNION ALL SELECT " + projection + " FROM (" + cleanSource("spans", d.snapshotSource("spans", partialFiles, batches)) + ") WHERE " + windowPredicate("start_time", w) + " AND trace_id IN (SELECT trace_id FROM (" + partial + "))" + tail += " UNION ALL SELECT " + projection + " FROM (" + cleanSource("spans", d.snapshotSource("spans", partialFiles, batches)) + ") WHERE " + windowPredicate("start_time", w) + " AND (namespace,trace_id) IN (SELECT namespace,trace_id FROM (" + partial + "))" // A partial trace may also have in-window contributions in other batches. // Include these parts as candidates; boundary rows merge them in the kernel. - sources["trace_candidates"] = `SELECT trace_id,min(min_start) AS min_start,max(max_end) AS max_end,max(has_error) AS has_error FROM read_trace_parts WHERE ` + scope + ` AND ` + interior + ` GROUP BY trace_id` + sources["trace_candidates"] = `SELECT namespace,trace_id,min(min_start) AS min_start,max(max_end) AS max_end,max(has_error) AS has_error FROM read_trace_parts WHERE ` + scope + ` AND ` + interior + ` GROUP BY namespace,trace_id` } else { // Every overlapping cached file lies inside the window, so a partial // trace's other spans are in files outside it. Its in-window parts are // still a candidate; the kernel needs one row per trace, and partial // traces are disjoint from the fully contained index rows. - sources["trace_candidates"] = "SELECT trace_id,min_start,max_end,has_error FROM (" + index + ") WHERE " + full + - ` UNION ALL SELECT trace_id,min(min_start),max(max_end),max(has_error) FROM read_trace_parts WHERE ` + scope + ` AND ` + interior + - ` AND trace_id IN (SELECT trace_id FROM (` + partial + `)) GROUP BY trace_id` + sources["trace_candidates"] = "SELECT namespace,trace_id,min_start,max_end,has_error FROM (" + index + ") WHERE " + full + + ` UNION ALL SELECT namespace,trace_id,min(min_start),max(max_end),max(has_error) FROM read_trace_parts WHERE ` + scope + ` AND ` + interior + + ` AND (namespace,trace_id) IN (SELECT namespace,trace_id FROM (` + partial + `)) GROUP BY namespace,trace_id` } sources["trace_tail"] = tail return nil @@ -594,10 +593,10 @@ func (d *Duck) transferBatchCacheLocked(ctx context.Context, output string, inpu // million traces that alone outgrew a 1 GB memory limit. const traceCandidateColumns = `VALUES (excluded.trace_id,excluded.namespace,excluded.service,excluded.min_start,excluded.max_start,excluded.max_end,excluded.has_error)` -// One candidate row per trace is sufficient while the requested namespace and -// service are unambiguous. Mixed scopes are NULL and use exact batch parts. +// One candidate per namespace and trace. Mixed services are NULL and use exact +// batch parts for a service selection. const traceCandidateSelect = `SELECT trace_id, - CASE WHEN min(namespace)=max(namespace) THEN min(namespace) END AS namespace, + namespace, CASE WHEN min(service)=max(service) THEN min(service) END AS service, min(min_start) AS min_start,max(max_start) AS max_start,max(max_end) AS max_end,max(has_error) AS has_error` @@ -666,8 +665,8 @@ func (d *Duck) traceCandidateSource(ctx context.Context, db snapshotSQL, batches } index := `SELECT * FROM read_trace_candidates` if len(changed) > 0 { - dirty := `SELECT trace_id FROM read_trace_parts WHERE batch_id IN (` + idsSQL(changed) + `)` - index += ` WHERE trace_id NOT IN (` + dirty + `) UNION ALL ` + traceCandidateSelect + ` FROM read_trace_parts WHERE batch_id IN (` + idsSQL(cachedIDs) + `) AND trace_id IN (` + dirty + `) GROUP BY trace_id` + dirty := `SELECT namespace,trace_id FROM read_trace_parts WHERE batch_id IN (` + idsSQL(changed) + `)` + index += ` WHERE (namespace,trace_id) NOT IN (` + dirty + `) UNION ALL ` + traceCandidateSelect + ` FROM read_trace_parts WHERE batch_id IN (` + idsSQL(cachedIDs) + `) AND (namespace,trace_id) IN (` + dirty + `) GROUP BY namespace,trace_id` } var exact, possible, ambiguous []string for _, field := range []struct{ name, value string }{{"namespace", w.Namespace}, {"service", w.Service}} { @@ -679,12 +678,12 @@ func (d *Duck) traceCandidateSource(ctx context.Context, db snapshotSQL, batches possible = append(possible, "("+match+" OR "+field.name+" IS NULL)") ambiguous = append(ambiguous, field.name+" IS NULL") } - projection := `trace_id,min_start,max_start,max_end,has_error` + projection := `namespace,trace_id,min_start,max_start,max_end,has_error` if len(exact) == 0 { return "SELECT " + projection + " FROM (" + index + ")", nil } result := "SELECT " + projection + " FROM (" + index + ") WHERE " + strings.Join(exact, " AND ") - mixed := "SELECT trace_id FROM (" + index + ") WHERE " + strings.Join(possible, " AND ") + " AND (" + strings.Join(ambiguous, " OR ") + ")" + mixed := "SELECT namespace,trace_id FROM (" + index + ") WHERE " + strings.Join(possible, " AND ") + " AND (" + strings.Join(ambiguous, " OR ") + ")" // Most traces belong to a single namespace/service. Do not put a retained // batch-parts aggregation in the ranking/join plan when this snapshot has no // ambiguous candidates for the requested scope. The probe shares its read @@ -697,7 +696,7 @@ func (d *Duck) traceCandidateSource(ctx context.Context, db snapshotSQL, batches return result, nil } scope := "batch_id IN (" + idsSQL(cachedIDs) + ") AND " + strings.Join(exact, " AND ") - return result + ` UNION ALL SELECT trace_id,min(min_start),max(max_start),max(max_end),max(has_error) FROM read_trace_parts WHERE ` + scope + ` AND trace_id IN (` + mixed + `) GROUP BY trace_id`, nil + return result + ` UNION ALL SELECT namespace,trace_id,min(min_start),max(max_start),max(max_end),max(has_error) FROM read_trace_parts WHERE ` + scope + ` AND (namespace,trace_id) IN (` + mixed + `) GROUP BY namespace,trace_id`, nil } func minTime(a, b time.Time) time.Time { diff --git a/internal/engine/duck.go b/internal/engine/duck.go index 0e5366ef9..d37a5312f 100644 --- a/internal/engine/duck.go +++ b/internal/engine/duck.go @@ -2382,6 +2382,13 @@ func (d *Duck) Trace(ctx context.Context, query telemetry.TraceQuery) ([]telemet return d.repository.Parquet.Trace(ctx, query) } +// TraceSummaries pins one snapshot for the bounded multi-trace root pass. +func (d *Duck) TraceSummaries(ctx context.Context, ids []telemetry.TraceIdentity, start, end int64) ([]telemetry.TraceTotals, error) { + pin := d.pinParquet() + defer d.unpinParquet(pin) + return d.repository.Parquet.TraceSummaries(ctx, ids, start, end) +} + // PublishParquet runs one catalog swap. Readers are never excluded or waited // for: the swap is atomic under the catalog lock, and what it replaces stays // on disk until no pinned reader can open it. The budget only bounds the diff --git a/internal/observability/batch_reads_test.go b/internal/observability/batch_reads_test.go index 2a43e32bb..036989326 100644 --- a/internal/observability/batch_reads_test.go +++ b/internal/observability/batch_reads_test.go @@ -34,9 +34,9 @@ func newCompletedReadTest(t *testing.T) (*engine.Duck, *telemetrystore.Repositor const rawTraceQuery = `SELECT trace_id FROM spans WHERE start_time >= ?::TIMESTAMP_NS::TIMESTAMPTZ_NS AND start_time < ?::TIMESTAMP_NS::TIMESTAMPTZ_NS AND (? = '' OR namespace = ?) AND trace_id <> '' AND (? = '' OR service = ?) -GROUP BY trace_id +GROUP BY namespace,trace_id ORDER BY MAX(CASE WHEN status = 'STATUS_CODE_ERROR' THEN 1 ELSE 0 END) DESC, - MAX(end_unix_nano) - MIN(start_unix_nano) DESC, trace_id ASC + MAX(end_unix_nano) - MIN(start_unix_nano) DESC, trace_id ASC, namespace ASC LIMIT 1` func rawTraceID(t testing.TB, d *engine.Duck, scope Scope, service string) (string, error) { diff --git a/internal/observability/service_test.go b/internal/observability/service_test.go index 322980e89..219cf27e1 100644 --- a/internal/observability/service_test.go +++ b/internal/observability/service_test.go @@ -188,7 +188,7 @@ func TestTraceSelectsRecentErrorAndCorrelatesLogs(t *testing.T) { end := start.Add(time.Hour) mock.ExpectQuery(regexp.QuoteMeta(queryrows.CompletedTraceQuery)). WithArgs(start, end, "prod", "prod", "checkout", "checkout", 1). - WillReturnRows(sqlmock.NewRows([]string{"trace_id"}).AddRow("trace-1")) + WillReturnRows(sqlmock.NewRows([]string{"namespace", "trace_id", "has_error"}).AddRow("prod", "trace-1", 1)) if err := repository.Commit(context.Background(), telemetrystore.Batch{ID: "trace-fixture", Spans: []telemetry.Span{ {Namespace: "prod", TraceID: "trace-1", SpanID: "root", ServiceName: "checkout", Name: "POST /pay", Kind: "SERVER", StartUnixNanos: start.UnixNano(), DurationMS: 200, StatusCode: "STATUS_CODE_ERROR", StatusMsg: "declined"}, {Namespace: "prod", TraceID: "trace-1", SpanID: "child", ParentSpanID: "root", ServiceName: "payments", Name: "charge", Kind: "CLIENT", StartUnixNanos: start.Add(20 * time.Millisecond).UnixNano(), DurationMS: 80, StatusCode: "OK"}, diff --git a/internal/observability/trace.go b/internal/observability/trace.go index 3a924df73..39ccf0450 100644 --- a/internal/observability/trace.go +++ b/internal/observability/trace.go @@ -36,7 +36,7 @@ func (s *Service) Trace(ctx context.Context, scope Scope, traceID, service strin return Result[TraceDetail]{}, fmt.Errorf("query recent trace: %w", queryErr) } if rows.Next() { - if err := rows.Scan(&traceID); err != nil { + if err := rows.Scan(&scope.Namespace, &traceID, new(int)); err != nil { rows.Close() return Result[TraceDetail]{}, fmt.Errorf("scan recent trace: %w", err) } diff --git a/internal/observability/trace_test.go b/internal/observability/trace_test.go index 3e6b64efa..11b833535 100644 --- a/internal/observability/trace_test.go +++ b/internal/observability/trace_test.go @@ -9,6 +9,7 @@ import ( "github.com/DATA-DOG/go-sqlmock" _ "github.com/duckdb/duckdb-go/v2" + "github.com/labstack/fanout/internal/queryrows" "github.com/labstack/fanout/internal/telemetry" telemetrystore "github.com/labstack/fanout/internal/telemetry/store" ) @@ -115,3 +116,27 @@ func TestTraceWholeTraceIsNotReportedAsTruncated(t *testing.T) { t.Fatal(err) } } + +func TestRecentTraceKeepsSelectedNamespace(t *testing.T) { + svc, mock, repo := newMockService(t) + start := time.Date(2026, 7, 20, 11, 0, 0, 0, time.UTC) + end := start.Add(time.Hour) + if err := repo.Commit(t.Context(), telemetrystore.Batch{ID: "namespaces", Spans: []telemetry.Span{ + {Namespace: "a", TraceID: "shared", SpanID: "a", ServiceName: "checkout", StartUnixNanos: start.UnixNano(), DurationMS: 1}, + {Namespace: "b", TraceID: "shared", SpanID: "b", ServiceName: "other", StartUnixNanos: start.UnixNano(), DurationMS: 9, StatusCode: "STATUS_CODE_ERROR"}, + }}); err != nil { + t.Fatal(err) + } + mock.ExpectQuery(regexp.QuoteMeta(queryrows.CompletedTraceQuery)).WithArgs(start, end, "", "", "checkout", "checkout", 1).WillReturnRows(sqlmock.NewRows([]string{"namespace", "trace_id", "has_error"}).AddRow("a", "shared", 0)) + mock.ExpectQuery(regexp.QuoteMeta(traceLogsQuery)).WithArgs("shared", start, end, "a", "a", 10).WillReturnRows(sqlmock.NewRows([]string{"time", "severity", "service", "body", "trace_id", "span_id"})) + got, err := svc.Trace(t.Context(), Scope{Start: start, End: end}, "", "checkout", 10) + if err != nil { + t.Fatal(err) + } + if got.Data.SpanCount != 1 || got.Data.HasError || len(got.Data.Spans) != 1 || got.Data.Spans[0].SpanID != "a" { + t.Fatalf("namespace merged: %+v", got) + } + if err := mock.ExpectationsWereMet(); err != nil { + t.Fatal(err) + } +} diff --git a/internal/query/check.go b/internal/query/check.go index 64cd1bab7..cb6af3143 100644 --- a/internal/query/check.go +++ b/internal/query/check.go @@ -226,7 +226,7 @@ func checkWith(ctx context.Context, parser Parser, d *Dashboard, memo *checkMemo for name := range earlier { sample.Vars[name] = Value{Values: []string{"check"}} } - if _, _, err := rollupFilterValue(ctx, parser, f, sample); err != nil { + if _, _, err := rollupFilterValue(f, sample); err != nil { if isOperational(err) { return nil, nil, err } diff --git a/internal/query/deploy_split.go b/internal/query/deploy_split.go index 2f73113ab..22080783f 100644 --- a/internal/query/deploy_split.go +++ b/internal/query/deploy_split.go @@ -26,7 +26,7 @@ func (e *Executor) runDeploySplit(ctx context.Context, p *Panel, checked *Checke continue } if f.EqField == "service" || f.EqField == "namespace" { - field, value, err := rollupFilterValue(ctx, e.engine, f, scope) + field, value, err := rollupFilterValue(f, scope) if err != nil { return nil, "", err } diff --git a/internal/query/exemplar_bounds_test.go b/internal/query/exemplar_bounds_test.go index 17569bcdc..4ee0a00cc 100644 --- a/internal/query/exemplar_bounds_test.go +++ b/internal/query/exemplar_bounds_test.go @@ -15,17 +15,21 @@ import ( type exemplarProbeEngine struct { Engine - query func(context.Context, string, ...any) (queryrows.Rows, error) + summaries func(context.Context, []telemetry.TraceIdentity, int64, int64) ([]telemetry.TraceTotals, error) + query func(context.Context, string, ...any) (queryrows.Rows, error) } func (e exemplarProbeEngine) QueryContext(ctx context.Context, text string, args ...any) (queryrows.Rows, error) { return e.query(ctx, text, args...) } -func (e exemplarProbeEngine) Trace(ctx context.Context, q telemetry.TraceQuery) ([]telemetry.IndexedSpan, telemetry.TraceTotals, error) { +func (e exemplarProbeEngine) TraceSummaries(ctx context.Context, ids []telemetry.TraceIdentity, start, end int64) ([]telemetry.TraceTotals, error) { + if e.summaries != nil { + return e.summaries(ctx, ids, start, end) + } return e.Engine.(interface { - Trace(context.Context, telemetry.TraceQuery) ([]telemetry.IndexedSpan, telemetry.TraceTotals, error) - }).Trace(ctx, q) + TraceSummaries(context.Context, []telemetry.TraceIdentity, int64, int64) ([]telemetry.TraceTotals, error) + }).TraceSummaries(ctx, ids, start, end) } func exemplarDashboard(from string) Dashboard { diff --git a/internal/query/exemplar_index_test.go b/internal/query/exemplar_index_test.go index d4c8674d9..8cbfe79f7 100644 --- a/internal/query/exemplar_index_test.go +++ b/internal/query/exemplar_index_test.go @@ -38,7 +38,14 @@ func TestIndexedExemplarsRankTracesAndTruncate(t *testing.T) { t.Fatal(err) } queries := 0 - probe := exemplarProbeEngine{Engine: duck, query: func(ctx context.Context, text string, args ...any) (queryrows.Rows, error) { + rootPasses := 0 + probe := exemplarProbeEngine{Engine: duck, summaries: func(ctx context.Context, ids []telemetry.TraceIdentity, start, end int64) ([]telemetry.TraceTotals, error) { + rootPasses++ + if len(ids) != 20 { + t.Fatalf("root identities=%d", len(ids)) + } + return duck.TraceSummaries(ctx, ids, start, end) + }, query: func(ctx context.Context, text string, args ...any) (queryrows.Rows, error) { queries++ w, ok := queryrows.ReadWindow(ctx) if text != queryrows.CompletedTraceQuery || !ok || w.Kind != queryrows.TraceCandidateRead || w.Namespace != "shop" || w.Service != "checkout" || args[len(args)-1] != 21 { @@ -59,8 +66,8 @@ func TestIndexedExemplarsRankTracesAndTruncate(t *testing.T) { if err != nil { t.Fatal(err) } - if queries != 1 { - t.Fatalf("candidate statements=%d", queries) + if queries != 1 || rootPasses != 1 { + t.Fatalf("candidate statements=%d root passes=%d", queries, rootPasses) } if len(got.Traces) != 20 || !got.Truncated || got.Traces[0].TraceID != "trace-00" || got.Traces[1].TraceID != "trace-01" || got.Traces[2].TraceID != "trace-20" { t.Fatalf("ranking/cap: %+v", got) @@ -177,3 +184,106 @@ func TestExemplarIndexEligibility(t *testing.T) { } } } + +func TestIndexedExemplarsKeepNamespacesSeparate(t *testing.T) { + duck, repo := newTestEngine(t) + n := fixtureStart.Add(time.Minute).UnixNano() + commit(t, repo, []telemetry.Span{ + {Namespace: "a", ServiceName: "checkout", TraceID: "shared", SpanID: "root-a", Name: "root a", StartUnixNanos: n, EndUnixNanos: n + 1000000, DurationMS: 1, StatusCode: "STATUS_CODE_OK"}, + {Namespace: "b", ServiceName: "other", TraceID: "shared", SpanID: "root-b", Name: "root b", StartUnixNanos: n - 1, EndUnixNanos: n + 9000000, DurationMS: 9, StatusCode: "STATUS_CODE_ERROR"}, + {Namespace: "b", ServiceName: "checkout", TraceID: "shared", SpanID: "child-b", ParentSpanID: "root-b", Name: "checkout b", StartUnixNanos: n, EndUnixNanos: n + 2000000, DurationMS: 2, StatusCode: "STATUS_CODE_OK"}, + {Namespace: "c", ServiceName: "other", TraceID: "shared", SpanID: "root-c", Name: "wrong root", StartUnixNanos: n - 1, EndUnixNanos: n + 9000000, DurationMS: 9, StatusCode: "STATUS_CODE_ERROR"}, + {Namespace: "a", ServiceName: "other", TraceID: "shared", SpanID: "child", ParentSpanID: "root-a", StartUnixNanos: n, EndUnixNanos: n + 1000000, DurationMS: 1, StatusCode: "STATUS_CODE_ERROR"}, + }, nil) + e := NewExecutor(duck, 30) + defer e.Close() + req := exemplarRequest(exemplarDashboard("spans")) + req.Dimensions = map[string]string{"service": "checkout"} + for _, cached := range []bool{false, true} { + if cached { + if _, err := duck.RefreshReadCaches(t.Context()); err != nil { + t.Fatal(err) + } + } + got, err := e.exemplars(t.Context(), req, fixtureStart.Add(time.Hour)) + if err != nil { + t.Fatal(err) + } + if len(got.Traces) != 2 || got.Traces[0].Namespace != "b" || got.Traces[1].Namespace != "a" || got.Traces[0].Operation != "root b" || got.Traces[1].Operation != "root a" || got.Traces[1].Status != "STATUS_CODE_OK" || got.Traces[0].Status != "STATUS_CODE_OK" { + t.Fatalf("cached=%v: %+v", cached, got) + } + } +} + +func TestIndexedExemplarsReportMissingRoots(t *testing.T) { + duck, repo := newTestEngine(t) + n := fixtureStart.Add(time.Minute).UnixNano() + commit(t, repo, []telemetry.Span{{Namespace: "shop", ServiceName: "checkout", TraceID: "retired", SpanID: "root", StartUnixNanos: n, EndUnixNanos: n + 1}}, nil) + calls := 0 + probe := exemplarProbeEngine{Engine: duck, query: duck.QueryContext, summaries: func(_ context.Context, ids []telemetry.TraceIdentity, start, end int64) ([]telemetry.TraceTotals, error) { + calls++ + if len(ids) != 1 || ids[0] != (telemetry.TraceIdentity{Namespace: "shop", TraceID: "retired"}) { + t.Fatalf("identities: %+v", ids) + } + return make([]telemetry.TraceTotals, len(ids)), nil + }} + e := NewExecutor(probe, 30) + defer e.Close() + req := exemplarRequest(exemplarDashboard("spans")) + req.Dimensions = map[string]string{"service": "checkout"} + got, err := e.exemplars(t.Context(), req, fixtureStart.Add(time.Hour)) + if err != nil || !got.Truncated || len(got.Traces) != 0 || calls != 1 { + t.Fatalf("missing roots: %+v calls=%d err=%v", got, calls, err) + } +} + +func TestExemplarIndexSelectionEligibility(t *testing.T) { + e := newFixtureExecutor(t) + for _, tc := range []struct { + name, from, viz, where string + value Value + dimension string + want bool + }{ + {name: "multi value", from: "spans", viz: "bar", where: "service IN $s", value: Value{Values: []string{"checkout", "frontend"}}, dimension: "checkout", want: false}, + {name: "empty list", from: "spans", viz: "bar", where: "service IN $s", value: Value{Values: []string{}}, dimension: "checkout", want: false}, + {name: "logs", from: "logs", viz: "bar", dimension: "checkout", want: false}, + {name: "health service", from: "spans", viz: "health", dimension: "checkout", want: true}, + {name: "map service", from: "spans", viz: "service_map", dimension: "checkout", want: true}, + {name: "health unscoped", from: "spans", viz: "health", want: false}, + {name: "map unscoped", from: "spans", viz: "service_map", want: false}, + } { + t.Run(tc.name, func(t *testing.T) { + d := exemplarDashboard(tc.from) + d.Panels[0].Viz = tc.viz + if tc.viz == "health" || tc.viz == "service_map" { + d.Panels[0].Query.Measures = nil + d.Panels[0].Query.By = nil + } + d.Variables = []Variable{{Name: "s", Kind: "custom", Multi: true, Options: []string{"checkout", "frontend"}}} + if tc.where != "" { + d.Panels[0].Query.Where = []string{tc.where} + } + Normalize(&d) + checked, err := e.check(t.Context(), &d) + if err != nil { + t.Fatal(err) + } + dimensions := map[string]string{} + if tc.dimension != "" { + dimensions["service"] = tc.dimension + } + _, got, _, err := e.exemplarTraceScope(t.Context(), &d.Panels[0], checked.Filters["p"], Scope{Vars: map[string]Value{"s": tc.value}}, dimensions, nil) + if err != nil || got != tc.want { + t.Fatalf("eligible=%v want=%v err=%v", got, tc.want, err) + } + }) + } + d := exemplarDashboard("spans") + d.Panels[0].Viz = "timeseries" + req := exemplarRequest(d) + req.Dimensions = map[string]string{"service": "Other (3)"} + if _, err := e.exemplars(t.Context(), req, fixtureStart.Add(time.Hour)); err == nil { + t.Fatal("accepted Other group") + } +} diff --git a/internal/query/exemplars.go b/internal/query/exemplars.go index 7ed709f3c..6f66e53f9 100644 --- a/internal/query/exemplars.go +++ b/internal/query/exemplars.go @@ -371,7 +371,7 @@ func (e *Executor) exemplarTraceScope(ctx context.Context, p *Panel, filters []F if scope.dropped(f) { continue } - field, value, err := rollupFilterValue(ctx, e.engine, f, scope) + field, value, err := rollupFilterValue(f, scope) if err != nil { if isOperational(err) { return w, false, false, err @@ -390,14 +390,17 @@ func (e *Executor) indexedExemplars(ctx context.Context, w queryrows.Window, pan if err != nil { return out, exemplarError(ctx, err) } - var ids []string + var ids []telemetry.TraceIdentity + var scopedErrors []int for rows.Next() { - var id string - if err := rows.Scan(&id); err != nil { + var id telemetry.TraceIdentity + var hasError int + if err := rows.Scan(&id.Namespace, &id.TraceID, &hasError); err != nil { rows.Close() return out, exemplarError(ctx, err) } ids = append(ids, id) + scopedErrors = append(scopedErrors, hasError) } err = rows.Err() rows.Close() @@ -412,25 +415,32 @@ func (e *Executor) indexedExemplars(ctx context.Context, w queryrows.Window, pan ids = ids[:20] } reader, ok := e.engine.(interface { - Trace(context.Context, telemetry.TraceQuery) ([]telemetry.IndexedSpan, telemetry.TraceTotals, error) + TraceSummaries(context.Context, []telemetry.TraceIdentity, int64, int64) ([]telemetry.TraceTotals, error) }) if !ok { return out, fmt.Errorf("indexed trace reader unavailable") } - for _, id := range ids { - _, totals, err := reader.Trace(ctx, telemetry.TraceQuery{TraceID: id, Namespace: w.Namespace, StartNanos: panelStart.UnixNano(), EndNanos: panelEnd.UnixNano(), Limit: 1}) - if err != nil { - return out, exemplarError(ctx, err) - } - if totals.Root == nil { + totals, err := reader.TraceSummaries(ctx, ids, panelStart.UnixNano(), panelEnd.UnixNano()) + if err != nil { + return out, exemplarError(ctx, err) + } + for i, id := range ids { + if totals[i].Root == nil { + // Retention may remove an identity between ranking and this read. + out.Truncated = true continue } - root := totals.Root + root := totals[i].Root + // Error priority and display both describe the selected service/window, + // independently of the root's service and the rest of the trace. status := root.StatusCode - if totals.HasError { + if status == "STATUS_CODE_ERROR" { + status = "STATUS_CODE_OK" + } + if scopedErrors[i] != 0 { status = "STATUS_CODE_ERROR" } - out.Traces = append(out.Traces, Exemplar{TraceID: id, Namespace: root.Namespace, Service: root.ServiceName, Operation: root.Name, DurationMS: root.DurationMS, Status: status, Start: time.Unix(0, root.StartUnixNanos).UTC()}) + out.Traces = append(out.Traces, Exemplar{TraceID: id.TraceID, Namespace: id.Namespace, Service: root.ServiceName, Operation: root.Name, DurationMS: root.DurationMS, Status: status, Start: time.Unix(0, root.StartUnixNanos).UTC()}) } return out, nil } diff --git a/internal/query/filter.go b/internal/query/filter.go index 2764914ae..a00b685d7 100644 --- a/internal/query/filter.go +++ b/internal/query/filter.go @@ -12,6 +12,7 @@ import ( // Filter is one checked filter expression. type Filter struct { + ScopeEquality *SQLExpr `json:"-"` // Original checked tree, before parameter casts. RoutePopulation bool `json:"-"` // Checked nonempty http.route population, safe for bounded endpoint summaries. Columns []string `json:"-"` // Checked column dependencies, including attribute roots. // Expr is DuckDB's canonical rendering of the checked expression; it is @@ -73,6 +74,7 @@ func checkFilter(ctx context.Context, parser Parser, sig *signal, vars map[strin f := Filter{Columns: []string{}, Source: strings.TrimSpace(expr), InParams: map[string]bool{}} uses := map[string]int{} typed := SQLExpression(node["where_clause"]) + f.ScopeEquality = typed if err := walkFilterExpr(typed, sig, vars, &f, uses); err != nil { return Filter{}, err } diff --git a/internal/query/rollup_panels.go b/internal/query/rollup_panels.go index 81f83d9d0..7805b88f5 100644 --- a/internal/query/rollup_panels.go +++ b/internal/query/rollup_panels.go @@ -15,14 +15,10 @@ type RollupReader interface { } func (e *Executor) SetRollupReader(reader RollupReader) { e.rollups = reader } -func rollupFilterValue(ctx context.Context, parser Parser, f Filter, scope Scope) (string, string, error) { - tree, err := parser.ParseSQL(ctx, "SELECT 1 FROM spans WHERE ("+f.Source+")") - if err != nil { - return "", "", err - } - // Parse the author's source, not the canonical filter with its injected - // parameter casts. Explicit casts cannot be represented by a string scope. - return rollupNodeValue(SQLExpression(tree["where_clause"]), scope) +func rollupFilterValue(f Filter, scope Scope) (string, string, error) { + // Use the checked author's tree, before canonical parameter casts. Explicit + // casts cannot be represented by a string scope. + return rollupNodeValue(f.ScopeEquality, scope) } func rollupNodeValue(node *SQLExpr, scope Scope) (string, string, error) { @@ -74,7 +70,7 @@ func (e *Executor) runRollupPanel(ctx context.Context, p *Panel, filters []Filte if scope.dropped(f) { continue } - field, value, err := rollupFilterValue(ctx, e.engine, f, scope) + field, value, err := rollupFilterValue(f, scope) if err != nil { return nil, "", err } diff --git a/internal/query/rollup_panels_test.go b/internal/query/rollup_panels_test.go index 277a5ce0d..e6fd0be9a 100644 --- a/internal/query/rollup_panels_test.go +++ b/internal/query/rollup_panels_test.go @@ -100,7 +100,7 @@ func TestRollupPanelsRejectNormalizedEquality(t *testing.T) { e := newFixtureExecutor(t) scope := Scope{Vars: map[string]Value{"s": {Values: []string{" checkout "}}}} for _, where := range []string{"namespace = ' shop '", "service = ' checkout '", "service = $s"} { - if _, _, err := rollupFilterValue(t.Context(), e.engine, Filter{Source: where}, scope); err == nil { + if _, _, err := rollupFilterValue(checkedRollupFilter(t, e, where), scope); err == nil { t.Fatalf("scope normalization would change equality: %s", where) } } @@ -111,13 +111,13 @@ func TestRollupPanelsScopesAndSelection(t *testing.T) { for _, viz := range []string{"health", "service_map"} { p := Panel{ID: "p", Title: "P", Viz: viz, Query: &Spec{From: "spans"}} scope := Scope{Start: fixtureStart, End: fixtureStart.Add(time.Hour), Vars: map[string]Value{"s": {Values: []string{"checkout"}}}} - field, value, err := rollupFilterValue(t.Context(), e.engine, Filter{Source: "$s = service"}, scope) + field, value, err := rollupFilterValue(checkedRollupFilter(t, e, "$s = service"), scope) if err != nil || field != "service" || value != "checkout" { t.Fatalf("variable: %s %s %v", field, value, err) } for _, v := range []Value{{All: true}, {Values: []string{"checkout", "payment"}}, {Values: []string{""}}} { scope.Vars["s"] = v - if _, _, err := rollupFilterValue(t.Context(), e.engine, Filter{Source: "service = $s"}, scope); err == nil { + if _, _, err := rollupFilterValue(checkedRollupFilter(t, e, "service = $s"), scope); err == nil { t.Fatalf("accepted %+v", v) } } @@ -233,3 +233,13 @@ func TestHealthPanelReadsOnlySummaryAndTrendForScopedRollups(t *testing.T) { t.Fatalf("scoped health percentages: %+v", out[0].Frame.Health) } } + +func checkedRollupFilter(t *testing.T, e *Executor, source string) Filter { + t.Helper() + sig, _ := lookupSignal("spans") + f, err := checkFilter(t.Context(), e.engine, sig, map[string]Variable{"s": {Name: "s", Kind: "custom", Options: []string{"checkout"}}}, source) + if err != nil { + t.Fatal(err) + } + return f +} diff --git a/internal/queryrows/trace_candidates.go b/internal/queryrows/trace_candidates.go index 99f94f8ee..8d8edb792 100644 --- a/internal/queryrows/trace_candidates.go +++ b/internal/queryrows/trace_candidates.go @@ -2,20 +2,21 @@ package queryrows // CompletedTraceQuery ranks scoped traces from completed batch candidates and // the uncached/boundary tail. Arguments are start, end, namespace twice, service -// twice, and limit. Both top-N stages must use the same limit. +// twice, and limit. Both top-N stages must use the same limit. Results are +// namespace, trace_id, and the selected scope's has_error flag. const CompletedTraceQuery = `WITH tail AS ( - SELECT trace_id,min(start_unix_nano) AS min_start,max(end_unix_nano) AS max_end, + SELECT namespace,trace_id,min(start_unix_nano) AS min_start,max(end_unix_nano) AS max_end, max(CASE WHEN status = 'STATUS_CODE_ERROR' THEN 1 ELSE 0 END) AS has_error FROM trace_tail WHERE start_time>=$1::TIMESTAMP_NS::TIMESTAMPTZ_NS AND start_time<$2::TIMESTAMP_NS::TIMESTAMPTZ_NS - AND ($3='' OR namespace=$4) AND ($5='' OR service=$6) AND trace_id<>'' GROUP BY trace_id + AND ($3='' OR namespace=$4) AND ($5='' OR service=$6) AND trace_id<>'' GROUP BY namespace,trace_id ), untouched AS ( - SELECT trace_id,min_start,max_end,has_error FROM trace_candidates - WHERE trace_id NOT IN (SELECT trace_id FROM tail) - ORDER BY has_error DESC,max_end-min_start DESC,trace_id LIMIT $7 + SELECT namespace,trace_id,min_start,max_end,has_error FROM trace_candidates + WHERE (namespace,trace_id) NOT IN (SELECT namespace,trace_id FROM tail) + ORDER BY has_error DESC,max_end-min_start DESC,trace_id,namespace LIMIT $7 ), touched AS ( - SELECT t.trace_id, least(t.min_start,c.min_start) AS min_start,greatest(t.max_end,c.max_end) AS max_end, + SELECT t.namespace,t.trace_id, least(t.min_start,c.min_start) AS min_start,greatest(t.max_end,c.max_end) AS max_end, greatest(t.has_error,c.has_error) AS has_error - FROM tail t LEFT JOIN trace_candidates c ON t.trace_id=c.trace_id + FROM tail t LEFT JOIN trace_candidates c ON t.namespace=c.namespace AND t.trace_id=c.trace_id ), candidates AS (SELECT * FROM untouched UNION ALL SELECT * FROM touched) -SELECT trace_id FROM candidates ORDER BY has_error DESC,max_end-min_start DESC,trace_id LIMIT $7` +SELECT namespace,trace_id,has_error FROM candidates ORDER BY has_error DESC,max_end-min_start DESC,trace_id,namespace LIMIT $7` diff --git a/internal/telemetry/parquet.go b/internal/telemetry/parquet.go index b67e7a182..b89588115 100644 --- a/internal/telemetry/parquet.go +++ b/internal/telemetry/parquet.go @@ -598,15 +598,50 @@ func (r indexedSpanRow) span() IndexedSpan { } func readIndexedTrace(ctx context.Context, batch *storedBatch, match traceRange, query TraceQuery, selected *indexedSpanHeap, totals *TraceTotals, services map[string]struct{}) (err error) { - if match.row > uint64(math.MaxInt64) { - return errors.New("trace index row exceeds Parquet reader limit") - } file, err := os.Open(filepath.Join(batch.dir, "spans.parquet")) if err != nil { return err } reader := parquet.NewGenericReader[indexedSpanRow](file) defer func() { err = errors.Join(err, reader.Close(), file.Close()) }() + return readIndexedRange(ctx, reader, match, query.EndNanos, func(row indexedSpanRow) { + if row.TraceID != query.TraceID || row.StartUnixNano < query.StartNanos || row.StartUnixNano >= query.EndNanos || (query.Namespace != "" && row.Namespace != query.Namespace) { + return + } + accumulateTraceRow(row, totals, services) + selected.Add(row.span(), query.Limit) + }) +} + +func accumulateTraceRow(row indexedSpanRow, totals *TraceTotals, services map[string]struct{}) { + // Counted before the heap, so the totals describe the trace while + // the heap keeps describing the page. + totals.Spans++ + if row.Service != "" { + services[row.Service] = struct{}{} + } + if row.Status == "STATUS_CODE_ERROR" { + totals.HasError = true + } + if totals.MinStartNano == 0 || row.StartUnixNano < totals.MinStartNano { + totals.MinStartNano = row.StartUnixNano + } + if end := row.StartUnixNano + int64(row.DurationMS*float64(time.Millisecond)); end > totals.MaxEndNano { + totals.MaxEndNano = end + } + span := row.span() + if totals.Root == nil { + totals.Root = new(IndexedSpan) + } + if totals.Spans == 1 || indexedRootEarlier(span, *totals.Root) { + *totals.Root = span + } +} + +func readIndexedRange(ctx context.Context, reader *parquet.GenericReader[indexedSpanRow], match traceRange, end int64, visit func(indexedSpanRow)) error { + if match.row > uint64(math.MaxInt64) { + return errors.New("trace index row exceeds Parquet reader limit") + } if err := reader.SeekToRow(int64(match.row)); err != nil { return err } @@ -616,48 +651,113 @@ func readIndexedTrace(ctx context.Context, batch *storedBatch, match traceRange, if err := ctx.Err(); err != nil { return err } - want := min(uint64(len(buffer)), remaining) - n, readErr := reader.Read(buffer[:int(want)]) - if readErr != nil && !errors.Is(readErr, io.EOF) { - return readErr + n, err := reader.Read(buffer[:int(min(uint64(len(buffer)), remaining))]) + if err != nil && !errors.Is(err, io.EOF) { + return err } if n == 0 { return io.ErrUnexpectedEOF } - for i := range n { - row := buffer[i] - if row.StartUnixNano >= query.EndNanos { + for _, row := range buffer[:n] { + // Rows within one hash range are ordered by event time. + if row.StartUnixNano >= end { return nil } - if row.TraceID != query.TraceID || row.StartUnixNano < query.StartNanos || - (query.Namespace != "" && row.Namespace != query.Namespace) { - continue - } - // Counted before the heap, so the totals describe the trace while - // the heap keeps describing the page. - totals.Spans++ - if row.Service != "" { - services[row.Service] = struct{}{} - } - if row.Status == "STATUS_CODE_ERROR" { - totals.HasError = true - } - if totals.MinStartNano == 0 || row.StartUnixNano < totals.MinStartNano { - totals.MinStartNano = row.StartUnixNano - } - if end := row.StartUnixNano + int64(row.DurationMS*float64(time.Millisecond)); end > totals.MaxEndNano { - totals.MaxEndNano = end - } - span := row.span() - if totals.Root == nil { - totals.Root = new(IndexedSpan) + visit(row) + } + remaining -= uint64(n) + } + return nil +} + +// TraceIdentity keeps equal trace IDs in different namespaces separate. +type TraceIdentity struct{ Namespace, TraceID string } + +// TraceSummaries reads bounded identities in one batch pass, opening each index +// and matching span file once. Hash collisions are resolved by exact identity. +func (p *ParquetStore) TraceSummaries(ctx context.Context, ids []TraceIdentity, start, end int64) ([]TraceTotals, error) { + if len(ids) > 20 { + return nil, errors.New("trace summaries limit exceeds 20") + } + if start >= end { + return nil, errors.New("trace query time range must be positive") + } + totals := make([]TraceTotals, len(ids)) + positions := make(map[TraceIdentity]int, len(ids)) + services := make([]map[string]struct{}, len(ids)) + hashes := make([]uint64, 0, len(ids)) + for i, id := range ids { + positions[id] = i + services[i] = map[string]struct{}{} + hashes = append(hashes, xxh3.HashString(id.TraceID)) + } + slices.Sort(hashes) + hashes = slices.Compact(hashes) + p.mu.RLock() + batches := make([]*storedBatch, 0, len(p.batches)) + for _, batch := range p.batches { + batches = append(batches, batch) + } + p.mu.RUnlock() + for _, batch := range batches { + if err := ctx.Err(); err != nil { + return nil, err + } + if bounds := batch.metadata.SpanTime; bounds.Known && (bounds.MaxNanos < start || bounds.MinNanos >= end) { + continue + } + if err := readBatchTraceSummaries(ctx, batch, hashes, end, func(row indexedSpanRow) { + if row.StartUnixNano < start || row.StartUnixNano >= end { + return } - if totals.Spans == 1 || indexedRootEarlier(span, *totals.Root) { - *totals.Root = span + if i, ok := positions[TraceIdentity{row.Namespace, row.TraceID}]; ok { + accumulateTraceRow(row, &totals[i], services[i]) } - selected.Add(span, query.Limit) + }); err != nil { + return nil, err + } + } + for i := range totals { + totals[i].Services = len(services[i]) + if totals[i].Spans > 0 && totals[i].MaxEndNano > totals[i].MinStartNano { + totals[i].DurationMS = float64(totals[i].MaxEndNano-totals[i].MinStartNano) / float64(time.Millisecond) + } + } + return totals, nil +} + +func readBatchTraceSummaries(ctx context.Context, batch *storedBatch, hashes []uint64, end int64, visit func(indexedSpanRow)) (err error) { + if len(hashes) == 0 || batch.traces.entries == 0 { + return nil + } + index, err := os.Open(batch.traces.path) + if err != nil { + return err + } + defer func() { err = errors.Join(err, index.Close()) }() + var matches []traceRange + for _, hash := range hashes { + match, found, err := batch.traces.lookup(index, hash) + if err != nil { + return err + } + if found { + matches = append(matches, match) + } + } + if len(matches) == 0 { + return nil + } + file, err := os.Open(filepath.Join(batch.dir, "spans.parquet")) + if err != nil { + return err + } + reader := parquet.NewGenericReader[indexedSpanRow](file) + defer func() { err = errors.Join(err, reader.Close(), file.Close()) }() + for _, match := range matches { + if err := readIndexedRange(ctx, reader, match, end, visit); err != nil { + return err } - remaining -= uint64(n) } return nil } diff --git a/internal/telemetry/trace_index.go b/internal/telemetry/trace_index.go index f2b2814cb..f65b7ce49 100644 --- a/internal/telemetry/trace_index.go +++ b/internal/telemetry/trace_index.go @@ -216,6 +216,10 @@ func (index traceIndex) Lookup(hash uint64) (match traceRange, found bool, err e return traceRange{}, false, err } defer func() { err = errors.Join(err, file.Close()) }() + return index.lookup(file, hash) +} + +func (index traceIndex) lookup(file *os.File, hash uint64) (traceRange, bool, error) { low, high := uint64(0), index.entries for low < high { middle := low + (high-low)/2 From f9280bbe9e55f8f0a4e0118fc4bda5e9ae307794 Mon Sep 17 00:00:00 2001 From: Vishal Rana Date: Fri, 6 Nov 2026 00:38:00 -0800 Subject: [PATCH 3/3] fix(traces): preserve namespace identity and exemplar semantics --- internal/engine/file_snapshot_test.go | 46 +++++++++++++ internal/observability/service_test.go | 12 ++-- internal/observability/trace.go | 14 ++-- internal/observability/trace_test.go | 59 ++++++++++------- internal/query/exemplar_index_test.go | 60 ++++++++++++++++- internal/query/exemplars.go | 25 ++++--- internal/telemetry/parquet.go | 14 ++-- internal/telemetry/trace_summaries_test.go | 77 ++++++++++++++++++++++ 8 files changed, 254 insertions(+), 53 deletions(-) create mode 100644 internal/telemetry/trace_summaries_test.go diff --git a/internal/engine/file_snapshot_test.go b/internal/engine/file_snapshot_test.go index 256d0412e..6cfe77d43 100644 --- a/internal/engine/file_snapshot_test.go +++ b/internal/engine/file_snapshot_test.go @@ -701,3 +701,49 @@ func TestReadCacheBudgetsAllSignalsAndReportsInventoryWork(t *testing.T) { t.Fatalf("remaining batch: %v %v", markers, err) } } + +func TestRetirementRepairsOnlyMatchingNamespaceIdentity(t *testing.T) { + d, repo := newBatchCacheTest(t) + at := time.Now().UTC().Truncate(time.Hour) + old, now := at.Add(-2*time.Hour).UnixNano(), at.UnixNano() + for _, batch := range []telemetrystore.Batch{ + {ID: "retire-a", Spans: []telemetry.Span{{Namespace: "a", TraceID: "shared", ServiceName: "retired", StartUnixNanos: old, EndUnixNanos: old + 1, IngestedAt: old, StatusCode: "STATUS_CODE_ERROR"}}}, + {ID: "keep-a", Spans: []telemetry.Span{{Namespace: "a", TraceID: "shared", ServiceName: "api", StartUnixNanos: now, EndUnixNanos: now + 10, IngestedAt: now}}}, + {ID: "keep-b", Spans: []telemetry.Span{{Namespace: "b", TraceID: "shared", ServiceName: "worker", StartUnixNanos: now + 20, EndUnixNanos: now + 30, IngestedAt: now, StatusCode: "STATUS_CODE_ERROR"}}}, + } { + if err := commitRestoredFixture(repo, t.Context(), batch); err != nil { + t.Fatal(err) + } + } + if _, err := d.RefreshReadCaches(t.Context()); err != nil { + t.Fatal(err) + } + if count, err := repo.PruneParquet(t.Context(), d, at.Add(-time.Hour).UnixNano(), 8); err != nil || count != 1 { + t.Fatalf("prune=%d: %v", count, err) + } + if _, err := d.RefreshReadCaches(t.Context()); err != nil { + t.Fatal(err) + } + for _, want := range []struct { + namespace, service string + start, end int64 + hasError int + }{ + {"a", "api", now, now + 10, 0}, + {"b", "worker", now + 20, now + 30, 1}, + } { + var service string + var start, maxStart, end int64 + var hasError int + if err := d.DB.QueryRow(`SELECT service,min_start,max_start,max_end,has_error FROM read_trace_candidates WHERE namespace=? AND trace_id='shared'`, want.namespace).Scan(&service, &start, &maxStart, &end, &hasError); err != nil { + t.Fatal(err) + } + if service != want.service || start != want.start || maxStart != want.start || end != want.end || hasError != want.hasError { + t.Fatalf("namespace=%s service=%s bounds=%d/%d/%d error=%d", want.namespace, service, start, maxStart, end, hasError) + } + } + var count int + if err := d.DB.QueryRow(`SELECT count(*) FROM read_trace_candidates WHERE trace_id='shared'`).Scan(&count); err != nil || count != 2 { + t.Fatalf("candidates=%d: %v", count, err) + } +} diff --git a/internal/observability/service_test.go b/internal/observability/service_test.go index 219cf27e1..6e0b6f2ad 100644 --- a/internal/observability/service_test.go +++ b/internal/observability/service_test.go @@ -199,7 +199,7 @@ func TestTraceSelectsRecentErrorAndCorrelatesLogs(t *testing.T) { t.Fatalf("commit trace fixture: %v", err) } mock.ExpectQuery(regexp.QuoteMeta(traceLogsQuery)). - WithArgs("trace-1", start, end, "prod", "prod", 20). + WithArgs("trace-1", start, end, true, "prod", 20). WillReturnRows(sqlmock.NewRows([]string{"time", "severity", "service", "body", "trace_id", "span_id"}). AddRow(start.Add(150*time.Millisecond), "ERROR", "checkout", "payment declined", "trace-1", "root"). AddRow(start.Add(160*time.Millisecond), "ERROR", "payments", `charge failed: token=abc123 {"client_secret":"cs_live_9"}`, "trace-1", "child")) @@ -237,7 +237,7 @@ func TestTraceLogsUseFullScopeEventTimeAcrossBatches(t *testing.T) { } } mock.ExpectQuery(regexp.QuoteMeta(traceLogsQuery)). - WithArgs("trace-order", start, start.Add(time.Hour), "prod", "prod", 10). + WithArgs("trace-order", start, start.Add(time.Hour), true, "prod", 10). WillReturnRows(sqlmock.NewRows([]string{"time", "severity", "service", "body", "trace_id", "span_id"}). AddRow(start.Add(10*time.Millisecond), "", "", "earliest", "trace-order", ""). AddRow(start.Add(20*time.Millisecond), "", "", "middle", "trace-order", ""). @@ -263,7 +263,7 @@ func TestTraceUsesIndexedParquet(t *testing.T) { t.Fatal(err) } mock.ExpectQuery(regexp.QuoteMeta(traceLogsQuery)). - WithArgs("parquet-trace", start, end, "prod", "prod", 10). + WithArgs("parquet-trace", start, end, true, "prod", 10). WillReturnRows(sqlmock.NewRows([]string{"time", "severity", "service", "body", "trace_id", "span_id"}). AddRow(start.Add(time.Millisecond), "ERROR", "checkout", "token=secret", "parquet-trace", "root")) result, err := svc.Trace(context.Background(), Scope{Namespace: "prod", Start: start, End: end}, "parquet-trace", "", 10) @@ -295,7 +295,7 @@ func TestTraceCombinesIndexedSpansAcrossBatches(t *testing.T) { t.Fatal(err) } mock.ExpectQuery(regexp.QuoteMeta(traceLogsQuery)). - WithArgs("split-trace", start, end, "prod", "prod", 10). + WithArgs("split-trace", start, end, true, "prod", 10). WillReturnRows(sqlmock.NewRows([]string{"time", "severity", "service", "body", "trace_id", "span_id"})) result, err := svc.Trace(context.Background(), Scope{Namespace: "prod", Start: start, End: end}, "split-trace", "", 10) if err != nil { @@ -321,7 +321,7 @@ func TestTraceReadsRecentRootFromParquetIndex(t *testing.T) { t.Fatal(err) } mock.ExpectQuery(regexp.QuoteMeta(traceLogsQuery)). - WithArgs("new-trace", start, end, "prod", "prod", 10). + WithArgs("new-trace", start, end, true, "prod", 10). WillReturnRows(sqlmock.NewRows([]string{"time", "severity", "service", "body", "trace_id", "span_id"})) result, err := svc.Trace(context.Background(), Scope{Namespace: "prod", Start: start, End: end}, "new-trace", "", 10) if err != nil { @@ -347,7 +347,7 @@ func TestTraceFiltersIndexedSpansByNamespace(t *testing.T) { t.Fatal(err) } mock.ExpectQuery(regexp.QuoteMeta(traceLogsQuery)). - WithArgs("shared-trace", start, end, "prod", "prod", 10). + WithArgs("shared-trace", start, end, true, "prod", 10). WillReturnRows(sqlmock.NewRows([]string{"time", "severity", "service", "body", "trace_id", "span_id"})) result, err := svc.Trace(context.Background(), Scope{Namespace: "prod", Start: start, End: end}, "shared-trace", "", 10) if err != nil { diff --git a/internal/observability/trace.go b/internal/observability/trace.go index 39ccf0450..29adf336b 100644 --- a/internal/observability/trace.go +++ b/internal/observability/trace.go @@ -14,7 +14,7 @@ import ( const traceLogsQuery = ` SELECT time::TIMESTAMP_NS, severity, coalesce(service, ''), body, coalesce(trace_id, ''), coalesce(span_id, '') FROM logs -WHERE trace_id = ? AND time >= ?::TIMESTAMP_NS::TIMESTAMPTZ_NS AND time < ?::TIMESTAMP_NS::TIMESTAMPTZ_NS AND (? = '' OR namespace = ?) +WHERE trace_id = ? AND time >= ?::TIMESTAMP_NS::TIMESTAMPTZ_NS AND time < ?::TIMESTAMP_NS::TIMESTAMPTZ_NS AND (NOT ? OR namespace = ?) ORDER BY time ASC LIMIT ?` @@ -29,6 +29,7 @@ func (s *Service) Trace(ctx context.Context, scope Scope, traceID, service strin } traceID, service = strings.TrimSpace(traceID), strings.TrimSpace(service) ctx = queryrows.WithWindow(ctx, queryrows.Window{Start: scope.Start, End: scope.End, Namespace: scope.Namespace, Service: service}) + namespace, exactNamespace := scope.Namespace, false if traceID == "" { candidateCtx := queryrows.WithWindow(ctx, queryrows.Window{Start: scope.Start, End: scope.End, Namespace: scope.Namespace, Service: service, Kind: queryrows.TraceCandidateRead}) rows, queryErr := s.db.QueryContext(candidateCtx, queryrows.CompletedTraceQuery, scope.Start, scope.End, scope.Namespace, scope.Namespace, service, service, 1) @@ -36,11 +37,12 @@ func (s *Service) Trace(ctx context.Context, scope Scope, traceID, service strin return Result[TraceDetail]{}, fmt.Errorf("query recent trace: %w", queryErr) } if rows.Next() { - if err := rows.Scan(&scope.Namespace, &traceID, new(int)); err != nil { + if err := rows.Scan(&namespace, &traceID, new(int)); err != nil { rows.Close() return Result[TraceDetail]{}, fmt.Errorf("scan recent trace: %w", err) } } + exactNamespace = true if err := rows.Err(); err != nil { rows.Close() return Result[TraceDetail]{}, fmt.Errorf("iterate recent trace: %w", err) @@ -51,7 +53,7 @@ func (s *Service) Trace(ctx context.Context, scope Scope, traceID, service strin data := TraceDetail{TraceID: traceID, Services: []string{}, Spans: []TraceSpan{}, Logs: []LogEntry{}} if traceID != "" { storedSpans, totals, readErr := s.repository.Trace(ctx, telemetry.TraceQuery{ - TraceID: traceID, Namespace: scope.Namespace, + TraceID: traceID, Namespace: namespace, ExactNamespace: exactNamespace, StartNanos: scope.Start.UnixNano(), EndNanos: scope.End.UnixNano(), Limit: limit, }) if readErr != nil { @@ -88,7 +90,7 @@ func (s *Service) Trace(ctx context.Context, scope Scope, traceID, service strin data.SpanCount, data.ServiceCount = totals.Spans, totals.Services data.DurationMS, data.HasError = totals.DurationMS, totals.HasError data.Truncated = data.SpanCount > len(data.Spans) - data.Logs, err = s.traceLogsFromParquet(ctx, scope, traceID, limit) + data.Logs, err = s.traceLogsFromParquet(ctx, scope, traceID, namespace, exactNamespace, limit) if err != nil { return Result[TraceDetail]{}, err } @@ -104,8 +106,8 @@ func (s *Service) Trace(ctx context.Context, scope Scope, traceID, service strin return Result[TraceDetail]{Summary: summary, Data: data, Provenance: s.provenanceFor(scope)}, nil } -func (s *Service) traceLogsFromParquet(ctx context.Context, scope Scope, traceID string, limit int) ([]LogEntry, error) { - rows, err := s.db.QueryContext(ctx, traceLogsQuery, traceID, scope.Start, scope.End, scope.Namespace, scope.Namespace, limit) +func (s *Service) traceLogsFromParquet(ctx context.Context, scope Scope, traceID, namespace string, exactNamespace bool, limit int) ([]LogEntry, error) { + rows, err := s.db.QueryContext(ctx, traceLogsQuery, traceID, scope.Start, scope.End, exactNamespace || namespace != "", namespace, limit) if err != nil { return nil, fmt.Errorf("query trace parquet logs: %w", err) } diff --git a/internal/observability/trace_test.go b/internal/observability/trace_test.go index 11b833535..ec6e2793a 100644 --- a/internal/observability/trace_test.go +++ b/internal/observability/trace_test.go @@ -9,7 +9,6 @@ import ( "github.com/DATA-DOG/go-sqlmock" _ "github.com/duckdb/duckdb-go/v2" - "github.com/labstack/fanout/internal/queryrows" "github.com/labstack/fanout/internal/telemetry" telemetrystore "github.com/labstack/fanout/internal/telemetry/store" ) @@ -45,7 +44,7 @@ func TestTracePagedSpansStillDescribeTheWholeTrace(t *testing.T) { t.Fatal(err) } mock.ExpectQuery(regexp.QuoteMeta(traceLogsQuery)). - WithArgs("wide-trace", start, end, "prod", "prod", 1). + WithArgs("wide-trace", start, end, true, "prod", 1). WillReturnRows(sqlmock.NewRows([]string{"time", "severity", "service", "body", "trace_id", "span_id"})) result, err := svc.Trace(context.Background(), Scope{Namespace: "prod", Start: start, End: end}, "wide-trace", "", 1) @@ -98,7 +97,7 @@ func TestTraceWholeTraceIsNotReportedAsTruncated(t *testing.T) { t.Fatal(err) } mock.ExpectQuery(regexp.QuoteMeta(traceLogsQuery)). - WithArgs("small-trace", start, end, "prod", "prod", 10). + WithArgs("small-trace", start, end, true, "prod", 10). WillReturnRows(sqlmock.NewRows([]string{"time", "severity", "service", "body", "trace_id", "span_id"})) result, err := svc.Trace(context.Background(), Scope{Namespace: "prod", Start: start, End: end}, "small-trace", "", 10) @@ -118,25 +117,39 @@ func TestTraceWholeTraceIsNotReportedAsTruncated(t *testing.T) { } func TestRecentTraceKeepsSelectedNamespace(t *testing.T) { - svc, mock, repo := newMockService(t) - start := time.Date(2026, 7, 20, 11, 0, 0, 0, time.UTC) - end := start.Add(time.Hour) - if err := repo.Commit(t.Context(), telemetrystore.Batch{ID: "namespaces", Spans: []telemetry.Span{ - {Namespace: "a", TraceID: "shared", SpanID: "a", ServiceName: "checkout", StartUnixNanos: start.UnixNano(), DurationMS: 1}, - {Namespace: "b", TraceID: "shared", SpanID: "b", ServiceName: "other", StartUnixNanos: start.UnixNano(), DurationMS: 9, StatusCode: "STATUS_CODE_ERROR"}, - }}); err != nil { - t.Fatal(err) - } - mock.ExpectQuery(regexp.QuoteMeta(queryrows.CompletedTraceQuery)).WithArgs(start, end, "", "", "checkout", "checkout", 1).WillReturnRows(sqlmock.NewRows([]string{"namespace", "trace_id", "has_error"}).AddRow("a", "shared", 0)) - mock.ExpectQuery(regexp.QuoteMeta(traceLogsQuery)).WithArgs("shared", start, end, "a", "a", 10).WillReturnRows(sqlmock.NewRows([]string{"time", "severity", "service", "body", "trace_id", "span_id"})) - got, err := svc.Trace(t.Context(), Scope{Start: start, End: end}, "", "checkout", 10) - if err != nil { - t.Fatal(err) - } - if got.Data.SpanCount != 1 || got.Data.HasError || len(got.Data.Spans) != 1 || got.Data.Spans[0].SpanID != "a" { - t.Fatalf("namespace merged: %+v", got) - } - if err := mock.ExpectationsWereMet(); err != nil { - t.Fatal(err) + for _, namespace := range []string{"a", ""} { + t.Run(namespace, func(t *testing.T) { + duck, repo, svc := newCompletedReadTest(t) + start := time.Date(2026, 7, 20, 11, 0, 0, 0, time.UTC) + end := start.Add(time.Hour) + svc.now = func() time.Time { return end } + // Bypass ingest normalization to retain a genuinely empty namespace. + if err := repo.Parquet.CommitBatch(t.Context(), &telemetry.BatchMetadata{ID: "namespaces"}, []telemetry.Span{ + {Namespace: namespace, TraceID: "shared", SpanID: "a", ServiceName: "checkout", StartUnixNanos: start.UnixNano(), EndUnixNanos: start.Add(time.Millisecond).UnixNano(), DurationMS: 1}, + {Namespace: "b", TraceID: "shared", SpanID: "b", ServiceName: "other", StartUnixNanos: start.UnixNano(), EndUnixNanos: start.Add(9 * time.Millisecond).UnixNano(), DurationMS: 9, StatusCode: "STATUS_CODE_ERROR"}, + }, []telemetry.Log{ + {Namespace: namespace, TraceID: "shared", SpanID: "a", ServiceName: "checkout", TimeUnixNanos: start.UnixNano(), Body: "selected"}, + {Namespace: "b", TraceID: "shared", SpanID: "b", ServiceName: "other", TimeUnixNanos: start.UnixNano(), Body: "unrelated"}, + }, nil); err != nil { + t.Fatal(err) + } + for _, cached := range []bool{false, true} { + if cached { + if _, err := duck.RefreshReadCaches(t.Context()); err != nil { + t.Fatal(err) + } + } + got, err := svc.Trace(t.Context(), Scope{Start: start, End: end}, "", "checkout", 10) + if err != nil { + t.Fatal(err) + } + if got.Data.SpanCount != 1 || got.Data.HasError || len(got.Data.Spans) != 1 || got.Data.Spans[0].SpanID != "a" || len(got.Data.Logs) != 1 || got.Data.Logs[0].Body != "selected" { + t.Fatalf("cached=%v namespace merged: %+v", cached, got) + } + if got.Provenance != svc.provenanceFor(Scope{Start: start, End: end}) { + t.Fatalf("requested provenance changed: %+v", got.Provenance) + } + } + }) } } diff --git a/internal/query/exemplar_index_test.go b/internal/query/exemplar_index_test.go index 8cbfe79f7..ac08de744 100644 --- a/internal/query/exemplar_index_test.go +++ b/internal/query/exemplar_index_test.go @@ -22,7 +22,7 @@ func TestIndexedExemplarsRankTracesAndTruncate(t *testing.T) { } spans = append(spans, telemetry.Span{Namespace: "shop", ServiceName: "checkout", TraceID: fmt.Sprintf("trace-%02d", i), SpanID: "root", Name: "root", StartUnixNanos: n, EndUnixNanos: n + int64(i+1)*1000000, DurationMS: float64(i + 1), StatusCode: status}) } - // Rank a long scoped trace ahead of a trace with one slower span. + // Select a long scoped trace, but order its displayed short root last. late := fixtureStart.Add(45 * time.Minute).UnixNano() spans = append(spans, telemetry.Span{Namespace: "shop", ServiceName: "checkout", TraceID: "trace-01", SpanID: "late", ParentSpanID: "root", StartUnixNanos: late, EndUnixNanos: late + 1000000, DurationMS: 1}) // A much longer trace in another namespace and a different service must not win. @@ -69,7 +69,7 @@ func TestIndexedExemplarsRankTracesAndTruncate(t *testing.T) { if queries != 1 || rootPasses != 1 { t.Fatalf("candidate statements=%d root passes=%d", queries, rootPasses) } - if len(got.Traces) != 20 || !got.Truncated || got.Traces[0].TraceID != "trace-00" || got.Traces[1].TraceID != "trace-01" || got.Traces[2].TraceID != "trace-20" { + if len(got.Traces) != 20 || !got.Truncated || got.Traces[0].TraceID != "trace-00" || got.Traces[1].TraceID != "trace-20" || got.Traces[19].TraceID != "trace-01" { t.Fatalf("ranking/cap: %+v", got) } for _, trace := range got.Traces { @@ -209,7 +209,7 @@ func TestIndexedExemplarsKeepNamespacesSeparate(t *testing.T) { if err != nil { t.Fatal(err) } - if len(got.Traces) != 2 || got.Traces[0].Namespace != "b" || got.Traces[1].Namespace != "a" || got.Traces[0].Operation != "root b" || got.Traces[1].Operation != "root a" || got.Traces[1].Status != "STATUS_CODE_OK" || got.Traces[0].Status != "STATUS_CODE_OK" { + if len(got.Traces) != 2 || got.Traces[0].Namespace != "b" || got.Traces[1].Namespace != "a" || got.Traces[0].Operation != "root b" || got.Traces[1].Operation != "root a" || got.Traces[1].Status != "STATUS_CODE_ERROR" || got.Traces[0].Status != "STATUS_CODE_ERROR" { t.Fatalf("cached=%v: %+v", cached, got) } } @@ -287,3 +287,57 @@ func TestExemplarIndexSelectionEligibility(t *testing.T) { t.Fatal("accepted Other group") } } + +func TestExemplarPathsAgreeOnDisplayedStatusAndOrder(t *testing.T) { + duck, repo := newTestEngine(t) + n := fixtureStart.Add(time.Minute).UnixNano() + span := func(ns, id, service, parent string, duration float64, status string) telemetry.Span { + return telemetry.Span{Namespace: ns, TraceID: id, SpanID: service, ParentSpanID: parent, ServiceName: service, StartUnixNanos: n, EndUnixNanos: n + int64(duration*1e6), DurationMS: duration, StatusCode: status} + } + commit(t, repo, []telemetry.Span{ + span("a", "child-error", "checkout", "", 10, "STATUS_CODE_OK"), + span("a", "child-error", "other", "checkout", 1, "STATUS_CODE_ERROR"), + span("a", "root-error", "other", "", 5, "STATUS_CODE_ERROR"), + span("a", "root-error", "checkout", "other", 1, "STATUS_CODE_OK"), + span("a", "slow", "other", "", 30, "STATUS_CODE_OK"), + span("a", "slow", "checkout", "other", 1, "STATUS_CODE_OK"), + span("a", "tie", "checkout", "", 20, "STATUS_CODE_OK"), + span("b", "tie", "checkout", "", 20, "STATUS_CODE_OK"), + // This error starts exactly at the panel end and must not affect status. + {Namespace: "a", TraceID: "slow", SpanID: "outside", ParentSpanID: "other", ServiceName: "other", StartUnixNanos: fixtureStart.Add(time.Hour).UnixNano(), DurationMS: 1, StatusCode: "STATUS_CODE_ERROR"}, + }, nil) + e := NewExecutor(duck, 30) + defer e.Close() + req := exemplarRequest(exemplarDashboard("spans")) + req.Dimensions = map[string]string{"service": "checkout"} + for _, cached := range []bool{false, true} { + if cached { + if _, err := duck.RefreshReadCaches(t.Context()); err != nil { + t.Fatal(err) + } + } + indexed, err := e.exemplars(t.Context(), req, fixtureStart.Add(time.Hour)) + if err != nil { + t.Fatal(err) + } + if len(indexed.Traces) != 5 { + t.Fatalf("indexed: %+v", indexed) + } + wantIDs := []string{"child-error", "root-error", "slow", "tie", "tie"} + for i, id := range wantIDs { + if indexed.Traces[i].TraceID != id || (indexed.Traces[i].Status == "STATUS_CODE_ERROR") != (i < 2) { + t.Fatalf("display order/status: %+v", indexed) + } + } + if indexed.Traces[3].Namespace != "a" || indexed.Traces[4].Namespace != "b" { + t.Fatalf("namespace tie: %+v", indexed) + } + // An equivalent duration predicate requires the raw selection path. + req.Dashboard.Panels[0].Query.Where = []string{"duration_ms >= 0"} + raw, err := e.exemplars(t.Context(), req, fixtureStart.Add(time.Hour)) + req.Dashboard.Panels[0].Query.Where = nil + if err != nil || !reflect.DeepEqual(raw, indexed) { + t.Fatalf("cached=%v raw=%+v indexed=%+v err=%v", cached, raw, indexed, err) + } + } +} diff --git a/internal/query/exemplars.go b/internal/query/exemplars.go index 6f66e53f9..e9ba6b118 100644 --- a/internal/query/exemplars.go +++ b/internal/query/exemplars.go @@ -289,7 +289,7 @@ AND s.start_time>=?::TIMESTAMP_NS::TIMESTAMPTZ_NS AND s.start_time b.DurationMS + } + if a.TraceID != b.TraceID { + return a.TraceID < b.TraceID + } + return a.Namespace < b.Namespace + }) return out, nil } diff --git a/internal/telemetry/parquet.go b/internal/telemetry/parquet.go index b89588115..9bf660c8c 100644 --- a/internal/telemetry/parquet.go +++ b/internal/telemetry/parquet.go @@ -97,11 +97,13 @@ type TimeRange struct { } type TraceQuery struct { - TraceID string - Namespace string - StartNanos int64 - EndNanos int64 - Limit int + TraceID string + Namespace string + // ExactNamespace matches Namespace exactly, including an empty namespace. + ExactNamespace bool + StartNanos int64 + EndNanos int64 + Limit int } type storedBatch struct { @@ -605,7 +607,7 @@ func readIndexedTrace(ctx context.Context, batch *storedBatch, match traceRange, reader := parquet.NewGenericReader[indexedSpanRow](file) defer func() { err = errors.Join(err, reader.Close(), file.Close()) }() return readIndexedRange(ctx, reader, match, query.EndNanos, func(row indexedSpanRow) { - if row.TraceID != query.TraceID || row.StartUnixNano < query.StartNanos || row.StartUnixNano >= query.EndNanos || (query.Namespace != "" && row.Namespace != query.Namespace) { + if row.TraceID != query.TraceID || row.StartUnixNano < query.StartNanos || row.StartUnixNano >= query.EndNanos || ((query.ExactNamespace || query.Namespace != "") && row.Namespace != query.Namespace) { return } accumulateTraceRow(row, totals, services) diff --git a/internal/telemetry/trace_summaries_test.go b/internal/telemetry/trace_summaries_test.go new file mode 100644 index 000000000..9494243a5 --- /dev/null +++ b/internal/telemetry/trace_summaries_test.go @@ -0,0 +1,77 @@ +package telemetry + +import ( + "fmt" + "path/filepath" + "testing" + "time" + + "github.com/parquet-go/parquet-go" + "github.com/zeebo/xxh3" +) + +func TestTraceSummariesResolveCollisionsNamespacesAndWindowBounds(t *testing.T) { + store, err := OpenParquetStore(t.TempDir()) + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { _ = store.Close() }) + start := int64(time.Second) + end := start + int64(10*time.Millisecond) + hash := xxh3.HashString("shared") + for i := range 3 { + dir := t.TempDir() + n := start + int64(i)*int64(time.Millisecond) + parent := "root" + if i == 2 { + parent = "" + } + status := "STATUS_CODE_OK" + if i == 1 { + status = "STATUS_CODE_ERROR" + } + rows := []indexedSpanRow{ + {TraceID: "shared", SpanID: "before", StartUnixNano: start - 1, Status: "STATUS_CODE_ERROR", DurationMS: 100}, + {TraceID: "shared", SpanID: fmt.Sprint(i), ParentSpanID: parent, Service: fmt.Sprint(i % 2), StartUnixNano: n, DurationMS: 1}, + {Namespace: "b", TraceID: "shared", SpanID: fmt.Sprint(i), Service: "worker", StartUnixNano: n + 100, DurationMS: 3, Status: status}, + {TraceID: "collision", SpanID: "wrong", Service: "wrong", StartUnixNano: n + 200, DurationMS: 100, Status: "STATUS_CODE_ERROR"}, + {TraceID: "shared", SpanID: "at-end", StartUnixNano: end, DurationMS: 100, Status: "STATUS_CODE_ERROR"}, + {Namespace: "b", TraceID: "shared", SpanID: "after-end", StartUnixNano: end + 1, DurationMS: 100}, + } + if err := parquet.WriteFile(filepath.Join(dir, "spans.parquet"), rows); err != nil { + t.Fatal(err) + } + // Inject a collision: both IDs occupy one time-ordered hash range. This + // avoids a production hash hook and exercises the public summary read. + hashes := make([]uint64, len(rows)) + for j := range hashes { + hashes[j] = hash + } + path := filepath.Join(dir, "trace.fidx") + if err := writeTraceIndex(path, hashes); err != nil { + t.Fatal(err) + } + index, err := loadTraceIndex(path, len(rows)) + if err != nil { + t.Fatal(err) + } + store.batches[fmt.Sprint(i)] = &storedBatch{dir: dir, traces: index} + } + ids := []TraceIdentity{{Namespace: "", TraceID: "shared"}, {Namespace: "b", TraceID: "shared"}, {Namespace: "missing", TraceID: "shared"}, {TraceID: "absent"}} + got, err := store.TraceSummaries(t.Context(), ids, start, end) + if err != nil { + t.Fatal(err) + } + a, b := got[0], got[1] + if a.Spans != 3 || a.Services != 2 || a.HasError || a.DurationMS != 3 || a.MinStartNano != start || a.MaxEndNano != start+int64(3*time.Millisecond) || a.Root == nil || a.Root.SpanID != "2" || a.Root.TraceID != "shared" || a.Root.Namespace != "" { + t.Fatalf("empty namespace: %+v root=%+v", a, a.Root) + } + if b.Spans != 3 || b.Services != 1 || !b.HasError || b.DurationMS != 5 || b.MinStartNano != start+100 || b.MaxEndNano != start+100+int64(5*time.Millisecond) || b.Root == nil || b.Root.SpanID != "0" || b.Root.Namespace != "b" { + t.Fatalf("namespace b: %+v root=%+v", b, b.Root) + } + for _, missing := range got[2:] { + if missing != (TraceTotals{}) { + t.Fatalf("missing identity: %+v", missing) + } + } +}