diff --git a/internal/engine/batch_cache.go b/internal/engine/batch_cache.go index 8c7387e4a..3889722ea 100644 --- a/internal/engine/batch_cache.go +++ b/internal/engine/batch_cache.go @@ -25,7 +25,7 @@ import ( // anomaly_log is not one of them: it holds detector output that nothing can // rebuild, so the drop keeps it. Its table is created once; any change to its // schema must be a separate, explicit migration step. -const readCacheVersion = 17 +const readCacheVersion = 18 const ( readCacheBatchLimit = 64 @@ -280,7 +280,7 @@ func (d *Duck) cacheBatches(ctx context.Context, batches []telemetry.BatchMetada return err } if len(changed) > 0 { - if err := exec(`CREATE OR REPLACE TEMP TABLE cache_retired_traces AS SELECT DISTINCT trace_id FROM read_trace_parts WHERE batch_id IN (` + idsSQL(changed) + `)`); err != nil { + if err := exec(`CREATE OR REPLACE TEMP TABLE cache_retired_traces AS SELECT DISTINCT namespace,trace_id FROM read_trace_parts WHERE batch_id IN (` + idsSQL(changed) + `)`); err != nil { return 0, err } } @@ -294,10 +294,10 @@ func (d *Duck) cacheBatches(ctx context.Context, batches []telemetry.BatchMetada if len(changed) > 0 { // Retention can change a trace's extrema or error flag. Compaction whose // complete replacement is cached preserves them and bypasses this repair. - if err := exec(`DELETE FROM read_trace_candidates WHERE trace_id IN (SELECT trace_id FROM cache_retired_traces) AND trace_id NOT IN (SELECT trace_id FROM read_trace_parts)`); err != nil { + if err := exec(`DELETE FROM read_trace_candidates WHERE (namespace,trace_id) IN (SELECT namespace,trace_id FROM cache_retired_traces) AND (namespace,trace_id) NOT IN (SELECT namespace,trace_id FROM read_trace_parts)`); err != nil { return 0, err } - if err := exec(`MERGE INTO read_trace_candidates c USING (` + traceCandidateSelect + ` FROM read_trace_parts WHERE trace_id IN (SELECT trace_id FROM cache_retired_traces) GROUP BY trace_id) AS excluded ON c.trace_id=excluded.trace_id + if err := exec(`MERGE INTO read_trace_candidates c USING (` + traceCandidateSelect + ` FROM read_trace_parts WHERE (namespace,trace_id) IN (SELECT namespace,trace_id FROM cache_retired_traces) GROUP BY namespace,trace_id) AS excluded ON c.namespace=excluded.namespace AND c.trace_id=excluded.trace_id WHEN MATCHED THEN UPDATE SET namespace=excluded.namespace,service=excluded.service,min_start=excluded.min_start,max_start=excluded.max_start,max_end=excluded.max_end,has_error=excluded.has_error WHEN NOT MATCHED THEN INSERT ` + traceCandidateColumns); err != nil { return 0, err @@ -359,10 +359,9 @@ func (d *Duck) cacheBatches(ctx context.Context, batches []telemetry.BatchMetada if err := exec(`INSERT INTO read_trace_parts SELECT * FROM cache_new_trace_parts`); err != nil { return 0, err } - candidates := traceCandidateSelect + ` FROM cache_new_trace_parts GROUP BY trace_id` - if err := exec(`MERGE INTO read_trace_candidates c USING (` + candidates + `) AS excluded ON c.trace_id=excluded.trace_id + candidates := traceCandidateSelect + ` FROM cache_new_trace_parts GROUP BY namespace,trace_id` + if err := exec(`MERGE INTO read_trace_candidates c USING (` + candidates + `) AS excluded ON c.namespace=excluded.namespace AND c.trace_id=excluded.trace_id WHEN MATCHED THEN UPDATE SET - namespace=CASE WHEN c.namespace=excluded.namespace THEN excluded.namespace END, service=CASE WHEN c.service=excluded.service THEN excluded.service END, min_start=least(c.min_start,excluded.min_start), max_start=greatest(c.max_start,excluded.max_start), @@ -466,18 +465,18 @@ func (d *Duck) aggregateSources(ctx context.Context, db snapshotSQL, batches []t } interior := fmt.Sprintf("min_start>=%d AND max_start<%d", w.Start.UnixNano(), w.End.UnixNano()) if len(partialFiles) > 0 { - tail += " UNION ALL SELECT " + projection + " FROM (" + cleanSource("spans", d.snapshotSource("spans", partialFiles, batches)) + ") WHERE " + windowPredicate("start_time", w) + " AND trace_id IN (SELECT trace_id FROM (" + partial + "))" + tail += " UNION ALL SELECT " + projection + " FROM (" + cleanSource("spans", d.snapshotSource("spans", partialFiles, batches)) + ") WHERE " + windowPredicate("start_time", w) + " AND (namespace,trace_id) IN (SELECT namespace,trace_id FROM (" + partial + "))" // A partial trace may also have in-window contributions in other batches. // Include these parts as candidates; boundary rows merge them in the kernel. - sources["trace_candidates"] = `SELECT trace_id,min(min_start) AS min_start,max(max_end) AS max_end,max(has_error) AS has_error FROM read_trace_parts WHERE ` + scope + ` AND ` + interior + ` GROUP BY trace_id` + sources["trace_candidates"] = `SELECT namespace,trace_id,min(min_start) AS min_start,max(max_end) AS max_end,max(has_error) AS has_error FROM read_trace_parts WHERE ` + scope + ` AND ` + interior + ` GROUP BY namespace,trace_id` } else { // Every overlapping cached file lies inside the window, so a partial // trace's other spans are in files outside it. Its in-window parts are // still a candidate; the kernel needs one row per trace, and partial // traces are disjoint from the fully contained index rows. - sources["trace_candidates"] = "SELECT trace_id,min_start,max_end,has_error FROM (" + index + ") WHERE " + full + - ` UNION ALL SELECT trace_id,min(min_start),max(max_end),max(has_error) FROM read_trace_parts WHERE ` + scope + ` AND ` + interior + - ` AND trace_id IN (SELECT trace_id FROM (` + partial + `)) GROUP BY trace_id` + sources["trace_candidates"] = "SELECT namespace,trace_id,min_start,max_end,has_error FROM (" + index + ") WHERE " + full + + ` UNION ALL SELECT namespace,trace_id,min(min_start),max(max_end),max(has_error) FROM read_trace_parts WHERE ` + scope + ` AND ` + interior + + ` AND (namespace,trace_id) IN (SELECT namespace,trace_id FROM (` + partial + `)) GROUP BY namespace,trace_id` } sources["trace_tail"] = tail return nil @@ -594,10 +593,10 @@ func (d *Duck) transferBatchCacheLocked(ctx context.Context, output string, inpu // million traces that alone outgrew a 1 GB memory limit. const traceCandidateColumns = `VALUES (excluded.trace_id,excluded.namespace,excluded.service,excluded.min_start,excluded.max_start,excluded.max_end,excluded.has_error)` -// One candidate row per trace is sufficient while the requested namespace and -// service are unambiguous. Mixed scopes are NULL and use exact batch parts. +// One candidate per namespace and trace. Mixed services are NULL and use exact +// batch parts for a service selection. const traceCandidateSelect = `SELECT trace_id, - CASE WHEN min(namespace)=max(namespace) THEN min(namespace) END AS namespace, + namespace, CASE WHEN min(service)=max(service) THEN min(service) END AS service, min(min_start) AS min_start,max(max_start) AS max_start,max(max_end) AS max_end,max(has_error) AS has_error` @@ -666,8 +665,8 @@ func (d *Duck) traceCandidateSource(ctx context.Context, db snapshotSQL, batches } index := `SELECT * FROM read_trace_candidates` if len(changed) > 0 { - dirty := `SELECT trace_id FROM read_trace_parts WHERE batch_id IN (` + idsSQL(changed) + `)` - index += ` WHERE trace_id NOT IN (` + dirty + `) UNION ALL ` + traceCandidateSelect + ` FROM read_trace_parts WHERE batch_id IN (` + idsSQL(cachedIDs) + `) AND trace_id IN (` + dirty + `) GROUP BY trace_id` + dirty := `SELECT namespace,trace_id FROM read_trace_parts WHERE batch_id IN (` + idsSQL(changed) + `)` + index += ` WHERE (namespace,trace_id) NOT IN (` + dirty + `) UNION ALL ` + traceCandidateSelect + ` FROM read_trace_parts WHERE batch_id IN (` + idsSQL(cachedIDs) + `) AND (namespace,trace_id) IN (` + dirty + `) GROUP BY namespace,trace_id` } var exact, possible, ambiguous []string for _, field := range []struct{ name, value string }{{"namespace", w.Namespace}, {"service", w.Service}} { @@ -679,12 +678,12 @@ func (d *Duck) traceCandidateSource(ctx context.Context, db snapshotSQL, batches possible = append(possible, "("+match+" OR "+field.name+" IS NULL)") ambiguous = append(ambiguous, field.name+" IS NULL") } - projection := `trace_id,min_start,max_start,max_end,has_error` + projection := `namespace,trace_id,min_start,max_start,max_end,has_error` if len(exact) == 0 { return "SELECT " + projection + " FROM (" + index + ")", nil } result := "SELECT " + projection + " FROM (" + index + ") WHERE " + strings.Join(exact, " AND ") - mixed := "SELECT trace_id FROM (" + index + ") WHERE " + strings.Join(possible, " AND ") + " AND (" + strings.Join(ambiguous, " OR ") + ")" + mixed := "SELECT namespace,trace_id FROM (" + index + ") WHERE " + strings.Join(possible, " AND ") + " AND (" + strings.Join(ambiguous, " OR ") + ")" // Most traces belong to a single namespace/service. Do not put a retained // batch-parts aggregation in the ranking/join plan when this snapshot has no // ambiguous candidates for the requested scope. The probe shares its read @@ -697,7 +696,7 @@ func (d *Duck) traceCandidateSource(ctx context.Context, db snapshotSQL, batches return result, nil } scope := "batch_id IN (" + idsSQL(cachedIDs) + ") AND " + strings.Join(exact, " AND ") - return result + ` UNION ALL SELECT trace_id,min(min_start),max(max_start),max(max_end),max(has_error) FROM read_trace_parts WHERE ` + scope + ` AND trace_id IN (` + mixed + `) GROUP BY trace_id`, nil + return result + ` UNION ALL SELECT namespace,trace_id,min(min_start),max(max_start),max(max_end),max(has_error) FROM read_trace_parts WHERE ` + scope + ` AND (namespace,trace_id) IN (` + mixed + `) GROUP BY namespace,trace_id`, nil } func minTime(a, b time.Time) time.Time { diff --git a/internal/engine/duck.go b/internal/engine/duck.go index 0e5366ef9..d37a5312f 100644 --- a/internal/engine/duck.go +++ b/internal/engine/duck.go @@ -2382,6 +2382,13 @@ func (d *Duck) Trace(ctx context.Context, query telemetry.TraceQuery) ([]telemet return d.repository.Parquet.Trace(ctx, query) } +// TraceSummaries pins one snapshot for the bounded multi-trace root pass. +func (d *Duck) TraceSummaries(ctx context.Context, ids []telemetry.TraceIdentity, start, end int64) ([]telemetry.TraceTotals, error) { + pin := d.pinParquet() + defer d.unpinParquet(pin) + return d.repository.Parquet.TraceSummaries(ctx, ids, start, end) +} + // PublishParquet runs one catalog swap. Readers are never excluded or waited // for: the swap is atomic under the catalog lock, and what it replaces stays // on disk until no pinned reader can open it. The budget only bounds the diff --git a/internal/engine/file_snapshot_test.go b/internal/engine/file_snapshot_test.go index 256d0412e..6cfe77d43 100644 --- a/internal/engine/file_snapshot_test.go +++ b/internal/engine/file_snapshot_test.go @@ -701,3 +701,49 @@ func TestReadCacheBudgetsAllSignalsAndReportsInventoryWork(t *testing.T) { t.Fatalf("remaining batch: %v %v", markers, err) } } + +func TestRetirementRepairsOnlyMatchingNamespaceIdentity(t *testing.T) { + d, repo := newBatchCacheTest(t) + at := time.Now().UTC().Truncate(time.Hour) + old, now := at.Add(-2*time.Hour).UnixNano(), at.UnixNano() + for _, batch := range []telemetrystore.Batch{ + {ID: "retire-a", Spans: []telemetry.Span{{Namespace: "a", TraceID: "shared", ServiceName: "retired", StartUnixNanos: old, EndUnixNanos: old + 1, IngestedAt: old, StatusCode: "STATUS_CODE_ERROR"}}}, + {ID: "keep-a", Spans: []telemetry.Span{{Namespace: "a", TraceID: "shared", ServiceName: "api", StartUnixNanos: now, EndUnixNanos: now + 10, IngestedAt: now}}}, + {ID: "keep-b", Spans: []telemetry.Span{{Namespace: "b", TraceID: "shared", ServiceName: "worker", StartUnixNanos: now + 20, EndUnixNanos: now + 30, IngestedAt: now, StatusCode: "STATUS_CODE_ERROR"}}}, + } { + if err := commitRestoredFixture(repo, t.Context(), batch); err != nil { + t.Fatal(err) + } + } + if _, err := d.RefreshReadCaches(t.Context()); err != nil { + t.Fatal(err) + } + if count, err := repo.PruneParquet(t.Context(), d, at.Add(-time.Hour).UnixNano(), 8); err != nil || count != 1 { + t.Fatalf("prune=%d: %v", count, err) + } + if _, err := d.RefreshReadCaches(t.Context()); err != nil { + t.Fatal(err) + } + for _, want := range []struct { + namespace, service string + start, end int64 + hasError int + }{ + {"a", "api", now, now + 10, 0}, + {"b", "worker", now + 20, now + 30, 1}, + } { + var service string + var start, maxStart, end int64 + var hasError int + if err := d.DB.QueryRow(`SELECT service,min_start,max_start,max_end,has_error FROM read_trace_candidates WHERE namespace=? AND trace_id='shared'`, want.namespace).Scan(&service, &start, &maxStart, &end, &hasError); err != nil { + t.Fatal(err) + } + if service != want.service || start != want.start || maxStart != want.start || end != want.end || hasError != want.hasError { + t.Fatalf("namespace=%s service=%s bounds=%d/%d/%d error=%d", want.namespace, service, start, maxStart, end, hasError) + } + } + var count int + if err := d.DB.QueryRow(`SELECT count(*) FROM read_trace_candidates WHERE trace_id='shared'`).Scan(&count); err != nil || count != 2 { + t.Fatalf("candidates=%d: %v", count, err) + } +} diff --git a/internal/observability/batch_reads_test.go b/internal/observability/batch_reads_test.go index 2a43e32bb..036989326 100644 --- a/internal/observability/batch_reads_test.go +++ b/internal/observability/batch_reads_test.go @@ -34,9 +34,9 @@ func newCompletedReadTest(t *testing.T) (*engine.Duck, *telemetrystore.Repositor const rawTraceQuery = `SELECT trace_id FROM spans WHERE start_time >= ?::TIMESTAMP_NS::TIMESTAMPTZ_NS AND start_time < ?::TIMESTAMP_NS::TIMESTAMPTZ_NS AND (? = '' OR namespace = ?) AND trace_id <> '' AND (? = '' OR service = ?) -GROUP BY trace_id +GROUP BY namespace,trace_id ORDER BY MAX(CASE WHEN status = 'STATUS_CODE_ERROR' THEN 1 ELSE 0 END) DESC, - MAX(end_unix_nano) - MIN(start_unix_nano) DESC, trace_id ASC + MAX(end_unix_nano) - MIN(start_unix_nano) DESC, trace_id ASC, namespace ASC LIMIT 1` func rawTraceID(t testing.TB, d *engine.Duck, scope Scope, service string) (string, error) { diff --git a/internal/observability/service_test.go b/internal/observability/service_test.go index a0d6c3bc2..6e0b6f2ad 100644 --- a/internal/observability/service_test.go +++ b/internal/observability/service_test.go @@ -186,9 +186,9 @@ func TestTraceSelectsRecentErrorAndCorrelatesLogs(t *testing.T) { svc, mock, repository := newMockService(t) start := time.Date(2026, 7, 20, 11, 0, 0, 0, time.UTC) end := start.Add(time.Hour) - mock.ExpectQuery(regexp.QuoteMeta(completedTraceQuery)). - WithArgs(start, end, "prod", "prod", "checkout", "checkout"). - WillReturnRows(sqlmock.NewRows([]string{"trace_id"}).AddRow("trace-1")) + mock.ExpectQuery(regexp.QuoteMeta(queryrows.CompletedTraceQuery)). + WithArgs(start, end, "prod", "prod", "checkout", "checkout", 1). + WillReturnRows(sqlmock.NewRows([]string{"namespace", "trace_id", "has_error"}).AddRow("prod", "trace-1", 1)) if err := repository.Commit(context.Background(), telemetrystore.Batch{ID: "trace-fixture", Spans: []telemetry.Span{ {Namespace: "prod", TraceID: "trace-1", SpanID: "root", ServiceName: "checkout", Name: "POST /pay", Kind: "SERVER", StartUnixNanos: start.UnixNano(), DurationMS: 200, StatusCode: "STATUS_CODE_ERROR", StatusMsg: "declined"}, {Namespace: "prod", TraceID: "trace-1", SpanID: "child", ParentSpanID: "root", ServiceName: "payments", Name: "charge", Kind: "CLIENT", StartUnixNanos: start.Add(20 * time.Millisecond).UnixNano(), DurationMS: 80, StatusCode: "OK"}, @@ -199,7 +199,7 @@ func TestTraceSelectsRecentErrorAndCorrelatesLogs(t *testing.T) { t.Fatalf("commit trace fixture: %v", err) } mock.ExpectQuery(regexp.QuoteMeta(traceLogsQuery)). - WithArgs("trace-1", start, end, "prod", "prod", 20). + WithArgs("trace-1", start, end, true, "prod", 20). WillReturnRows(sqlmock.NewRows([]string{"time", "severity", "service", "body", "trace_id", "span_id"}). AddRow(start.Add(150*time.Millisecond), "ERROR", "checkout", "payment declined", "trace-1", "root"). AddRow(start.Add(160*time.Millisecond), "ERROR", "payments", `charge failed: token=abc123 {"client_secret":"cs_live_9"}`, "trace-1", "child")) @@ -237,7 +237,7 @@ func TestTraceLogsUseFullScopeEventTimeAcrossBatches(t *testing.T) { } } mock.ExpectQuery(regexp.QuoteMeta(traceLogsQuery)). - WithArgs("trace-order", start, start.Add(time.Hour), "prod", "prod", 10). + WithArgs("trace-order", start, start.Add(time.Hour), true, "prod", 10). WillReturnRows(sqlmock.NewRows([]string{"time", "severity", "service", "body", "trace_id", "span_id"}). AddRow(start.Add(10*time.Millisecond), "", "", "earliest", "trace-order", ""). AddRow(start.Add(20*time.Millisecond), "", "", "middle", "trace-order", ""). @@ -263,7 +263,7 @@ func TestTraceUsesIndexedParquet(t *testing.T) { t.Fatal(err) } mock.ExpectQuery(regexp.QuoteMeta(traceLogsQuery)). - WithArgs("parquet-trace", start, end, "prod", "prod", 10). + WithArgs("parquet-trace", start, end, true, "prod", 10). WillReturnRows(sqlmock.NewRows([]string{"time", "severity", "service", "body", "trace_id", "span_id"}). AddRow(start.Add(time.Millisecond), "ERROR", "checkout", "token=secret", "parquet-trace", "root")) result, err := svc.Trace(context.Background(), Scope{Namespace: "prod", Start: start, End: end}, "parquet-trace", "", 10) @@ -295,7 +295,7 @@ func TestTraceCombinesIndexedSpansAcrossBatches(t *testing.T) { t.Fatal(err) } mock.ExpectQuery(regexp.QuoteMeta(traceLogsQuery)). - WithArgs("split-trace", start, end, "prod", "prod", 10). + WithArgs("split-trace", start, end, true, "prod", 10). WillReturnRows(sqlmock.NewRows([]string{"time", "severity", "service", "body", "trace_id", "span_id"})) result, err := svc.Trace(context.Background(), Scope{Namespace: "prod", Start: start, End: end}, "split-trace", "", 10) if err != nil { @@ -321,7 +321,7 @@ func TestTraceReadsRecentRootFromParquetIndex(t *testing.T) { t.Fatal(err) } mock.ExpectQuery(regexp.QuoteMeta(traceLogsQuery)). - WithArgs("new-trace", start, end, "prod", "prod", 10). + WithArgs("new-trace", start, end, true, "prod", 10). WillReturnRows(sqlmock.NewRows([]string{"time", "severity", "service", "body", "trace_id", "span_id"})) result, err := svc.Trace(context.Background(), Scope{Namespace: "prod", Start: start, End: end}, "new-trace", "", 10) if err != nil { @@ -347,7 +347,7 @@ func TestTraceFiltersIndexedSpansByNamespace(t *testing.T) { t.Fatal(err) } mock.ExpectQuery(regexp.QuoteMeta(traceLogsQuery)). - WithArgs("shared-trace", start, end, "prod", "prod", 10). + WithArgs("shared-trace", start, end, true, "prod", 10). WillReturnRows(sqlmock.NewRows([]string{"time", "severity", "service", "body", "trace_id", "span_id"})) result, err := svc.Trace(context.Background(), Scope{Namespace: "prod", Start: start, End: end}, "shared-trace", "", 10) if err != nil { diff --git a/internal/observability/trace.go b/internal/observability/trace.go index efb691020..29adf336b 100644 --- a/internal/observability/trace.go +++ b/internal/observability/trace.go @@ -14,7 +14,7 @@ import ( const traceLogsQuery = ` SELECT time::TIMESTAMP_NS, severity, coalesce(service, ''), body, coalesce(trace_id, ''), coalesce(span_id, '') FROM logs -WHERE trace_id = ? AND time >= ?::TIMESTAMP_NS::TIMESTAMPTZ_NS AND time < ?::TIMESTAMP_NS::TIMESTAMPTZ_NS AND (? = '' OR namespace = ?) +WHERE trace_id = ? AND time >= ?::TIMESTAMP_NS::TIMESTAMPTZ_NS AND time < ?::TIMESTAMP_NS::TIMESTAMPTZ_NS AND (NOT ? OR namespace = ?) ORDER BY time ASC LIMIT ?` @@ -29,18 +29,20 @@ func (s *Service) Trace(ctx context.Context, scope Scope, traceID, service strin } traceID, service = strings.TrimSpace(traceID), strings.TrimSpace(service) ctx = queryrows.WithWindow(ctx, queryrows.Window{Start: scope.Start, End: scope.End, Namespace: scope.Namespace, Service: service}) + namespace, exactNamespace := scope.Namespace, false if traceID == "" { candidateCtx := queryrows.WithWindow(ctx, queryrows.Window{Start: scope.Start, End: scope.End, Namespace: scope.Namespace, Service: service, Kind: queryrows.TraceCandidateRead}) - rows, queryErr := s.db.QueryContext(candidateCtx, completedTraceQuery, scope.Start, scope.End, scope.Namespace, scope.Namespace, service, service) + rows, queryErr := s.db.QueryContext(candidateCtx, queryrows.CompletedTraceQuery, scope.Start, scope.End, scope.Namespace, scope.Namespace, service, service, 1) if queryErr != nil { return Result[TraceDetail]{}, fmt.Errorf("query recent trace: %w", queryErr) } if rows.Next() { - if err := rows.Scan(&traceID); err != nil { + if err := rows.Scan(&namespace, &traceID, new(int)); err != nil { rows.Close() return Result[TraceDetail]{}, fmt.Errorf("scan recent trace: %w", err) } } + exactNamespace = true if err := rows.Err(); err != nil { rows.Close() return Result[TraceDetail]{}, fmt.Errorf("iterate recent trace: %w", err) @@ -51,7 +53,7 @@ func (s *Service) Trace(ctx context.Context, scope Scope, traceID, service strin data := TraceDetail{TraceID: traceID, Services: []string{}, Spans: []TraceSpan{}, Logs: []LogEntry{}} if traceID != "" { storedSpans, totals, readErr := s.repository.Trace(ctx, telemetry.TraceQuery{ - TraceID: traceID, Namespace: scope.Namespace, + TraceID: traceID, Namespace: namespace, ExactNamespace: exactNamespace, StartNanos: scope.Start.UnixNano(), EndNanos: scope.End.UnixNano(), Limit: limit, }) if readErr != nil { @@ -88,7 +90,7 @@ func (s *Service) Trace(ctx context.Context, scope Scope, traceID, service strin data.SpanCount, data.ServiceCount = totals.Spans, totals.Services data.DurationMS, data.HasError = totals.DurationMS, totals.HasError data.Truncated = data.SpanCount > len(data.Spans) - data.Logs, err = s.traceLogsFromParquet(ctx, scope, traceID, limit) + data.Logs, err = s.traceLogsFromParquet(ctx, scope, traceID, namespace, exactNamespace, limit) if err != nil { return Result[TraceDetail]{}, err } @@ -104,8 +106,8 @@ func (s *Service) Trace(ctx context.Context, scope Scope, traceID, service strin return Result[TraceDetail]{Summary: summary, Data: data, Provenance: s.provenanceFor(scope)}, nil } -func (s *Service) traceLogsFromParquet(ctx context.Context, scope Scope, traceID string, limit int) ([]LogEntry, error) { - rows, err := s.db.QueryContext(ctx, traceLogsQuery, traceID, scope.Start, scope.End, scope.Namespace, scope.Namespace, limit) +func (s *Service) traceLogsFromParquet(ctx context.Context, scope Scope, traceID, namespace string, exactNamespace bool, limit int) ([]LogEntry, error) { + rows, err := s.db.QueryContext(ctx, traceLogsQuery, traceID, scope.Start, scope.End, exactNamespace || namespace != "", namespace, limit) if err != nil { return nil, fmt.Errorf("query trace parquet logs: %w", err) } @@ -126,20 +128,3 @@ func (s *Service) traceLogsFromParquet(ctx context.Context, scope Scope, traceID rows.Close() return logs, nil } - -const completedTraceQuery = `WITH tail AS ( - SELECT trace_id,min(start_unix_nano) AS min_start,max(end_unix_nano) AS max_end, - max(CASE WHEN status = 'STATUS_CODE_ERROR' THEN 1 ELSE 0 END) AS has_error - FROM trace_tail - WHERE start_time>=$1::TIMESTAMP_NS::TIMESTAMPTZ_NS AND start_time<$2::TIMESTAMP_NS::TIMESTAMPTZ_NS - AND ($3='' OR namespace=$4) AND ($5='' OR service=$6) AND trace_id<>'' GROUP BY trace_id -), untouched AS ( - SELECT trace_id,min_start,max_end,has_error FROM trace_candidates - WHERE trace_id NOT IN (SELECT trace_id FROM tail) - ORDER BY has_error DESC,max_end-min_start DESC,trace_id LIMIT 1 -), touched AS ( - SELECT t.trace_id, least(t.min_start,c.min_start) AS min_start,greatest(t.max_end,c.max_end) AS max_end, - greatest(t.has_error,c.has_error) AS has_error - FROM tail t LEFT JOIN trace_candidates c ON t.trace_id=c.trace_id -), candidates AS (SELECT * FROM untouched UNION ALL SELECT * FROM touched) -SELECT trace_id FROM candidates ORDER BY has_error DESC,max_end-min_start DESC,trace_id LIMIT 1` diff --git a/internal/observability/trace_test.go b/internal/observability/trace_test.go index 3e6b64efa..ec6e2793a 100644 --- a/internal/observability/trace_test.go +++ b/internal/observability/trace_test.go @@ -44,7 +44,7 @@ func TestTracePagedSpansStillDescribeTheWholeTrace(t *testing.T) { t.Fatal(err) } mock.ExpectQuery(regexp.QuoteMeta(traceLogsQuery)). - WithArgs("wide-trace", start, end, "prod", "prod", 1). + WithArgs("wide-trace", start, end, true, "prod", 1). WillReturnRows(sqlmock.NewRows([]string{"time", "severity", "service", "body", "trace_id", "span_id"})) result, err := svc.Trace(context.Background(), Scope{Namespace: "prod", Start: start, End: end}, "wide-trace", "", 1) @@ -97,7 +97,7 @@ func TestTraceWholeTraceIsNotReportedAsTruncated(t *testing.T) { t.Fatal(err) } mock.ExpectQuery(regexp.QuoteMeta(traceLogsQuery)). - WithArgs("small-trace", start, end, "prod", "prod", 10). + WithArgs("small-trace", start, end, true, "prod", 10). WillReturnRows(sqlmock.NewRows([]string{"time", "severity", "service", "body", "trace_id", "span_id"})) result, err := svc.Trace(context.Background(), Scope{Namespace: "prod", Start: start, End: end}, "small-trace", "", 10) @@ -115,3 +115,41 @@ func TestTraceWholeTraceIsNotReportedAsTruncated(t *testing.T) { t.Fatal(err) } } + +func TestRecentTraceKeepsSelectedNamespace(t *testing.T) { + for _, namespace := range []string{"a", ""} { + t.Run(namespace, func(t *testing.T) { + duck, repo, svc := newCompletedReadTest(t) + start := time.Date(2026, 7, 20, 11, 0, 0, 0, time.UTC) + end := start.Add(time.Hour) + svc.now = func() time.Time { return end } + // Bypass ingest normalization to retain a genuinely empty namespace. + if err := repo.Parquet.CommitBatch(t.Context(), &telemetry.BatchMetadata{ID: "namespaces"}, []telemetry.Span{ + {Namespace: namespace, TraceID: "shared", SpanID: "a", ServiceName: "checkout", StartUnixNanos: start.UnixNano(), EndUnixNanos: start.Add(time.Millisecond).UnixNano(), DurationMS: 1}, + {Namespace: "b", TraceID: "shared", SpanID: "b", ServiceName: "other", StartUnixNanos: start.UnixNano(), EndUnixNanos: start.Add(9 * time.Millisecond).UnixNano(), DurationMS: 9, StatusCode: "STATUS_CODE_ERROR"}, + }, []telemetry.Log{ + {Namespace: namespace, TraceID: "shared", SpanID: "a", ServiceName: "checkout", TimeUnixNanos: start.UnixNano(), Body: "selected"}, + {Namespace: "b", TraceID: "shared", SpanID: "b", ServiceName: "other", TimeUnixNanos: start.UnixNano(), Body: "unrelated"}, + }, nil); err != nil { + t.Fatal(err) + } + for _, cached := range []bool{false, true} { + if cached { + if _, err := duck.RefreshReadCaches(t.Context()); err != nil { + t.Fatal(err) + } + } + got, err := svc.Trace(t.Context(), Scope{Start: start, End: end}, "", "checkout", 10) + if err != nil { + t.Fatal(err) + } + if got.Data.SpanCount != 1 || got.Data.HasError || len(got.Data.Spans) != 1 || got.Data.Spans[0].SpanID != "a" || len(got.Data.Logs) != 1 || got.Data.Logs[0].Body != "selected" { + t.Fatalf("cached=%v namespace merged: %+v", cached, got) + } + if got.Provenance != svc.provenanceFor(Scope{Start: start, End: end}) { + t.Fatalf("requested provenance changed: %+v", got.Provenance) + } + } + }) + } +} diff --git a/internal/query/check.go b/internal/query/check.go index 64cd1bab7..cb6af3143 100644 --- a/internal/query/check.go +++ b/internal/query/check.go @@ -226,7 +226,7 @@ func checkWith(ctx context.Context, parser Parser, d *Dashboard, memo *checkMemo for name := range earlier { sample.Vars[name] = Value{Values: []string{"check"}} } - if _, _, err := rollupFilterValue(ctx, parser, f, sample); err != nil { + if _, _, err := rollupFilterValue(f, sample); err != nil { if isOperational(err) { return nil, nil, err } diff --git a/internal/query/deploy_split.go b/internal/query/deploy_split.go index 2f73113ab..22080783f 100644 --- a/internal/query/deploy_split.go +++ b/internal/query/deploy_split.go @@ -26,7 +26,7 @@ func (e *Executor) runDeploySplit(ctx context.Context, p *Panel, checked *Checke continue } if f.EqField == "service" || f.EqField == "namespace" { - field, value, err := rollupFilterValue(ctx, e.engine, f, scope) + field, value, err := rollupFilterValue(f, scope) if err != nil { return nil, "", err } diff --git a/internal/query/exemplar_bounds_test.go b/internal/query/exemplar_bounds_test.go index a4c54caae..4ee0a00cc 100644 --- a/internal/query/exemplar_bounds_test.go +++ b/internal/query/exemplar_bounds_test.go @@ -15,13 +15,23 @@ import ( type exemplarProbeEngine struct { Engine - query func(context.Context, string, ...any) (queryrows.Rows, error) + summaries func(context.Context, []telemetry.TraceIdentity, int64, int64) ([]telemetry.TraceTotals, error) + query func(context.Context, string, ...any) (queryrows.Rows, error) } func (e exemplarProbeEngine) QueryContext(ctx context.Context, text string, args ...any) (queryrows.Rows, error) { return e.query(ctx, text, args...) } +func (e exemplarProbeEngine) TraceSummaries(ctx context.Context, ids []telemetry.TraceIdentity, start, end int64) ([]telemetry.TraceTotals, error) { + if e.summaries != nil { + return e.summaries(ctx, ids, start, end) + } + return e.Engine.(interface { + TraceSummaries(context.Context, []telemetry.TraceIdentity, int64, int64) ([]telemetry.TraceTotals, error) + }).TraceSummaries(ctx, ids, start, end) +} + func exemplarDashboard(from string) Dashboard { return Dashboard{Name: "Selection", Time: Time{Range: "1h"}, Panels: []Panel{{ID: "p", Title: "P", Viz: "bar", Query: &Spec{From: from, Measures: []string{"count()"}, By: []string{"service"}}}}} } @@ -378,7 +388,7 @@ func TestExemplarSampleKeepsRootOrderingAndTraceErrors(t *testing.T) { req.From, req.To = fixtureStart.Add(time.Minute), fixtureStart.Add(2*time.Minute) req.Dimensions = map[string]string{"service": "selected"} got, err := e.Exemplars(t.Context(), req) - if err != nil || len(got.Traces) != 1 || !got.Truncated { + if err != nil || len(got.Traces) != 1 || got.Truncated { t.Fatalf("sample: %+v %v", got, err) } root := got.Traces[0] diff --git a/internal/query/exemplar_index_test.go b/internal/query/exemplar_index_test.go new file mode 100644 index 000000000..ac08de744 --- /dev/null +++ b/internal/query/exemplar_index_test.go @@ -0,0 +1,343 @@ +package query + +import ( + "context" + "fmt" + "github.com/labstack/fanout/internal/queryrows" + "github.com/labstack/fanout/internal/telemetry" + telemetrystore "github.com/labstack/fanout/internal/telemetry/store" + "reflect" + "testing" + "time" +) + +func TestIndexedExemplarsRankTracesAndTruncate(t *testing.T) { + duck, repo := newTestEngine(t) + var spans []telemetry.Span + for i := range 21 { + n := fixtureStart.Add(time.Minute).UnixNano() + int64(i) + status := "STATUS_CODE_OK" + if i == 0 { + status = "STATUS_CODE_ERROR" + } + spans = append(spans, telemetry.Span{Namespace: "shop", ServiceName: "checkout", TraceID: fmt.Sprintf("trace-%02d", i), SpanID: "root", Name: "root", StartUnixNanos: n, EndUnixNanos: n + int64(i+1)*1000000, DurationMS: float64(i + 1), StatusCode: status}) + } + // Select a long scoped trace, but order its displayed short root last. + late := fixtureStart.Add(45 * time.Minute).UnixNano() + spans = append(spans, telemetry.Span{Namespace: "shop", ServiceName: "checkout", TraceID: "trace-01", SpanID: "late", ParentSpanID: "root", StartUnixNanos: late, EndUnixNanos: late + 1000000, DurationMS: 1}) + // A much longer trace in another namespace and a different service must not win. + for _, s := range []struct{ ns, service string }{{"other", "checkout"}, {"shop", "other"}} { + id := s.ns + s.service + if s.ns == "other" { + id = "trace-20" + } + spans = append(spans, telemetry.Span{Namespace: s.ns, ServiceName: s.service, TraceID: id, SpanID: "root", StartUnixNanos: fixtureStart.UnixNano(), EndUnixNanos: fixtureStart.Add(time.Hour).UnixNano(), DurationMS: 3600000, StatusCode: "STATUS_CODE_ERROR"}) + } + commit(t, repo, spans, nil) + if _, err := duck.RefreshReadCaches(t.Context()); err != nil { + t.Fatal(err) + } + queries := 0 + rootPasses := 0 + probe := exemplarProbeEngine{Engine: duck, summaries: func(ctx context.Context, ids []telemetry.TraceIdentity, start, end int64) ([]telemetry.TraceTotals, error) { + rootPasses++ + if len(ids) != 20 { + t.Fatalf("root identities=%d", len(ids)) + } + return duck.TraceSummaries(ctx, ids, start, end) + }, query: func(ctx context.Context, text string, args ...any) (queryrows.Rows, error) { + queries++ + w, ok := queryrows.ReadWindow(ctx) + if text != queryrows.CompletedTraceQuery || !ok || w.Kind != queryrows.TraceCandidateRead || w.Namespace != "shop" || w.Service != "checkout" || args[len(args)-1] != 21 { + t.Fatalf("unbounded candidate read: %s window=%+v args=%v", text, w, args) + } + return duck.QueryContext(ctx, text, args...) + }} + e := NewExecutor(probe, 30) + defer e.Close() + e.now = func() time.Time { return fixtureStart.Add(time.Hour) } + d := exemplarDashboard("spans") + d.Variables = []Variable{{Name: "service", Kind: "custom", Options: []string{"checkout", "other"}, IncludeAll: true, Default: AllValue}} + d.Panels[0].Query.Where = []string{"namespace = 'shop'", "service = $service"} + req := exemplarRequest(d) + req.Dimensions = map[string]string{"service": "checkout"} + req.Vars = map[string]Value{"service": {All: true}} + got, err := e.exemplars(t.Context(), req, e.now()) + if err != nil { + t.Fatal(err) + } + if queries != 1 || rootPasses != 1 { + t.Fatalf("candidate statements=%d root passes=%d", queries, rootPasses) + } + if len(got.Traces) != 20 || !got.Truncated || got.Traces[0].TraceID != "trace-00" || got.Traces[1].TraceID != "trace-20" || got.Traces[19].TraceID != "trace-01" { + t.Fatalf("ranking/cap: %+v", got) + } + for _, trace := range got.Traces { + if trace.Namespace != "shop" || trace.Service != "checkout" { + t.Fatalf("scope escaped: %+v", trace) + } + } + // Exactly 20 distinct traces is complete. + req.From = fixtureStart.Add(time.Minute + time.Nanosecond) + got, err = e.exemplars(t.Context(), req, e.now()) + if err != nil || len(got.Traces) != 20 || got.Truncated { + t.Fatalf("exact cap: %+v %v", got, err) + } + req.Dashboard.Panels[0].Query.Where = append(req.Dashboard.Panels[0].Query.Where, "service = 'other'") + got, err = e.exemplars(t.Context(), req, e.now()) + if err != nil || len(got.Traces) != 0 { + t.Fatalf("conflict escaped: %+v %v", got, err) + } +} + +func TestIndexedExemplarsSeeTailCompactionAndRetirement(t *testing.T) { + duck, repo := newTestEngine(t) + // Fresh ingest stamps let the compactor select these small batches. + at := time.Now().UTC().Add(-time.Minute) + start, end := at.Add(-time.Minute), at.Add(time.Minute) + for i := range 8 { + n := at.UnixNano() + int64(i)*1000000 + if err := repo.Commit(t.Context(), telemetrystore.Batch{ID: fmt.Sprintf("index-%d", i), Spans: []telemetry.Span{{Namespace: "shop", ServiceName: "checkout", TraceID: "shared", SpanID: fmt.Sprint(i), ParentSpanID: "missing", StartUnixNanos: n, EndUnixNanos: n + 1000000, DurationMS: 1, IngestedAt: n}}}); err != nil { + t.Fatal(err) + } + } + if _, err := duck.RefreshReadCaches(t.Context()); err != nil { + t.Fatal(err) + } + // The parentless root sorts after the children and is outside a Limit:1 page. + n := at.Add(time.Second).UnixNano() + if err := repo.Commit(t.Context(), telemetrystore.Batch{ID: "tail", Spans: []telemetry.Span{ + {Namespace: "shop", ServiceName: "frontend", TraceID: "shared", SpanID: "root", Name: "real root", StartUnixNanos: n, EndUnixNanos: n + 10000000, DurationMS: 10, IngestedAt: n}, + {Namespace: "shop", ServiceName: "checkout", TraceID: "new-error", SpanID: "root", StartUnixNanos: n, EndUnixNanos: n + 1000000, DurationMS: 1, StatusCode: "STATUS_CODE_ERROR", IngestedAt: n}, + }}); err != nil { + t.Fatal(err) + } + e := NewExecutor(duck, 30) + defer e.Close() + d := Dashboard{Name: "Indexed", Time: Time{From: &start, To: &end}, Panels: []Panel{{ID: "p", Title: "Services", Viz: "service_map", Query: &Spec{From: "spans", Where: []string{"namespace = 'shop'"}}}}} + req := ExemplarRequest{Dashboard: d, PanelID: "p", From: start, To: end, Dimensions: map[string]string{"service": "checkout"}} + read := func() ExemplarResponse { + t.Helper() + got, err := e.exemplars(t.Context(), req, end) + if err != nil { + t.Fatal(err) + } + return got + } + want := read() + if len(want.Traces) != 2 || want.Traces[0].TraceID != "new-error" || want.Traces[1].Operation != "real root" || want.Traces[1].Service != "frontend" || want.Truncated { + t.Fatalf("tail/root: %+v", want) + } + if _, err := duck.RefreshReadCaches(t.Context()); err != nil { + t.Fatal(err) + } + if got := read(); !reflect.DeepEqual(got, want) { + t.Fatalf("cache changed results: %+v want %+v", got, want) + } + if n, err := repo.CompactParquet(t.Context(), duck, 8); err != nil || n != 8 { + t.Fatalf("compact %d: %v", n, err) + } + if got := read(); !reflect.DeepEqual(got, want) { + t.Fatalf("compaction changed results: %+v want %+v", got, want) + } + if _, err := duck.RefreshReadCaches(t.Context()); err != nil { + t.Fatal(err) + } + if got := read(); !reflect.DeepEqual(got, want) { + t.Fatalf("retired cache changed results: %+v", got) + } + if n, err := repo.PruneParquet(t.Context(), duck, end.Add(time.Minute).UnixNano(), 8); err != nil || n == 0 { + t.Fatalf("prune %d: %v", n, err) + } + if got := read(); len(got.Traces) != 0 { + t.Fatalf("retired traces visible: %+v", got) + } +} + +func TestExemplarIndexEligibility(t *testing.T) { + e := newFixtureExecutor(t) + for _, tc := range []struct { + where []string + dimensions map[string]string + bucket *SelectionBucket + want bool + }{ + {nil, map[string]string{"service": "checkout"}, nil, true}, + {[]string{"service = 'checkout'"}, nil, nil, true}, + {[]string{"namespace = 'shop'"}, map[string]string{"service": "checkout"}, nil, true}, + {[]string{"status = 'STATUS_CODE_ERROR'"}, map[string]string{"service": "checkout"}, nil, false}, + {[]string{"lower(service) = 'checkout'"}, nil, nil, false}, + {nil, map[string]string{"service": "checkout"}, &SelectionBucket{Lower: 1}, false}, + {nil, map[string]string{"operation": "GET"}, nil, false}, + {nil, nil, nil, false}, + } { + d := exemplarDashboard("spans") + d.Panels[0].Query.Where = tc.where + Normalize(&d) + checked, err := e.check(t.Context(), &d) + if err != nil { + t.Fatal(err) + } + _, got, _, err := e.exemplarTraceScope(t.Context(), &d.Panels[0], checked.Filters["p"], Scope{}, tc.dimensions, tc.bucket) + if err != nil || got != tc.want { + t.Fatalf("where=%v dimensions=%v eligible=%v err=%v", tc.where, tc.dimensions, got, err) + } + } +} + +func TestIndexedExemplarsKeepNamespacesSeparate(t *testing.T) { + duck, repo := newTestEngine(t) + n := fixtureStart.Add(time.Minute).UnixNano() + commit(t, repo, []telemetry.Span{ + {Namespace: "a", ServiceName: "checkout", TraceID: "shared", SpanID: "root-a", Name: "root a", StartUnixNanos: n, EndUnixNanos: n + 1000000, DurationMS: 1, StatusCode: "STATUS_CODE_OK"}, + {Namespace: "b", ServiceName: "other", TraceID: "shared", SpanID: "root-b", Name: "root b", StartUnixNanos: n - 1, EndUnixNanos: n + 9000000, DurationMS: 9, StatusCode: "STATUS_CODE_ERROR"}, + {Namespace: "b", ServiceName: "checkout", TraceID: "shared", SpanID: "child-b", ParentSpanID: "root-b", Name: "checkout b", StartUnixNanos: n, EndUnixNanos: n + 2000000, DurationMS: 2, StatusCode: "STATUS_CODE_OK"}, + {Namespace: "c", ServiceName: "other", TraceID: "shared", SpanID: "root-c", Name: "wrong root", StartUnixNanos: n - 1, EndUnixNanos: n + 9000000, DurationMS: 9, StatusCode: "STATUS_CODE_ERROR"}, + {Namespace: "a", ServiceName: "other", TraceID: "shared", SpanID: "child", ParentSpanID: "root-a", StartUnixNanos: n, EndUnixNanos: n + 1000000, DurationMS: 1, StatusCode: "STATUS_CODE_ERROR"}, + }, nil) + e := NewExecutor(duck, 30) + defer e.Close() + req := exemplarRequest(exemplarDashboard("spans")) + req.Dimensions = map[string]string{"service": "checkout"} + for _, cached := range []bool{false, true} { + if cached { + if _, err := duck.RefreshReadCaches(t.Context()); err != nil { + t.Fatal(err) + } + } + got, err := e.exemplars(t.Context(), req, fixtureStart.Add(time.Hour)) + if err != nil { + t.Fatal(err) + } + if len(got.Traces) != 2 || got.Traces[0].Namespace != "b" || got.Traces[1].Namespace != "a" || got.Traces[0].Operation != "root b" || got.Traces[1].Operation != "root a" || got.Traces[1].Status != "STATUS_CODE_ERROR" || got.Traces[0].Status != "STATUS_CODE_ERROR" { + t.Fatalf("cached=%v: %+v", cached, got) + } + } +} + +func TestIndexedExemplarsReportMissingRoots(t *testing.T) { + duck, repo := newTestEngine(t) + n := fixtureStart.Add(time.Minute).UnixNano() + commit(t, repo, []telemetry.Span{{Namespace: "shop", ServiceName: "checkout", TraceID: "retired", SpanID: "root", StartUnixNanos: n, EndUnixNanos: n + 1}}, nil) + calls := 0 + probe := exemplarProbeEngine{Engine: duck, query: duck.QueryContext, summaries: func(_ context.Context, ids []telemetry.TraceIdentity, start, end int64) ([]telemetry.TraceTotals, error) { + calls++ + if len(ids) != 1 || ids[0] != (telemetry.TraceIdentity{Namespace: "shop", TraceID: "retired"}) { + t.Fatalf("identities: %+v", ids) + } + return make([]telemetry.TraceTotals, len(ids)), nil + }} + e := NewExecutor(probe, 30) + defer e.Close() + req := exemplarRequest(exemplarDashboard("spans")) + req.Dimensions = map[string]string{"service": "checkout"} + got, err := e.exemplars(t.Context(), req, fixtureStart.Add(time.Hour)) + if err != nil || !got.Truncated || len(got.Traces) != 0 || calls != 1 { + t.Fatalf("missing roots: %+v calls=%d err=%v", got, calls, err) + } +} + +func TestExemplarIndexSelectionEligibility(t *testing.T) { + e := newFixtureExecutor(t) + for _, tc := range []struct { + name, from, viz, where string + value Value + dimension string + want bool + }{ + {name: "multi value", from: "spans", viz: "bar", where: "service IN $s", value: Value{Values: []string{"checkout", "frontend"}}, dimension: "checkout", want: false}, + {name: "empty list", from: "spans", viz: "bar", where: "service IN $s", value: Value{Values: []string{}}, dimension: "checkout", want: false}, + {name: "logs", from: "logs", viz: "bar", dimension: "checkout", want: false}, + {name: "health service", from: "spans", viz: "health", dimension: "checkout", want: true}, + {name: "map service", from: "spans", viz: "service_map", dimension: "checkout", want: true}, + {name: "health unscoped", from: "spans", viz: "health", want: false}, + {name: "map unscoped", from: "spans", viz: "service_map", want: false}, + } { + t.Run(tc.name, func(t *testing.T) { + d := exemplarDashboard(tc.from) + d.Panels[0].Viz = tc.viz + if tc.viz == "health" || tc.viz == "service_map" { + d.Panels[0].Query.Measures = nil + d.Panels[0].Query.By = nil + } + d.Variables = []Variable{{Name: "s", Kind: "custom", Multi: true, Options: []string{"checkout", "frontend"}}} + if tc.where != "" { + d.Panels[0].Query.Where = []string{tc.where} + } + Normalize(&d) + checked, err := e.check(t.Context(), &d) + if err != nil { + t.Fatal(err) + } + dimensions := map[string]string{} + if tc.dimension != "" { + dimensions["service"] = tc.dimension + } + _, got, _, err := e.exemplarTraceScope(t.Context(), &d.Panels[0], checked.Filters["p"], Scope{Vars: map[string]Value{"s": tc.value}}, dimensions, nil) + if err != nil || got != tc.want { + t.Fatalf("eligible=%v want=%v err=%v", got, tc.want, err) + } + }) + } + d := exemplarDashboard("spans") + d.Panels[0].Viz = "timeseries" + req := exemplarRequest(d) + req.Dimensions = map[string]string{"service": "Other (3)"} + if _, err := e.exemplars(t.Context(), req, fixtureStart.Add(time.Hour)); err == nil { + t.Fatal("accepted Other group") + } +} + +func TestExemplarPathsAgreeOnDisplayedStatusAndOrder(t *testing.T) { + duck, repo := newTestEngine(t) + n := fixtureStart.Add(time.Minute).UnixNano() + span := func(ns, id, service, parent string, duration float64, status string) telemetry.Span { + return telemetry.Span{Namespace: ns, TraceID: id, SpanID: service, ParentSpanID: parent, ServiceName: service, StartUnixNanos: n, EndUnixNanos: n + int64(duration*1e6), DurationMS: duration, StatusCode: status} + } + commit(t, repo, []telemetry.Span{ + span("a", "child-error", "checkout", "", 10, "STATUS_CODE_OK"), + span("a", "child-error", "other", "checkout", 1, "STATUS_CODE_ERROR"), + span("a", "root-error", "other", "", 5, "STATUS_CODE_ERROR"), + span("a", "root-error", "checkout", "other", 1, "STATUS_CODE_OK"), + span("a", "slow", "other", "", 30, "STATUS_CODE_OK"), + span("a", "slow", "checkout", "other", 1, "STATUS_CODE_OK"), + span("a", "tie", "checkout", "", 20, "STATUS_CODE_OK"), + span("b", "tie", "checkout", "", 20, "STATUS_CODE_OK"), + // This error starts exactly at the panel end and must not affect status. + {Namespace: "a", TraceID: "slow", SpanID: "outside", ParentSpanID: "other", ServiceName: "other", StartUnixNanos: fixtureStart.Add(time.Hour).UnixNano(), DurationMS: 1, StatusCode: "STATUS_CODE_ERROR"}, + }, nil) + e := NewExecutor(duck, 30) + defer e.Close() + req := exemplarRequest(exemplarDashboard("spans")) + req.Dimensions = map[string]string{"service": "checkout"} + for _, cached := range []bool{false, true} { + if cached { + if _, err := duck.RefreshReadCaches(t.Context()); err != nil { + t.Fatal(err) + } + } + indexed, err := e.exemplars(t.Context(), req, fixtureStart.Add(time.Hour)) + if err != nil { + t.Fatal(err) + } + if len(indexed.Traces) != 5 { + t.Fatalf("indexed: %+v", indexed) + } + wantIDs := []string{"child-error", "root-error", "slow", "tie", "tie"} + for i, id := range wantIDs { + if indexed.Traces[i].TraceID != id || (indexed.Traces[i].Status == "STATUS_CODE_ERROR") != (i < 2) { + t.Fatalf("display order/status: %+v", indexed) + } + } + if indexed.Traces[3].Namespace != "a" || indexed.Traces[4].Namespace != "b" { + t.Fatalf("namespace tie: %+v", indexed) + } + // An equivalent duration predicate requires the raw selection path. + req.Dashboard.Panels[0].Query.Where = []string{"duration_ms >= 0"} + raw, err := e.exemplars(t.Context(), req, fixtureStart.Add(time.Hour)) + req.Dashboard.Panels[0].Query.Where = nil + if err != nil || !reflect.DeepEqual(raw, indexed) { + t.Fatalf("cached=%v raw=%+v indexed=%+v err=%v", cached, raw, indexed, err) + } + } +} diff --git a/internal/query/exemplars.go b/internal/query/exemplars.go index 0e9036b27..e9ba6b118 100644 --- a/internal/query/exemplars.go +++ b/internal/query/exemplars.go @@ -11,6 +11,7 @@ import ( "time" "github.com/labstack/fanout/internal/queryrows" + "github.com/labstack/fanout/internal/telemetry" ) type SelectionBucket struct { @@ -214,6 +215,17 @@ func (e *Executor) exemplars(ctx context.Context, req ExemplarRequest, now time. return out, err } + indexedScope, indexed, empty, err := e.exemplarTraceScope(ctx, p, checked.Filters[p.ID], Scope{Start: lo, End: hi, Vars: vars}, req.Dimensions, req.Bucket) + if err != nil { + return out, err + } + if indexed { + if empty { + return out, nil + } + return e.indexedExemplars(ctx, indexedScope, start, end) + } + if p.Query.From == "logs" && p.Options != nil && p.Options.Highlight != "" { where += " AND contains(lower(body),lower(?))" args = append(args, p.Options.Highlight) @@ -277,7 +289,7 @@ AND s.start_time>=?::TIMESTAMP_NS::TIMESTAMPTZ_NS AND s.start_time 20 + if out.Truncated { + ids = ids[:20] + } + reader, ok := e.engine.(interface { + TraceSummaries(context.Context, []telemetry.TraceIdentity, int64, int64) ([]telemetry.TraceTotals, error) + }) + if !ok { + return out, fmt.Errorf("indexed trace reader unavailable") + } + totals, err := reader.TraceSummaries(ctx, ids, panelStart.UnixNano(), panelEnd.UnixNano()) + if err != nil { + return out, exemplarError(ctx, err) + } + for i, id := range ids { + if totals[i].Root == nil { + // Retention may remove an identity between ranking and this read. + out.Truncated = true + continue + } + root := totals[i].Root + status := root.StatusCode + if totals[i].HasError { + status = "STATUS_CODE_ERROR" + } + out.Traces = append(out.Traces, Exemplar{TraceID: id.TraceID, Namespace: id.Namespace, Service: root.ServiceName, Operation: root.Name, DurationMS: root.DurationMS, Status: status, Start: time.Unix(0, root.StartUnixNanos).UTC()}) + } + // Candidate ranking is scoped; the final order describes the displayed roots. + sort.Slice(out.Traces, func(i, j int) bool { + a, b := out.Traces[i], out.Traces[j] + if (a.Status == "STATUS_CODE_ERROR") != (b.Status == "STATUS_CODE_ERROR") { + return a.Status == "STATUS_CODE_ERROR" + } + if a.DurationMS != b.DurationMS { + return a.DurationMS > b.DurationMS + } + if a.TraceID != b.TraceID { + return a.TraceID < b.TraceID + } + return a.Namespace < b.Namespace + }) + return out, nil +} diff --git a/internal/query/filter.go b/internal/query/filter.go index 2764914ae..a00b685d7 100644 --- a/internal/query/filter.go +++ b/internal/query/filter.go @@ -12,6 +12,7 @@ import ( // Filter is one checked filter expression. type Filter struct { + ScopeEquality *SQLExpr `json:"-"` // Original checked tree, before parameter casts. RoutePopulation bool `json:"-"` // Checked nonempty http.route population, safe for bounded endpoint summaries. Columns []string `json:"-"` // Checked column dependencies, including attribute roots. // Expr is DuckDB's canonical rendering of the checked expression; it is @@ -73,6 +74,7 @@ func checkFilter(ctx context.Context, parser Parser, sig *signal, vars map[strin f := Filter{Columns: []string{}, Source: strings.TrimSpace(expr), InParams: map[string]bool{}} uses := map[string]int{} typed := SQLExpression(node["where_clause"]) + f.ScopeEquality = typed if err := walkFilterExpr(typed, sig, vars, &f, uses); err != nil { return Filter{}, err } diff --git a/internal/query/rollup_panels.go b/internal/query/rollup_panels.go index 81f83d9d0..7805b88f5 100644 --- a/internal/query/rollup_panels.go +++ b/internal/query/rollup_panels.go @@ -15,14 +15,10 @@ type RollupReader interface { } func (e *Executor) SetRollupReader(reader RollupReader) { e.rollups = reader } -func rollupFilterValue(ctx context.Context, parser Parser, f Filter, scope Scope) (string, string, error) { - tree, err := parser.ParseSQL(ctx, "SELECT 1 FROM spans WHERE ("+f.Source+")") - if err != nil { - return "", "", err - } - // Parse the author's source, not the canonical filter with its injected - // parameter casts. Explicit casts cannot be represented by a string scope. - return rollupNodeValue(SQLExpression(tree["where_clause"]), scope) +func rollupFilterValue(f Filter, scope Scope) (string, string, error) { + // Use the checked author's tree, before canonical parameter casts. Explicit + // casts cannot be represented by a string scope. + return rollupNodeValue(f.ScopeEquality, scope) } func rollupNodeValue(node *SQLExpr, scope Scope) (string, string, error) { @@ -74,7 +70,7 @@ func (e *Executor) runRollupPanel(ctx context.Context, p *Panel, filters []Filte if scope.dropped(f) { continue } - field, value, err := rollupFilterValue(ctx, e.engine, f, scope) + field, value, err := rollupFilterValue(f, scope) if err != nil { return nil, "", err } diff --git a/internal/query/rollup_panels_test.go b/internal/query/rollup_panels_test.go index 277a5ce0d..e6fd0be9a 100644 --- a/internal/query/rollup_panels_test.go +++ b/internal/query/rollup_panels_test.go @@ -100,7 +100,7 @@ func TestRollupPanelsRejectNormalizedEquality(t *testing.T) { e := newFixtureExecutor(t) scope := Scope{Vars: map[string]Value{"s": {Values: []string{" checkout "}}}} for _, where := range []string{"namespace = ' shop '", "service = ' checkout '", "service = $s"} { - if _, _, err := rollupFilterValue(t.Context(), e.engine, Filter{Source: where}, scope); err == nil { + if _, _, err := rollupFilterValue(checkedRollupFilter(t, e, where), scope); err == nil { t.Fatalf("scope normalization would change equality: %s", where) } } @@ -111,13 +111,13 @@ func TestRollupPanelsScopesAndSelection(t *testing.T) { for _, viz := range []string{"health", "service_map"} { p := Panel{ID: "p", Title: "P", Viz: viz, Query: &Spec{From: "spans"}} scope := Scope{Start: fixtureStart, End: fixtureStart.Add(time.Hour), Vars: map[string]Value{"s": {Values: []string{"checkout"}}}} - field, value, err := rollupFilterValue(t.Context(), e.engine, Filter{Source: "$s = service"}, scope) + field, value, err := rollupFilterValue(checkedRollupFilter(t, e, "$s = service"), scope) if err != nil || field != "service" || value != "checkout" { t.Fatalf("variable: %s %s %v", field, value, err) } for _, v := range []Value{{All: true}, {Values: []string{"checkout", "payment"}}, {Values: []string{""}}} { scope.Vars["s"] = v - if _, _, err := rollupFilterValue(t.Context(), e.engine, Filter{Source: "service = $s"}, scope); err == nil { + if _, _, err := rollupFilterValue(checkedRollupFilter(t, e, "service = $s"), scope); err == nil { t.Fatalf("accepted %+v", v) } } @@ -233,3 +233,13 @@ func TestHealthPanelReadsOnlySummaryAndTrendForScopedRollups(t *testing.T) { t.Fatalf("scoped health percentages: %+v", out[0].Frame.Health) } } + +func checkedRollupFilter(t *testing.T, e *Executor, source string) Filter { + t.Helper() + sig, _ := lookupSignal("spans") + f, err := checkFilter(t.Context(), e.engine, sig, map[string]Variable{"s": {Name: "s", Kind: "custom", Options: []string{"checkout"}}}, source) + if err != nil { + t.Fatal(err) + } + return f +} diff --git a/internal/queryrows/trace_candidates.go b/internal/queryrows/trace_candidates.go new file mode 100644 index 000000000..8d8edb792 --- /dev/null +++ b/internal/queryrows/trace_candidates.go @@ -0,0 +1,22 @@ +package queryrows + +// CompletedTraceQuery ranks scoped traces from completed batch candidates and +// the uncached/boundary tail. Arguments are start, end, namespace twice, service +// twice, and limit. Both top-N stages must use the same limit. Results are +// namespace, trace_id, and the selected scope's has_error flag. +const CompletedTraceQuery = `WITH tail AS ( + SELECT namespace,trace_id,min(start_unix_nano) AS min_start,max(end_unix_nano) AS max_end, + max(CASE WHEN status = 'STATUS_CODE_ERROR' THEN 1 ELSE 0 END) AS has_error + FROM trace_tail + WHERE start_time>=$1::TIMESTAMP_NS::TIMESTAMPTZ_NS AND start_time<$2::TIMESTAMP_NS::TIMESTAMPTZ_NS + AND ($3='' OR namespace=$4) AND ($5='' OR service=$6) AND trace_id<>'' GROUP BY namespace,trace_id +), untouched AS ( + SELECT namespace,trace_id,min_start,max_end,has_error FROM trace_candidates + WHERE (namespace,trace_id) NOT IN (SELECT namespace,trace_id FROM tail) + ORDER BY has_error DESC,max_end-min_start DESC,trace_id,namespace LIMIT $7 +), touched AS ( + SELECT t.namespace,t.trace_id, least(t.min_start,c.min_start) AS min_start,greatest(t.max_end,c.max_end) AS max_end, + greatest(t.has_error,c.has_error) AS has_error + FROM tail t LEFT JOIN trace_candidates c ON t.namespace=c.namespace AND t.trace_id=c.trace_id +), candidates AS (SELECT * FROM untouched UNION ALL SELECT * FROM touched) +SELECT namespace,trace_id,has_error FROM candidates ORDER BY has_error DESC,max_end-min_start DESC,trace_id,namespace LIMIT $7` diff --git a/internal/telemetry/parquet.go b/internal/telemetry/parquet.go index a32775fc7..9bf660c8c 100644 --- a/internal/telemetry/parquet.go +++ b/internal/telemetry/parquet.go @@ -97,11 +97,13 @@ type TimeRange struct { } type TraceQuery struct { - TraceID string - Namespace string - StartNanos int64 - EndNanos int64 - Limit int + TraceID string + Namespace string + // ExactNamespace matches Namespace exactly, including an empty namespace. + ExactNamespace bool + StartNanos int64 + EndNanos int64 + Limit int } type storedBatch struct { @@ -506,6 +508,9 @@ func (p *ParquetStore) RestoreRetiredInputs(inputs []string, replacementID strin // bounded by Limit. Deriving them here removes a second pass over every batch // file -- the thing that made trace_detail pay for its own correctness. type TraceTotals struct { + // Root prefers a parentless span, then start time and span ID, across all + // matching indexed ranges, independently of the detail page limit. + Root *IndexedSpan Spans int Services int DurationMS float64 @@ -595,15 +600,50 @@ func (r indexedSpanRow) span() IndexedSpan { } func readIndexedTrace(ctx context.Context, batch *storedBatch, match traceRange, query TraceQuery, selected *indexedSpanHeap, totals *TraceTotals, services map[string]struct{}) (err error) { - if match.row > uint64(math.MaxInt64) { - return errors.New("trace index row exceeds Parquet reader limit") - } file, err := os.Open(filepath.Join(batch.dir, "spans.parquet")) if err != nil { return err } reader := parquet.NewGenericReader[indexedSpanRow](file) defer func() { err = errors.Join(err, reader.Close(), file.Close()) }() + return readIndexedRange(ctx, reader, match, query.EndNanos, func(row indexedSpanRow) { + if row.TraceID != query.TraceID || row.StartUnixNano < query.StartNanos || row.StartUnixNano >= query.EndNanos || ((query.ExactNamespace || query.Namespace != "") && row.Namespace != query.Namespace) { + return + } + accumulateTraceRow(row, totals, services) + selected.Add(row.span(), query.Limit) + }) +} + +func accumulateTraceRow(row indexedSpanRow, totals *TraceTotals, services map[string]struct{}) { + // Counted before the heap, so the totals describe the trace while + // the heap keeps describing the page. + totals.Spans++ + if row.Service != "" { + services[row.Service] = struct{}{} + } + if row.Status == "STATUS_CODE_ERROR" { + totals.HasError = true + } + if totals.MinStartNano == 0 || row.StartUnixNano < totals.MinStartNano { + totals.MinStartNano = row.StartUnixNano + } + if end := row.StartUnixNano + int64(row.DurationMS*float64(time.Millisecond)); end > totals.MaxEndNano { + totals.MaxEndNano = end + } + span := row.span() + if totals.Root == nil { + totals.Root = new(IndexedSpan) + } + if totals.Spans == 1 || indexedRootEarlier(span, *totals.Root) { + *totals.Root = span + } +} + +func readIndexedRange(ctx context.Context, reader *parquet.GenericReader[indexedSpanRow], match traceRange, end int64, visit func(indexedSpanRow)) error { + if match.row > uint64(math.MaxInt64) { + return errors.New("trace index row exceeds Parquet reader limit") + } if err := reader.SeekToRow(int64(match.row)); err != nil { return err } @@ -613,41 +653,113 @@ func readIndexedTrace(ctx context.Context, batch *storedBatch, match traceRange, if err := ctx.Err(); err != nil { return err } - want := min(uint64(len(buffer)), remaining) - n, readErr := reader.Read(buffer[:int(want)]) - if readErr != nil && !errors.Is(readErr, io.EOF) { - return readErr + n, err := reader.Read(buffer[:int(min(uint64(len(buffer)), remaining))]) + if err != nil && !errors.Is(err, io.EOF) { + return err } if n == 0 { return io.ErrUnexpectedEOF } - for i := range n { - row := buffer[i] - if row.StartUnixNano >= query.EndNanos { + for _, row := range buffer[:n] { + // Rows within one hash range are ordered by event time. + if row.StartUnixNano >= end { return nil } - if row.TraceID != query.TraceID || row.StartUnixNano < query.StartNanos || - (query.Namespace != "" && row.Namespace != query.Namespace) { - continue - } - // Counted before the heap, so the totals describe the trace while - // the heap keeps describing the page. - totals.Spans++ - if row.Service != "" { - services[row.Service] = struct{}{} - } - if row.Status == "STATUS_CODE_ERROR" { - totals.HasError = true - } - if totals.MinStartNano == 0 || row.StartUnixNano < totals.MinStartNano { - totals.MinStartNano = row.StartUnixNano + visit(row) + } + remaining -= uint64(n) + } + return nil +} + +// TraceIdentity keeps equal trace IDs in different namespaces separate. +type TraceIdentity struct{ Namespace, TraceID string } + +// TraceSummaries reads bounded identities in one batch pass, opening each index +// and matching span file once. Hash collisions are resolved by exact identity. +func (p *ParquetStore) TraceSummaries(ctx context.Context, ids []TraceIdentity, start, end int64) ([]TraceTotals, error) { + if len(ids) > 20 { + return nil, errors.New("trace summaries limit exceeds 20") + } + if start >= end { + return nil, errors.New("trace query time range must be positive") + } + totals := make([]TraceTotals, len(ids)) + positions := make(map[TraceIdentity]int, len(ids)) + services := make([]map[string]struct{}, len(ids)) + hashes := make([]uint64, 0, len(ids)) + for i, id := range ids { + positions[id] = i + services[i] = map[string]struct{}{} + hashes = append(hashes, xxh3.HashString(id.TraceID)) + } + slices.Sort(hashes) + hashes = slices.Compact(hashes) + p.mu.RLock() + batches := make([]*storedBatch, 0, len(p.batches)) + for _, batch := range p.batches { + batches = append(batches, batch) + } + p.mu.RUnlock() + for _, batch := range batches { + if err := ctx.Err(); err != nil { + return nil, err + } + if bounds := batch.metadata.SpanTime; bounds.Known && (bounds.MaxNanos < start || bounds.MinNanos >= end) { + continue + } + if err := readBatchTraceSummaries(ctx, batch, hashes, end, func(row indexedSpanRow) { + if row.StartUnixNano < start || row.StartUnixNano >= end { + return } - if end := row.StartUnixNano + int64(row.DurationMS*float64(time.Millisecond)); end > totals.MaxEndNano { - totals.MaxEndNano = end + if i, ok := positions[TraceIdentity{row.Namespace, row.TraceID}]; ok { + accumulateTraceRow(row, &totals[i], services[i]) } - selected.Add(row.span(), query.Limit) + }); err != nil { + return nil, err + } + } + for i := range totals { + totals[i].Services = len(services[i]) + if totals[i].Spans > 0 && totals[i].MaxEndNano > totals[i].MinStartNano { + totals[i].DurationMS = float64(totals[i].MaxEndNano-totals[i].MinStartNano) / float64(time.Millisecond) + } + } + return totals, nil +} + +func readBatchTraceSummaries(ctx context.Context, batch *storedBatch, hashes []uint64, end int64, visit func(indexedSpanRow)) (err error) { + if len(hashes) == 0 || batch.traces.entries == 0 { + return nil + } + index, err := os.Open(batch.traces.path) + if err != nil { + return err + } + defer func() { err = errors.Join(err, index.Close()) }() + var matches []traceRange + for _, hash := range hashes { + match, found, err := batch.traces.lookup(index, hash) + if err != nil { + return err + } + if found { + matches = append(matches, match) + } + } + if len(matches) == 0 { + return nil + } + file, err := os.Open(filepath.Join(batch.dir, "spans.parquet")) + if err != nil { + return err + } + reader := parquet.NewGenericReader[indexedSpanRow](file) + defer func() { err = errors.Join(err, reader.Close(), file.Close()) }() + for _, match := range matches { + if err := readIndexedRange(ctx, reader, match, end, visit); err != nil { + return err } - remaining -= uint64(n) } return nil } @@ -680,6 +792,19 @@ func (h *indexedSpanHeap) Add(span IndexedSpan, limit int) { } } +func indexedRootEarlier(left, right IndexedSpan) bool { + if (left.ParentSpanID == "") != (right.ParentSpanID == "") { + return left.ParentSpanID == "" + } + if left.StartUnixNanos != right.StartUnixNanos { + return left.StartUnixNanos < right.StartUnixNanos + } + if left.SpanID != right.SpanID { + return left.SpanID < right.SpanID + } + return left.Namespace < right.Namespace +} + func indexedSpanEarlier(left, right IndexedSpan) bool { if left.StartUnixNanos != right.StartUnixNanos { return left.StartUnixNanos < right.StartUnixNanos diff --git a/internal/telemetry/trace_index.go b/internal/telemetry/trace_index.go index f2b2814cb..f65b7ce49 100644 --- a/internal/telemetry/trace_index.go +++ b/internal/telemetry/trace_index.go @@ -216,6 +216,10 @@ func (index traceIndex) Lookup(hash uint64) (match traceRange, found bool, err e return traceRange{}, false, err } defer func() { err = errors.Join(err, file.Close()) }() + return index.lookup(file, hash) +} + +func (index traceIndex) lookup(file *os.File, hash uint64) (traceRange, bool, error) { low, high := uint64(0), index.entries for low < high { middle := low + (high-low)/2 diff --git a/internal/telemetry/trace_summaries_test.go b/internal/telemetry/trace_summaries_test.go new file mode 100644 index 000000000..9494243a5 --- /dev/null +++ b/internal/telemetry/trace_summaries_test.go @@ -0,0 +1,77 @@ +package telemetry + +import ( + "fmt" + "path/filepath" + "testing" + "time" + + "github.com/parquet-go/parquet-go" + "github.com/zeebo/xxh3" +) + +func TestTraceSummariesResolveCollisionsNamespacesAndWindowBounds(t *testing.T) { + store, err := OpenParquetStore(t.TempDir()) + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { _ = store.Close() }) + start := int64(time.Second) + end := start + int64(10*time.Millisecond) + hash := xxh3.HashString("shared") + for i := range 3 { + dir := t.TempDir() + n := start + int64(i)*int64(time.Millisecond) + parent := "root" + if i == 2 { + parent = "" + } + status := "STATUS_CODE_OK" + if i == 1 { + status = "STATUS_CODE_ERROR" + } + rows := []indexedSpanRow{ + {TraceID: "shared", SpanID: "before", StartUnixNano: start - 1, Status: "STATUS_CODE_ERROR", DurationMS: 100}, + {TraceID: "shared", SpanID: fmt.Sprint(i), ParentSpanID: parent, Service: fmt.Sprint(i % 2), StartUnixNano: n, DurationMS: 1}, + {Namespace: "b", TraceID: "shared", SpanID: fmt.Sprint(i), Service: "worker", StartUnixNano: n + 100, DurationMS: 3, Status: status}, + {TraceID: "collision", SpanID: "wrong", Service: "wrong", StartUnixNano: n + 200, DurationMS: 100, Status: "STATUS_CODE_ERROR"}, + {TraceID: "shared", SpanID: "at-end", StartUnixNano: end, DurationMS: 100, Status: "STATUS_CODE_ERROR"}, + {Namespace: "b", TraceID: "shared", SpanID: "after-end", StartUnixNano: end + 1, DurationMS: 100}, + } + if err := parquet.WriteFile(filepath.Join(dir, "spans.parquet"), rows); err != nil { + t.Fatal(err) + } + // Inject a collision: both IDs occupy one time-ordered hash range. This + // avoids a production hash hook and exercises the public summary read. + hashes := make([]uint64, len(rows)) + for j := range hashes { + hashes[j] = hash + } + path := filepath.Join(dir, "trace.fidx") + if err := writeTraceIndex(path, hashes); err != nil { + t.Fatal(err) + } + index, err := loadTraceIndex(path, len(rows)) + if err != nil { + t.Fatal(err) + } + store.batches[fmt.Sprint(i)] = &storedBatch{dir: dir, traces: index} + } + ids := []TraceIdentity{{Namespace: "", TraceID: "shared"}, {Namespace: "b", TraceID: "shared"}, {Namespace: "missing", TraceID: "shared"}, {TraceID: "absent"}} + got, err := store.TraceSummaries(t.Context(), ids, start, end) + if err != nil { + t.Fatal(err) + } + a, b := got[0], got[1] + if a.Spans != 3 || a.Services != 2 || a.HasError || a.DurationMS != 3 || a.MinStartNano != start || a.MaxEndNano != start+int64(3*time.Millisecond) || a.Root == nil || a.Root.SpanID != "2" || a.Root.TraceID != "shared" || a.Root.Namespace != "" { + t.Fatalf("empty namespace: %+v root=%+v", a, a.Root) + } + if b.Spans != 3 || b.Services != 1 || !b.HasError || b.DurationMS != 5 || b.MinStartNano != start+100 || b.MaxEndNano != start+100+int64(5*time.Millisecond) || b.Root == nil || b.Root.SpanID != "0" || b.Root.Namespace != "b" { + t.Fatalf("namespace b: %+v root=%+v", b, b.Root) + } + for _, missing := range got[2:] { + if missing != (TraceTotals{}) { + t.Fatalf("missing identity: %+v", missing) + } + } +}