Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
39 changes: 19 additions & 20 deletions internal/engine/batch_cache.go
Original file line number Diff line number Diff line change
Expand Up @@ -25,7 +25,7 @@ import (
// anomaly_log is not one of them: it holds detector output that nothing can
// rebuild, so the drop keeps it. Its table is created once; any change to its
// schema must be a separate, explicit migration step.
const readCacheVersion = 17
const readCacheVersion = 18

const (
readCacheBatchLimit = 64
Expand Down Expand Up @@ -280,7 +280,7 @@ func (d *Duck) cacheBatches(ctx context.Context, batches []telemetry.BatchMetada
return err
}
if len(changed) > 0 {
if err := exec(`CREATE OR REPLACE TEMP TABLE cache_retired_traces AS SELECT DISTINCT trace_id FROM read_trace_parts WHERE batch_id IN (` + idsSQL(changed) + `)`); err != nil {
if err := exec(`CREATE OR REPLACE TEMP TABLE cache_retired_traces AS SELECT DISTINCT namespace,trace_id FROM read_trace_parts WHERE batch_id IN (` + idsSQL(changed) + `)`); err != nil {
return 0, err
}
}
Expand All @@ -294,10 +294,10 @@ func (d *Duck) cacheBatches(ctx context.Context, batches []telemetry.BatchMetada
if len(changed) > 0 {
// Retention can change a trace's extrema or error flag. Compaction whose
// complete replacement is cached preserves them and bypasses this repair.
if err := exec(`DELETE FROM read_trace_candidates WHERE trace_id IN (SELECT trace_id FROM cache_retired_traces) AND trace_id NOT IN (SELECT trace_id FROM read_trace_parts)`); err != nil {
if err := exec(`DELETE FROM read_trace_candidates WHERE (namespace,trace_id) IN (SELECT namespace,trace_id FROM cache_retired_traces) AND (namespace,trace_id) NOT IN (SELECT namespace,trace_id FROM read_trace_parts)`); err != nil {
return 0, err
}
if err := exec(`MERGE INTO read_trace_candidates c USING (` + traceCandidateSelect + ` FROM read_trace_parts WHERE trace_id IN (SELECT trace_id FROM cache_retired_traces) GROUP BY trace_id) AS excluded ON c.trace_id=excluded.trace_id
if err := exec(`MERGE INTO read_trace_candidates c USING (` + traceCandidateSelect + ` FROM read_trace_parts WHERE (namespace,trace_id) IN (SELECT namespace,trace_id FROM cache_retired_traces) GROUP BY namespace,trace_id) AS excluded ON c.namespace=excluded.namespace AND c.trace_id=excluded.trace_id
WHEN MATCHED THEN UPDATE SET namespace=excluded.namespace,service=excluded.service,min_start=excluded.min_start,max_start=excluded.max_start,max_end=excluded.max_end,has_error=excluded.has_error
WHEN NOT MATCHED THEN INSERT ` + traceCandidateColumns); err != nil {
return 0, err
Expand Down Expand Up @@ -359,10 +359,9 @@ func (d *Duck) cacheBatches(ctx context.Context, batches []telemetry.BatchMetada
if err := exec(`INSERT INTO read_trace_parts SELECT * FROM cache_new_trace_parts`); err != nil {
return 0, err
}
candidates := traceCandidateSelect + ` FROM cache_new_trace_parts GROUP BY trace_id`
if err := exec(`MERGE INTO read_trace_candidates c USING (` + candidates + `) AS excluded ON c.trace_id=excluded.trace_id
candidates := traceCandidateSelect + ` FROM cache_new_trace_parts GROUP BY namespace,trace_id`
if err := exec(`MERGE INTO read_trace_candidates c USING (` + candidates + `) AS excluded ON c.namespace=excluded.namespace AND c.trace_id=excluded.trace_id
WHEN MATCHED THEN UPDATE SET
namespace=CASE WHEN c.namespace=excluded.namespace THEN excluded.namespace END,
service=CASE WHEN c.service=excluded.service THEN excluded.service END,
min_start=least(c.min_start,excluded.min_start),
max_start=greatest(c.max_start,excluded.max_start),
Expand Down Expand Up @@ -466,18 +465,18 @@ func (d *Duck) aggregateSources(ctx context.Context, db snapshotSQL, batches []t
}
interior := fmt.Sprintf("min_start>=%d AND max_start<%d", w.Start.UnixNano(), w.End.UnixNano())
if len(partialFiles) > 0 {
tail += " UNION ALL SELECT " + projection + " FROM (" + cleanSource("spans", d.snapshotSource("spans", partialFiles, batches)) + ") WHERE " + windowPredicate("start_time", w) + " AND trace_id IN (SELECT trace_id FROM (" + partial + "))"
tail += " UNION ALL SELECT " + projection + " FROM (" + cleanSource("spans", d.snapshotSource("spans", partialFiles, batches)) + ") WHERE " + windowPredicate("start_time", w) + " AND (namespace,trace_id) IN (SELECT namespace,trace_id FROM (" + partial + "))"
// A partial trace may also have in-window contributions in other batches.
// Include these parts as candidates; boundary rows merge them in the kernel.
sources["trace_candidates"] = `SELECT trace_id,min(min_start) AS min_start,max(max_end) AS max_end,max(has_error) AS has_error FROM read_trace_parts WHERE ` + scope + ` AND ` + interior + ` GROUP BY trace_id`
sources["trace_candidates"] = `SELECT namespace,trace_id,min(min_start) AS min_start,max(max_end) AS max_end,max(has_error) AS has_error FROM read_trace_parts WHERE ` + scope + ` AND ` + interior + ` GROUP BY namespace,trace_id`
} else {
// Every overlapping cached file lies inside the window, so a partial
// trace's other spans are in files outside it. Its in-window parts are
// still a candidate; the kernel needs one row per trace, and partial
// traces are disjoint from the fully contained index rows.
sources["trace_candidates"] = "SELECT trace_id,min_start,max_end,has_error FROM (" + index + ") WHERE " + full +
` UNION ALL SELECT trace_id,min(min_start),max(max_end),max(has_error) FROM read_trace_parts WHERE ` + scope + ` AND ` + interior +
` AND trace_id IN (SELECT trace_id FROM (` + partial + `)) GROUP BY trace_id`
sources["trace_candidates"] = "SELECT namespace,trace_id,min_start,max_end,has_error FROM (" + index + ") WHERE " + full +
` UNION ALL SELECT namespace,trace_id,min(min_start),max(max_end),max(has_error) FROM read_trace_parts WHERE ` + scope + ` AND ` + interior +
` AND (namespace,trace_id) IN (SELECT namespace,trace_id FROM (` + partial + `)) GROUP BY namespace,trace_id`
}
sources["trace_tail"] = tail
return nil
Expand Down Expand Up @@ -594,10 +593,10 @@ func (d *Duck) transferBatchCacheLocked(ctx context.Context, output string, inpu
// million traces that alone outgrew a 1 GB memory limit.
const traceCandidateColumns = `VALUES (excluded.trace_id,excluded.namespace,excluded.service,excluded.min_start,excluded.max_start,excluded.max_end,excluded.has_error)`

// One candidate row per trace is sufficient while the requested namespace and
// service are unambiguous. Mixed scopes are NULL and use exact batch parts.
// One candidate per namespace and trace. Mixed services are NULL and use exact
// batch parts for a service selection.
const traceCandidateSelect = `SELECT trace_id,
CASE WHEN min(namespace)=max(namespace) THEN min(namespace) END AS namespace,
namespace,
CASE WHEN min(service)=max(service) THEN min(service) END AS service,
min(min_start) AS min_start,max(max_start) AS max_start,max(max_end) AS max_end,max(has_error) AS has_error`

Expand Down Expand Up @@ -666,8 +665,8 @@ func (d *Duck) traceCandidateSource(ctx context.Context, db snapshotSQL, batches
}
index := `SELECT * FROM read_trace_candidates`
if len(changed) > 0 {
dirty := `SELECT trace_id FROM read_trace_parts WHERE batch_id IN (` + idsSQL(changed) + `)`
index += ` WHERE trace_id NOT IN (` + dirty + `) UNION ALL ` + traceCandidateSelect + ` FROM read_trace_parts WHERE batch_id IN (` + idsSQL(cachedIDs) + `) AND trace_id IN (` + dirty + `) GROUP BY trace_id`
dirty := `SELECT namespace,trace_id FROM read_trace_parts WHERE batch_id IN (` + idsSQL(changed) + `)`
index += ` WHERE (namespace,trace_id) NOT IN (` + dirty + `) UNION ALL ` + traceCandidateSelect + ` FROM read_trace_parts WHERE batch_id IN (` + idsSQL(cachedIDs) + `) AND (namespace,trace_id) IN (` + dirty + `) GROUP BY namespace,trace_id`
}
var exact, possible, ambiguous []string
for _, field := range []struct{ name, value string }{{"namespace", w.Namespace}, {"service", w.Service}} {
Expand All @@ -679,12 +678,12 @@ func (d *Duck) traceCandidateSource(ctx context.Context, db snapshotSQL, batches
possible = append(possible, "("+match+" OR "+field.name+" IS NULL)")
ambiguous = append(ambiguous, field.name+" IS NULL")
}
projection := `trace_id,min_start,max_start,max_end,has_error`
projection := `namespace,trace_id,min_start,max_start,max_end,has_error`
if len(exact) == 0 {
return "SELECT " + projection + " FROM (" + index + ")", nil
}
result := "SELECT " + projection + " FROM (" + index + ") WHERE " + strings.Join(exact, " AND ")
mixed := "SELECT trace_id FROM (" + index + ") WHERE " + strings.Join(possible, " AND ") + " AND (" + strings.Join(ambiguous, " OR ") + ")"
mixed := "SELECT namespace,trace_id FROM (" + index + ") WHERE " + strings.Join(possible, " AND ") + " AND (" + strings.Join(ambiguous, " OR ") + ")"
// Most traces belong to a single namespace/service. Do not put a retained
// batch-parts aggregation in the ranking/join plan when this snapshot has no
// ambiguous candidates for the requested scope. The probe shares its read
Expand All @@ -697,7 +696,7 @@ func (d *Duck) traceCandidateSource(ctx context.Context, db snapshotSQL, batches
return result, nil
}
scope := "batch_id IN (" + idsSQL(cachedIDs) + ") AND " + strings.Join(exact, " AND ")
return result + ` UNION ALL SELECT trace_id,min(min_start),max(max_start),max(max_end),max(has_error) FROM read_trace_parts WHERE ` + scope + ` AND trace_id IN (` + mixed + `) GROUP BY trace_id`, nil
return result + ` UNION ALL SELECT namespace,trace_id,min(min_start),max(max_start),max(max_end),max(has_error) FROM read_trace_parts WHERE ` + scope + ` AND (namespace,trace_id) IN (` + mixed + `) GROUP BY namespace,trace_id`, nil
}

func minTime(a, b time.Time) time.Time {
Expand Down
7 changes: 7 additions & 0 deletions internal/engine/duck.go
Original file line number Diff line number Diff line change
Expand Up @@ -2382,6 +2382,13 @@ func (d *Duck) Trace(ctx context.Context, query telemetry.TraceQuery) ([]telemet
return d.repository.Parquet.Trace(ctx, query)
}

// TraceSummaries pins one snapshot for the bounded multi-trace root pass.
func (d *Duck) TraceSummaries(ctx context.Context, ids []telemetry.TraceIdentity, start, end int64) ([]telemetry.TraceTotals, error) {
pin := d.pinParquet()
defer d.unpinParquet(pin)
return d.repository.Parquet.TraceSummaries(ctx, ids, start, end)
}

// PublishParquet runs one catalog swap. Readers are never excluded or waited
// for: the swap is atomic under the catalog lock, and what it replaces stays
// on disk until no pinned reader can open it. The budget only bounds the
Expand Down
46 changes: 46 additions & 0 deletions internal/engine/file_snapshot_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -701,3 +701,49 @@ func TestReadCacheBudgetsAllSignalsAndReportsInventoryWork(t *testing.T) {
t.Fatalf("remaining batch: %v %v", markers, err)
}
}

func TestRetirementRepairsOnlyMatchingNamespaceIdentity(t *testing.T) {
d, repo := newBatchCacheTest(t)
at := time.Now().UTC().Truncate(time.Hour)
old, now := at.Add(-2*time.Hour).UnixNano(), at.UnixNano()
for _, batch := range []telemetrystore.Batch{
{ID: "retire-a", Spans: []telemetry.Span{{Namespace: "a", TraceID: "shared", ServiceName: "retired", StartUnixNanos: old, EndUnixNanos: old + 1, IngestedAt: old, StatusCode: "STATUS_CODE_ERROR"}}},
{ID: "keep-a", Spans: []telemetry.Span{{Namespace: "a", TraceID: "shared", ServiceName: "api", StartUnixNanos: now, EndUnixNanos: now + 10, IngestedAt: now}}},
{ID: "keep-b", Spans: []telemetry.Span{{Namespace: "b", TraceID: "shared", ServiceName: "worker", StartUnixNanos: now + 20, EndUnixNanos: now + 30, IngestedAt: now, StatusCode: "STATUS_CODE_ERROR"}}},
} {
if err := commitRestoredFixture(repo, t.Context(), batch); err != nil {
t.Fatal(err)
}
}
if _, err := d.RefreshReadCaches(t.Context()); err != nil {
t.Fatal(err)
}
if count, err := repo.PruneParquet(t.Context(), d, at.Add(-time.Hour).UnixNano(), 8); err != nil || count != 1 {
t.Fatalf("prune=%d: %v", count, err)
}
if _, err := d.RefreshReadCaches(t.Context()); err != nil {
t.Fatal(err)
}
for _, want := range []struct {
namespace, service string
start, end int64
hasError int
}{
{"a", "api", now, now + 10, 0},
{"b", "worker", now + 20, now + 30, 1},
} {
var service string
var start, maxStart, end int64
var hasError int
if err := d.DB.QueryRow(`SELECT service,min_start,max_start,max_end,has_error FROM read_trace_candidates WHERE namespace=? AND trace_id='shared'`, want.namespace).Scan(&service, &start, &maxStart, &end, &hasError); err != nil {
t.Fatal(err)
}
if service != want.service || start != want.start || maxStart != want.start || end != want.end || hasError != want.hasError {
t.Fatalf("namespace=%s service=%s bounds=%d/%d/%d error=%d", want.namespace, service, start, maxStart, end, hasError)
}
}
var count int
if err := d.DB.QueryRow(`SELECT count(*) FROM read_trace_candidates WHERE trace_id='shared'`).Scan(&count); err != nil || count != 2 {
t.Fatalf("candidates=%d: %v", count, err)
}
}
4 changes: 2 additions & 2 deletions internal/observability/batch_reads_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -34,9 +34,9 @@ func newCompletedReadTest(t *testing.T) (*engine.Duck, *telemetrystore.Repositor
const rawTraceQuery = `SELECT trace_id FROM spans
WHERE start_time >= ?::TIMESTAMP_NS::TIMESTAMPTZ_NS AND start_time < ?::TIMESTAMP_NS::TIMESTAMPTZ_NS
AND (? = '' OR namespace = ?) AND trace_id <> '' AND (? = '' OR service = ?)
GROUP BY trace_id
GROUP BY namespace,trace_id
ORDER BY MAX(CASE WHEN status = 'STATUS_CODE_ERROR' THEN 1 ELSE 0 END) DESC,
MAX(end_unix_nano) - MIN(start_unix_nano) DESC, trace_id ASC
MAX(end_unix_nano) - MIN(start_unix_nano) DESC, trace_id ASC, namespace ASC
LIMIT 1`

func rawTraceID(t testing.TB, d *engine.Duck, scope Scope, service string) (string, error) {
Expand Down
18 changes: 9 additions & 9 deletions internal/observability/service_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -186,9 +186,9 @@ func TestTraceSelectsRecentErrorAndCorrelatesLogs(t *testing.T) {
svc, mock, repository := newMockService(t)
start := time.Date(2026, 7, 20, 11, 0, 0, 0, time.UTC)
end := start.Add(time.Hour)
mock.ExpectQuery(regexp.QuoteMeta(completedTraceQuery)).
WithArgs(start, end, "prod", "prod", "checkout", "checkout").
WillReturnRows(sqlmock.NewRows([]string{"trace_id"}).AddRow("trace-1"))
mock.ExpectQuery(regexp.QuoteMeta(queryrows.CompletedTraceQuery)).
WithArgs(start, end, "prod", "prod", "checkout", "checkout", 1).
WillReturnRows(sqlmock.NewRows([]string{"namespace", "trace_id", "has_error"}).AddRow("prod", "trace-1", 1))
if err := repository.Commit(context.Background(), telemetrystore.Batch{ID: "trace-fixture", Spans: []telemetry.Span{
{Namespace: "prod", TraceID: "trace-1", SpanID: "root", ServiceName: "checkout", Name: "POST /pay", Kind: "SERVER", StartUnixNanos: start.UnixNano(), DurationMS: 200, StatusCode: "STATUS_CODE_ERROR", StatusMsg: "declined"},
{Namespace: "prod", TraceID: "trace-1", SpanID: "child", ParentSpanID: "root", ServiceName: "payments", Name: "charge", Kind: "CLIENT", StartUnixNanos: start.Add(20 * time.Millisecond).UnixNano(), DurationMS: 80, StatusCode: "OK"},
Expand All @@ -199,7 +199,7 @@ func TestTraceSelectsRecentErrorAndCorrelatesLogs(t *testing.T) {
t.Fatalf("commit trace fixture: %v", err)
}
mock.ExpectQuery(regexp.QuoteMeta(traceLogsQuery)).
WithArgs("trace-1", start, end, "prod", "prod", 20).
WithArgs("trace-1", start, end, true, "prod", 20).
WillReturnRows(sqlmock.NewRows([]string{"time", "severity", "service", "body", "trace_id", "span_id"}).
AddRow(start.Add(150*time.Millisecond), "ERROR", "checkout", "payment declined", "trace-1", "root").
AddRow(start.Add(160*time.Millisecond), "ERROR", "payments", `charge failed: token=abc123 {"client_secret":"cs_live_9"}`, "trace-1", "child"))
Expand Down Expand Up @@ -237,7 +237,7 @@ func TestTraceLogsUseFullScopeEventTimeAcrossBatches(t *testing.T) {
}
}
mock.ExpectQuery(regexp.QuoteMeta(traceLogsQuery)).
WithArgs("trace-order", start, start.Add(time.Hour), "prod", "prod", 10).
WithArgs("trace-order", start, start.Add(time.Hour), true, "prod", 10).
WillReturnRows(sqlmock.NewRows([]string{"time", "severity", "service", "body", "trace_id", "span_id"}).
AddRow(start.Add(10*time.Millisecond), "", "", "earliest", "trace-order", "").
AddRow(start.Add(20*time.Millisecond), "", "", "middle", "trace-order", "").
Expand All @@ -263,7 +263,7 @@ func TestTraceUsesIndexedParquet(t *testing.T) {
t.Fatal(err)
}
mock.ExpectQuery(regexp.QuoteMeta(traceLogsQuery)).
WithArgs("parquet-trace", start, end, "prod", "prod", 10).
WithArgs("parquet-trace", start, end, true, "prod", 10).
WillReturnRows(sqlmock.NewRows([]string{"time", "severity", "service", "body", "trace_id", "span_id"}).
AddRow(start.Add(time.Millisecond), "ERROR", "checkout", "token=secret", "parquet-trace", "root"))
result, err := svc.Trace(context.Background(), Scope{Namespace: "prod", Start: start, End: end}, "parquet-trace", "", 10)
Expand Down Expand Up @@ -295,7 +295,7 @@ func TestTraceCombinesIndexedSpansAcrossBatches(t *testing.T) {
t.Fatal(err)
}
mock.ExpectQuery(regexp.QuoteMeta(traceLogsQuery)).
WithArgs("split-trace", start, end, "prod", "prod", 10).
WithArgs("split-trace", start, end, true, "prod", 10).
WillReturnRows(sqlmock.NewRows([]string{"time", "severity", "service", "body", "trace_id", "span_id"}))
result, err := svc.Trace(context.Background(), Scope{Namespace: "prod", Start: start, End: end}, "split-trace", "", 10)
if err != nil {
Expand All @@ -321,7 +321,7 @@ func TestTraceReadsRecentRootFromParquetIndex(t *testing.T) {
t.Fatal(err)
}
mock.ExpectQuery(regexp.QuoteMeta(traceLogsQuery)).
WithArgs("new-trace", start, end, "prod", "prod", 10).
WithArgs("new-trace", start, end, true, "prod", 10).
WillReturnRows(sqlmock.NewRows([]string{"time", "severity", "service", "body", "trace_id", "span_id"}))
result, err := svc.Trace(context.Background(), Scope{Namespace: "prod", Start: start, End: end}, "new-trace", "", 10)
if err != nil {
Expand All @@ -347,7 +347,7 @@ func TestTraceFiltersIndexedSpansByNamespace(t *testing.T) {
t.Fatal(err)
}
mock.ExpectQuery(regexp.QuoteMeta(traceLogsQuery)).
WithArgs("shared-trace", start, end, "prod", "prod", 10).
WithArgs("shared-trace", start, end, true, "prod", 10).
WillReturnRows(sqlmock.NewRows([]string{"time", "severity", "service", "body", "trace_id", "span_id"}))
result, err := svc.Trace(context.Background(), Scope{Namespace: "prod", Start: start, End: end}, "shared-trace", "", 10)
if err != nil {
Expand Down
Loading
Loading