diff --git a/internal/engine/annotations.go b/internal/engine/annotations.go index 85f672ee..1cf9df3e 100644 --- a/internal/engine/annotations.go +++ b/internal/engine/annotations.go @@ -6,6 +6,7 @@ import ( "errors" "fmt" "log/slog" + "slices" "sort" "strings" "time" @@ -142,7 +143,8 @@ func (d *Duck) refreshVersionRollup(ctx context.Context, waitBudget time.Duratio if err != nil { return 0, false, err } - batches := d.repository.Parquet.BatchMetadata() + // BatchMetadata returns a shared, read-only snapshot; clone it before sorting. + batches := slices.Clone(d.repository.Parquet.BatchMetadata()) // Compaction generations must not put old unprocessed outputs behind new ingest. sort.Slice(batches, func(i, j int) bool { if batches[i].MinIngestedNanos != batches[j].MinIngestedNanos { diff --git a/internal/engine/batch_cache.go b/internal/engine/batch_cache.go index 855f8ede..e22b8234 100644 --- a/internal/engine/batch_cache.go +++ b/internal/engine/batch_cache.go @@ -5,6 +5,7 @@ import ( "database/sql" "fmt" "log/slog" + "slices" "sort" "strings" "time" @@ -193,7 +194,8 @@ func (d *Duck) refreshReadCaches(ctx context.Context) (rows int64, backlog bool, if err := d.applyCacheTransfers(ctx); err != nil { return 0, true, err } - batches := d.repository.Parquet.BatchMetadata() + // BatchMetadata returns a shared, read-only snapshot; clone it before sorting. + batches := slices.Clone(d.repository.Parquet.BatchMetadata()) markers, err := batchMarkers(ctx, d.writer()) if err != nil { return 0, false, err diff --git a/internal/engine/duck.go b/internal/engine/duck.go index 8bb93d7a..9876a64d 100644 --- a/internal/engine/duck.go +++ b/internal/engine/duck.go @@ -1613,24 +1613,31 @@ WHERE bucket >= ?::TIMESTAMP_NS::TIMESTAMPTZ_NS // Aggregate each bounded signal scan before the final affected-bucket joins. // Schema-first Parquet scans can have a zero cardinality estimate in DuckDB 2; // joining raw events to the affected table can then build a hash table of every -// event. Logs and metrics aggregate before joining. Spans must be filtered to -// the affected keys first: each unaffected service-minute otherwise allocates -// its own t-digest states. A semi-join against service_pass_affected reverses -// into a build of every raw span, so the keys are bound as constants instead. +// event. Spans, logs and metrics are all filtered to the affected keys first, +// bound as a constant list rather than a semi-join against +// service_pass_affected, which reverses into a build of every raw event. A late +// or clock-skewed event pulls the event-time hull back by days; without the key +// filter the log and metric aggregates then scanned and grouped every +// service-minute of that whole hull, blowing memory and freezing the watermark. +// For spans each unaffected service-minute otherwise also allocates its own +// t-digest states. The final join only reads the materialized aggregates. func insertServiceRollup(ctx context.Context, tx rollupSQL, lo, hi time.Time) (sql.Result, error) { - // Materialize only bounded span aggregates. Logs, metrics and the final - // affected-key join run once, preserving empty and NULL-key output rows. - _, err := tx.ExecContext(ctx, `CREATE OR REPLACE TEMP TABLE service_pass_spans ( + for _, create := range []string{ + `CREATE OR REPLACE TEMP TABLE service_pass_spans ( namespace VARCHAR, bucket TIMESTAMP_NS, service VARCHAR, spans BIGINT, served_spans BIGINT, - p50_ms DOUBLE, p95_ms DOUBLE, error_rate DOUBLE)`) - if err != nil { - return nil, err + p50_ms DOUBLE, p95_ms DOUBLE, error_rate DOUBLE)`, + `CREATE OR REPLACE TEMP TABLE service_pass_logs ( + namespace VARCHAR, bucket TIMESTAMP_NS, service VARCHAR, log_count BIGINT)`, + `CREATE OR REPLACE TEMP TABLE service_pass_metrics ( + namespace VARCHAR, bucket TIMESTAMP_NS, service VARCHAR, metric_count BIGINT)`, + } { + if _, err := tx.ExecContext(ctx, create); err != nil { + return nil, err + } } - err = forAffectedKeys(ctx, tx, - `SELECT DISTINCT namespace, epoch_ns(bucket), service FROM service_pass_affected - WHERE namespace IS NOT NULL AND bucket IS NOT NULL AND service IS NOT NULL`, nil, + err := forAffectedKeys(ctx, tx, serviceRollupAffectedKeysSQL, nil, `struct_pack(namespace := s.namespace, bucket := date_trunc('minute', s.start_time::TIMESTAMP_NS), service := s.service)`, - `struct_pack(namespace := ?::VARCHAR, bucket := make_timestamp_ns(?::BIGINT), service := ?::VARCHAR)`, + serviceRollupAffectedKeyExpr, func(keys string, args []any) error { _, err := tx.ExecContext(ctx, "INSERT INTO service_pass_spans "+serviceRollupSpanSQL(keys)+" SELECT * FROM span_agg", append(args, lo, hi)...) if err != nil { @@ -1642,13 +1649,37 @@ func insertServiceRollup(ctx context.Context, tx rollupSQL, lo, hi time.Time) (s if err != nil { return nil, err } - res, err := tx.ExecContext(ctx, "WITH span_agg AS (SELECT * FROM service_pass_spans)"+serviceRollupJoinSQL, lo, hi, lo, hi) + // Logs and metrics aggregate the same affected keys over the same bounded + // event-time range, so one old bucket cannot force a full-hull scan. + err = forAffectedKeys(ctx, tx, serviceRollupAffectedKeysSQL, nil, + `struct_pack(namespace := l.namespace, bucket := date_trunc('minute', l.time::TIMESTAMP_NS), service := l.service)`, + serviceRollupAffectedKeyExpr, + func(keys string, args []any) error { + _, err := tx.ExecContext(ctx, serviceRollupLogInsertSQL(keys), append(args, lo, hi)...) + return err + }) + if err != nil { + return nil, err + } + err = forAffectedKeys(ctx, tx, serviceRollupAffectedKeysSQL, nil, + `struct_pack(namespace := m.namespace, bucket := date_trunc('minute', m.time::TIMESTAMP_NS), service := m.service)`, + serviceRollupAffectedKeyExpr, + func(keys string, args []any) error { + _, err := tx.ExecContext(ctx, serviceRollupMetricInsertSQL(keys), append(args, lo, hi)...) + return err + }) if err != nil { return nil, err } - if _, err := tx.ExecContext(ctx, "DROP TABLE service_pass_spans"); err != nil { + res, err := tx.ExecContext(ctx, serviceRollupJoinSQL) + if err != nil { return nil, err } + for _, table := range []string{"service_pass_spans", "service_pass_logs", "service_pass_metrics"} { + if _, err := tx.ExecContext(ctx, "DROP TABLE "+table); err != nil { + return nil, err + } + } return res, nil } @@ -1710,6 +1741,40 @@ func forAffectedKeys(ctx context.Context, tx rollupSQL, query string, queryArgs return nil } +// serviceRollupAffectedKeysSQL lists the distinct affected service-minutes, and +// serviceRollupAffectedKeyExpr is the constant each scan probes against. Both +// are shared by the span, log and metric aggregates so every signal is filtered +// to the same keys. +const serviceRollupAffectedKeysSQL = `SELECT DISTINCT namespace, epoch_ns(bucket), service FROM service_pass_affected + WHERE namespace IS NOT NULL AND bucket IS NOT NULL AND service IS NOT NULL` + +const serviceRollupAffectedKeyExpr = `struct_pack(namespace := ?::VARCHAR, bucket := make_timestamp_ns(?::BIGINT), service := ?::VARCHAR)` + +// Log and metric aggregates probe the same constant affected-key list and the +// same bounded event-time range as the span path, so one old or clock-skewed +// bucket cannot widen them into a scan and group of the whole hull. Keys are +// distinct, so each affected service-minute lands in exactly one chunk and its +// COUNT and COUNT(DISTINCT name) are exact. +func serviceRollupLogInsertSQL(keys string) string { + return `INSERT INTO service_pass_logs + SELECT l.namespace, date_trunc('minute', l.time::TIMESTAMP_NS), l.service, COUNT(*) + FROM logs l + WHERE ` + keys + ` + AND l.time >= ?::TIMESTAMP_NS::TIMESTAMPTZ_NS + AND l.time < ?::TIMESTAMP_NS::TIMESTAMPTZ_NS + GROUP BY l.namespace, date_trunc('minute', l.time::TIMESTAMP_NS), l.service` +} + +func serviceRollupMetricInsertSQL(keys string) string { + return `INSERT INTO service_pass_metrics + SELECT m.namespace, date_trunc('minute', m.time::TIMESTAMP_NS), m.service, COUNT(DISTINCT m.name) + FROM metrics m + WHERE ` + keys + ` + AND m.time >= ?::TIMESTAMP_NS::TIMESTAMPTZ_NS + AND m.time < ?::TIMESTAMP_NS::TIMESTAMPTZ_NS + GROUP BY m.namespace, date_trunc('minute', m.time::TIMESTAMP_NS), m.service` +} + // Aggregate after probing the constant affected-key list, never by hashing raw // spans into a join against service_pass_affected. The histogram is O(buckets). func spanRollupInsertSQL(keys string) string { @@ -1762,29 +1827,13 @@ WITH span_agg AS ( )` } -const serviceRollupJoinSQL = `, -log_agg AS ( - SELECT - l.namespace, - date_trunc('minute', l.time::TIMESTAMP_NS) AS bucket, - l.service, - COUNT(*) AS log_count - FROM logs l - WHERE l.time >= ?::TIMESTAMP_NS::TIMESTAMPTZ_NS - AND l.time < ?::TIMESTAMP_NS::TIMESTAMPTZ_NS - GROUP BY l.namespace, date_trunc('minute', l.time::TIMESTAMP_NS), l.service -), -metric_agg AS ( - SELECT - m.namespace, - date_trunc('minute', m.time::TIMESTAMP_NS) AS bucket, - m.service, - COUNT(DISTINCT m.name) AS metric_count - FROM metrics m - WHERE m.time >= ?::TIMESTAMP_NS::TIMESTAMPTZ_NS - AND m.time < ?::TIMESTAMP_NS::TIMESTAMPTZ_NS - GROUP BY m.namespace, date_trunc('minute', m.time::TIMESTAMP_NS), m.service -) +// serviceRollupJoinSQL reads the materialized span, log and metric aggregates, +// each already filtered to the affected keys, and joins them to every affected +// service-minute so empty signals still produce a zeroed row. +const serviceRollupJoinSQL = ` +WITH span_agg AS (SELECT * FROM service_pass_spans), +log_agg AS (SELECT * FROM service_pass_logs), +metric_agg AS (SELECT * FROM service_pass_metrics) INSERT INTO service_rollup ( namespace, bucket, diff --git a/internal/engine/duck_test.go b/internal/engine/duck_test.go index 2ed3bc87..c71537ed 100644 --- a/internal/engine/duck_test.go +++ b/internal/engine/duck_test.go @@ -283,11 +283,19 @@ func TestRollupAdmissionDoesNotCancelAdmittedWork(t *testing.T) { mock.ExpectExec("DELETE FROM resource_rollup").WillReturnResult(sqlmock.NewResult(0, 0)) mock.ExpectExec("DELETE FROM receiver_entity_rollup").WillReturnResult(sqlmock.NewResult(0, 0)) mock.ExpectExec("CREATE OR REPLACE TEMP TABLE service_pass_spans").WillReturnResult(sqlmock.NewResult(0, 0)) + mock.ExpectExec("CREATE OR REPLACE TEMP TABLE service_pass_logs").WillReturnResult(sqlmock.NewResult(0, 0)) + mock.ExpectExec("CREATE OR REPLACE TEMP TABLE service_pass_metrics").WillReturnResult(sqlmock.NewResult(0, 0)) mock.ExpectQuery("FROM service_pass_affected").WillReturnRows(sqlmock.NewRows([]string{"namespace", "bucket", "service"}).AddRow("ns", int64(0), "api")) mock.ExpectExec("INSERT INTO service_pass_spans").WillReturnResult(sqlmock.NewResult(0, 0)) mock.ExpectExec("INSERT INTO span_rollup").WillReturnResult(sqlmock.NewResult(0, 0)) + mock.ExpectQuery("FROM service_pass_affected").WillReturnRows(sqlmock.NewRows([]string{"namespace", "bucket", "service"}).AddRow("ns", int64(0), "api")) + mock.ExpectExec("INSERT INTO service_pass_logs").WillReturnResult(sqlmock.NewResult(0, 0)) + mock.ExpectQuery("FROM service_pass_affected").WillReturnRows(sqlmock.NewRows([]string{"namespace", "bucket", "service"}).AddRow("ns", int64(0), "api")) + mock.ExpectExec("INSERT INTO service_pass_metrics").WillReturnResult(sqlmock.NewResult(0, 0)) mock.ExpectExec("WITH span_agg").WillReturnResult(sqlmock.NewResult(0, 0)) mock.ExpectExec("DROP TABLE service_pass_spans").WillReturnResult(sqlmock.NewResult(0, 0)) + mock.ExpectExec("DROP TABLE service_pass_logs").WillReturnResult(sqlmock.NewResult(0, 0)) + mock.ExpectExec("DROP TABLE service_pass_metrics").WillReturnResult(sqlmock.NewResult(0, 0)) mock.ExpectQuery("FROM service_pass_affected").WillReturnRows(sqlmock.NewRows([]string{"namespace", "bucket", "service"}).AddRow("ns", int64(0), "api")) mock.ExpectExec("INSERT INTO resource_rollup").WillReturnResult(sqlmock.NewResult(0, 0)) mock.ExpectExec("INSERT INTO endpoint_rollup").WillReturnResult(sqlmock.NewResult(0, 0)) diff --git a/internal/engine/resource_inventory_test.go b/internal/engine/resource_inventory_test.go index 409c1da2..67e216fd 100644 --- a/internal/engine/resource_inventory_test.go +++ b/internal/engine/resource_inventory_test.go @@ -7,6 +7,7 @@ import ( "io" "os" "path/filepath" + "slices" "strings" "testing" "time" @@ -154,7 +155,9 @@ func TestResourceInventorySignalSpecificUnknownBoundsAndPruning(t *testing.T) { } w := queryrows.Window{Start: at, End: time.Unix(0, 120)} assertInventoryOracle(t, d, w, 3) - batches := repo.Parquet.BatchMetadata() + // BatchMetadata returns a shared, read-only snapshot; clone before mutating + // SpanTime below so the mutation stays local to this test. + batches := slices.Clone(repo.Parquet.BatchMetadata()) source := d.resourceInventorySource(batches, map[string]bool{"mixed": true}, w) if strings.Contains(source, "spans.parquet") || !strings.Contains(source, "logs.parquet") || strings.Contains(source, "metrics.parquet") { t.Fatal(source) diff --git a/internal/engine/rollup_keys_test.go b/internal/engine/rollup_keys_test.go index def53490..74c960ad 100644 --- a/internal/engine/rollup_keys_test.go +++ b/internal/engine/rollup_keys_test.go @@ -408,7 +408,63 @@ func TestEdgeRollupKeysMatchConstantList(t *testing.T) { } } -// Unchunked reference for differential tests of the bounded implementation. +// Unchunked, full-scan reference for differential tests of the bounded +// implementation. span_agg comes from one bounded span scan; log_agg and +// metric_agg scan logs and metrics directly over the same event-time range. +// Production insertServiceRollup instead materializes all three per +// affected-key chunk; these tests pin that both produce identical rows. func serviceRollupInsertSQL(keys string) string { - return serviceRollupSpanSQL(keys) + serviceRollupJoinSQL + return serviceRollupSpanSQL(keys) + serviceRollupReferenceJoinSQL } + +const serviceRollupReferenceJoinSQL = `, +log_agg AS ( + SELECT + l.namespace, + date_trunc('minute', l.time::TIMESTAMP_NS) AS bucket, + l.service, + COUNT(*) AS log_count + FROM logs l + WHERE l.time >= ?::TIMESTAMP_NS::TIMESTAMPTZ_NS + AND l.time < ?::TIMESTAMP_NS::TIMESTAMPTZ_NS + GROUP BY l.namespace, date_trunc('minute', l.time::TIMESTAMP_NS), l.service +), +metric_agg AS ( + SELECT + m.namespace, + date_trunc('minute', m.time::TIMESTAMP_NS) AS bucket, + m.service, + COUNT(DISTINCT m.name) AS metric_count + FROM metrics m + WHERE m.time >= ?::TIMESTAMP_NS::TIMESTAMPTZ_NS + AND m.time < ?::TIMESTAMP_NS::TIMESTAMPTZ_NS + GROUP BY m.namespace, date_trunc('minute', m.time::TIMESTAMP_NS), m.service +) +INSERT INTO service_rollup ( + namespace, + bucket, + service, + spans, + served_spans, + p50_ms, + p95_ms, + error_rate, + log_count, + metric_count +) +SELECT + a.namespace, + a.bucket, + a.service, + COALESCE(s.spans, 0), + COALESCE(s.served_spans, 0), + COALESCE(s.p50_ms, 0), + COALESCE(s.p95_ms, 0), + COALESCE(s.error_rate, 0), + COALESCE(l.log_count, 0), + COALESCE(m.metric_count, 0) +FROM service_pass_affected a +LEFT JOIN span_agg s USING (namespace, bucket, service) +LEFT JOIN log_agg l USING (namespace, bucket, service) +LEFT JOIN metric_agg m USING (namespace, bucket, service) +WHERE a.service != '';` diff --git a/internal/engine/rollup_late_signal_memory_test.go b/internal/engine/rollup_late_signal_memory_test.go new file mode 100644 index 00000000..f47c285c --- /dev/null +++ b/internal/engine/rollup_late_signal_memory_test.go @@ -0,0 +1,81 @@ +package engine + +import ( + "fmt" + "path/filepath" + "testing" + "time" +) + +// One late span (or one skewed clock) pulls the affected hull back many days. +// The span path already filters its scan and aggregation to the affected keys, +// but the log and metric aggregates used to run over the whole hull with no +// key filter: a single old bucket then forced a hash aggregate over every +// service-minute of dense history, which blew memory and froze the watermark. +// This pins that the log/metric aggregates stay bounded to the affected keys. +func TestServiceLateSignalLogMetricMemory(t *testing.T) { + if !rollupMemoryProcess(t) { + return + } + db := openTestDuck(t) + if err := CreateTables(db); err != nil { + t.Fatal(err) + } + if err := CreateViews(db); err != nil { + t.Fatal(err) + } + dir := t.TempDir() + logSchema, logData := filepath.Join(dir, "log_schema.parquet"), filepath.Join(dir, "log_data.parquet") + metricSchema, metricData := filepath.Join(dir, "metric_schema.parquet"), filepath.Join(dir, "metric_data.parquet") + + // A dense 20-day hull: one log and one metric for every service-minute. The + // affected set is five recent service-minutes plus one 20-day-old bucket, + // which is all the pass should aggregate. + const minutes, services, oldMinute = 28800, 40, 0 + recent := minutes - 1 + affected := fmt.Sprintf(`SELECT 'ns' AS namespace,make_timestamp_ns(%d::BIGINT*60000000000)::TIMESTAMP_NS AS bucket,'svc-'||i::VARCHAR AS service FROM range(5) t(i) + UNION ALL SELECT 'ns',make_timestamp_ns(%d::BIGINT*60000000000)::TIMESTAMP_NS,'svc-0'`, recent, oldMinute) + + for _, q := range []string{ + "SET memory_limit='2GB'", "SET threads=4", + fmt.Sprintf(`CREATE VIEW log_fixture AS SELECT 'ns' AS namespace,'svc-'||(i%%%d)::VARCHAR AS service, + make_timestamp_ns((i//%d)::BIGINT*60000000000)::TIMESTAMPTZ_NS AS time + FROM range(%d) t(i)`, services, services, minutes*services), + fmt.Sprintf(`CREATE VIEW metric_fixture AS SELECT 'ns' AS namespace,'svc-'||(i%%%d)::VARCHAR AS service, + make_timestamp_ns((i//%d)::BIGINT*60000000000)::TIMESTAMPTZ_NS AS time,'m-0' AS name + FROM range(%d) t(i)`, services, services, minutes*services), + "COPY (SELECT * FROM log_fixture WHERE false) TO " + sqlLiteral(logSchema) + " (FORMAT PARQUET)", + "COPY log_fixture TO " + sqlLiteral(logData) + " (FORMAT PARQUET)", + "COPY (SELECT * FROM metric_fixture WHERE false) TO " + sqlLiteral(metricSchema) + " (FORMAT PARQUET)", + "COPY metric_fixture TO " + sqlLiteral(metricData) + " (FORMAT PARQUET)", + "CREATE OR REPLACE VIEW logs AS SELECT * FROM read_parquet([" + sqlLiteral(logSchema) + "," + sqlLiteral(logData) + "],union_by_name=false,hive_partitioning=false)", + "CREATE OR REPLACE VIEW metrics AS SELECT * FROM read_parquet([" + sqlLiteral(metricSchema) + "," + sqlLiteral(metricData) + "],union_by_name=false,hive_partitioning=false)", + "CREATE TEMP TABLE service_pass_affected AS " + affected, + } { + if _, err := db.Exec(q); err != nil { + t.Fatal(err) + } + } + + before := peakRSS(t) + started := time.Now() + lo, hi := time.Unix(0, 0).UTC(), time.Unix(int64(minutes*60), 0).UTC() + if _, err := insertServiceRollup(t.Context(), db, lo, hi); err != nil { + t.Fatal(err) + } + growth := peakRSS(t) - before + t.Logf("service log/metric rollup: %s, peak RSS growth %.1f MiB", time.Since(started), float64(growth)/(1<<20)) + if growth > 96<<20 { + t.Errorf("log/metric aggregates over %d service-minutes grew RSS by %d, want <=96 MiB; a late bucket must not force a full-hull aggregate", minutes*services, growth) + } + + // Only the six affected service-minutes are summarized, each with its one + // log and one metric. The dense history behind the hull is never counted. + var rows, logCount, metricCount int + if err := db.QueryRow("SELECT count(*),sum(log_count),sum(metric_count) FROM service_rollup").Scan(&rows, &logCount, &metricCount); err != nil { + t.Fatal(err) + } + if rows != 6 || logCount != 6 || metricCount != 6 { + t.Fatalf("service_rollup rows=%d log_count=%d metric_count=%d, want 6/6/6", rows, logCount, metricCount) + } +} diff --git a/internal/query/result_cache.go b/internal/query/result_cache.go index a30eb91f..26ac11b9 100644 --- a/internal/query/result_cache.go +++ b/internal/query/result_cache.go @@ -97,6 +97,10 @@ type cacheFlight struct { background bool started atomic.Bool cancel context.CancelFunc + // cancelled records that this flight's last caller left and its run was + // cancelled. It stays registered until its function returns, but a new + // caller must not join a run that will end in context.Canceled. + cancelled bool } type flightContextKey struct{} @@ -234,6 +238,7 @@ retry: c.mu.Lock() flight.waiters-- if flight.waiters == 0 && (!flight.detached || !flight.started.Load() && !flight.background) { + flight.cancelled = true flight.cancel() // Retain the flight until fn exits, even if cancellation is slow. } @@ -301,6 +306,7 @@ func (e *Executor) availablePanels(ctx context.Context, lookups []panelCacheLook } flight.waiters-- if flight.waiters == 0 && (!flight.detached || !flight.started.Load() && !flight.background) { + flight.cancelled = true flight.cancel() } } @@ -333,6 +339,11 @@ func (e *Executor) availablePanels(ctx context.Context, lookups []panelCacheLook // window is identical and every read snapshot is acquired after registration. // c.mu is held. func (e *Executor) joinable(flight *cacheFlight, stamp time.Time, bypass, fixedWindow bool) bool { + // A cancelled flight will end in context.Canceled; a new caller waits for + // it to clear and runs its own instead of adopting a dead run. + if flight.cancelled { + return false + } return !bypass || !flight.stamp.Before(stamp) || fixedWindow && flight.fixedWindow && !flight.registeredAt.Before(stamp) } diff --git a/internal/query/result_cache_test.go b/internal/query/result_cache_test.go index 81f7f3f5..ed812678 100644 --- a/internal/query/result_cache_test.go +++ b/internal/query/result_cache_test.go @@ -627,6 +627,49 @@ func TestResultCacheKeepsCancelledFlightUntilItExits(t *testing.T) { } } +// A non-bypass caller that arrives while a cancelled flight is still +// registered must not adopt it: the cancelled run ends in context.Canceled, +// which would surface as a failed panel for a live caller. The newcomer waits +// for the dead run to clear and then runs its own, getting a real answer. +func TestResultCacheRejectsCancelledFlight(t *testing.T) { + e, _, _ := cachedExecutor(t) + key := cacheKey{47} + gone, leave := context.WithCancel(t.Context()) + release := make(chan struct{}) + cancelled := make(chan struct{}) + left := make(chan struct{}) + go func() { + _, _, _ = loadCached(e, gone, key, e.now(), func(run context.Context) (int, int64, bool, error) { + <-run.Done() + close(cancelled) + <-release // slow to honor cancellation + return 0, 0, false, run.Err() + }) + close(left) + }() + waitForFlight(t, e, key) + leave() // the lone caller leaves, cancelling the flight + <-left + <-cancelled // the flight is cancelled but still registered + result := make(chan error, 1) + var calls atomic.Int64 + go func() { + _, _, err := loadCached(e, t.Context(), key, e.now(), func(context.Context) (int, int64, bool, error) { + calls.Add(1) + return 1, 8, true, nil + }) + result <- err + }() + waitForWaiters(t, e, key, 1) + close(release) // the cancelled run returns context.Canceled + if err := <-result; err != nil { + t.Fatalf("a live caller adopted a cancelled flight: %v", err) + } + if calls.Load() != 1 { + t.Fatalf("the newcomer did not run its own query: calls=%d", calls.Load()) + } +} + // A run nobody will read later, because the cache is off or only no-cache // callers wait for it, is cancelled once its last caller leaves, and not // stored. diff --git a/internal/telemetry/batch_metadata_snapshot_test.go b/internal/telemetry/batch_metadata_snapshot_test.go new file mode 100644 index 00000000..926e895a --- /dev/null +++ b/internal/telemetry/batch_metadata_snapshot_test.go @@ -0,0 +1,61 @@ +package telemetry + +import ( + "slices" + "sort" + "strings" + "testing" +) + +func orderedBatchIDs(batches []BatchMetadata) string { + ids := make([]string, len(batches)) + for i, b := range batches { + ids[i] = b.ID + } + return strings.Join(ids, ",") +} + +// BatchMetadata publishes an immutable, pre-sorted snapshot rebuilt on each +// catalog mutation, rather than allocating and sorting the whole catalog on +// every call under the lock. The order is canonical (generation, then min +// ingest time, then ID), the same snapshot is returned until the catalog +// changes, callers must not mutate it, and a mutation shows on the next call. +func TestBatchMetadataSharesSortedSnapshot(t *testing.T) { + p := &ParquetStore{batches: make(map[string]*storedBatch)} + install := func(id string, generation uint32, minIngest int64) { + p.installBatch(&storedBatch{metadata: BatchMetadata{ID: id, Generation: generation, MinIngestedNanos: minIngest, Spans: 1}}) + } + // Installed out of canonical order. + install("z", 1, 10) + install("a", 0, 30) + install("b", 0, 20) + install("c", 0, 20) + + if got := orderedBatchIDs(p.BatchMetadata()); got != "b,c,a,z" { + t.Fatalf("canonical order = %q, want \"b,c,a,z\"", got) + } + + // No per-call allocation or sort: the same snapshot backs each call until + // the catalog mutates. + first, second := p.BatchMetadata(), p.BatchMetadata() + if len(first) == 0 || &first[0] != &second[0] { + t.Fatal("BatchMetadata reallocated the catalog without a mutation") + } + + // The snapshot is shared and read-only: a caller sorting a clone must not + // disturb what the next reader sees. + clone := slices.Clone(first) + sort.Slice(clone, func(i, j int) bool { return clone[i].ID > clone[j].ID }) + if got := orderedBatchIDs(p.BatchMetadata()); got != "b,c,a,z" { + t.Fatalf("shared snapshot was mutated: %q", got) + } + + // A prune and a commit are both reflected on the next call, re-sorted. + p.mu.Lock() + p.setBatchLocked("a", nil) // prune + p.mu.Unlock() + install("m", 0, 5) // commit + if got := orderedBatchIDs(p.BatchMetadata()); got != "m,b,c,z" { + t.Fatalf("after prune+commit = %q, want \"m,b,c,z\"", got) + } +} diff --git a/internal/telemetry/compaction_debt.go b/internal/telemetry/compaction_debt.go index eee8890d..3342a959 100644 --- a/internal/telemetry/compaction_debt.go +++ b/internal/telemetry/compaction_debt.go @@ -78,6 +78,7 @@ func (p *ParquetStore) setBatchLocked(id string, batch *storedBatch) { p.changeGeneration0Locked(batch.metadata, 1) } p.generation0Snapshot.Store(int64(p.generation0Batches)) + p.publishMetadataLocked() if p.compactionDebt >= CompactionDebtLimit && p.debtLimitReached != nil { close(p.debtLimitReached) p.debtLimitReached = nil diff --git a/internal/telemetry/parquet.go b/internal/telemetry/parquet.go index 69723db3..3fa74b99 100644 --- a/internal/telemetry/parquet.go +++ b/internal/telemetry/parquet.go @@ -124,6 +124,10 @@ type ParquetStore struct { generation0Batches int // Published after each catalog mutation; scrapes never take mu. generation0Snapshot atomic.Int64 + // The canonically sorted catalog, republished after each mutation so + // BatchMetadata reads it without taking mu or sorting per call. The slice + // is shared and immutable; callers must not mutate it. + metadataSnapshot atomic.Pointer[[]BatchMetadata] generation0ByWindow map[int64]int compactionDebt int debtLimitReached chan struct{} @@ -215,9 +219,22 @@ func (p *ParquetStore) MergeParquet(ctx context.Context, signal string, inputs [ return stats, err } +// BatchMetadata returns the installed batches in canonical order: generation, +// then earliest ingest time, then ID. The result is a shared, immutable +// snapshot republished on every catalog mutation, so this call takes no lock +// and allocates nothing. Callers MUST NOT mutate it — no sorting, appending in +// place, or assigning elements; clone it first (slices.Clone) if they need to. func (p *ParquetStore) BatchMetadata() []BatchMetadata { - p.mu.RLock() - defer p.mu.RUnlock() + if snapshot := p.metadataSnapshot.Load(); snapshot != nil { + return *snapshot + } + return nil +} + +// publishMetadataLocked rebuilds the canonically sorted catalog snapshot. It +// runs under mu from setBatchLocked, the sole catalog mutation, so every commit, +// replacement, restoration, rollback, startup load and retention refreshes it. +func (p *ParquetStore) publishMetadataLocked() { out := make([]BatchMetadata, 0, len(p.batches)) for _, batch := range p.batches { out = append(out, batch.metadata) @@ -231,7 +248,7 @@ func (p *ParquetStore) BatchMetadata() []BatchMetadata { } return out[i].ID < out[j].ID }) - return out + p.metadataSnapshot.Store(&out) } // batchMetadataByID returns one installed batch without scanning the catalog.