From 1dc90be0d6498e1c12925910909c799d3dee77cb Mon Sep 17 00:00:00 2001 From: Vishal Rana Date: Sat, 14 Nov 2026 19:08:35 -0800 Subject: [PATCH 1/5] fix(query): reject cancelled result-cache flights from new joiners When a flight's last caller leaves, its run is cancelled but the flight stays registered in c.flights until its function returns. A non-bypass caller arriving in that window joined the dead flight, waited for it, and received context.Canceled, which exec.go rendered as a failed panel for a live caller. Mark the flight cancelled under c.mu at both cancel sites and have joinable (used by loadCached and availablePanels) reject a cancelled flight. The newcomer now waits for the dead run to clear and then runs its own query, getting a real answer. The existing fixedWindow joinable logic and the "keep the flight until its function exits" invariant are preserved. --- internal/query/result_cache.go | 11 ++++++++ internal/query/result_cache_test.go | 43 +++++++++++++++++++++++++++++ 2 files changed, 54 insertions(+) diff --git a/internal/query/result_cache.go b/internal/query/result_cache.go index a30eb91f..26ac11b9 100644 --- a/internal/query/result_cache.go +++ b/internal/query/result_cache.go @@ -97,6 +97,10 @@ type cacheFlight struct { background bool started atomic.Bool cancel context.CancelFunc + // cancelled records that this flight's last caller left and its run was + // cancelled. It stays registered until its function returns, but a new + // caller must not join a run that will end in context.Canceled. + cancelled bool } type flightContextKey struct{} @@ -234,6 +238,7 @@ retry: c.mu.Lock() flight.waiters-- if flight.waiters == 0 && (!flight.detached || !flight.started.Load() && !flight.background) { + flight.cancelled = true flight.cancel() // Retain the flight until fn exits, even if cancellation is slow. } @@ -301,6 +306,7 @@ func (e *Executor) availablePanels(ctx context.Context, lookups []panelCacheLook } flight.waiters-- if flight.waiters == 0 && (!flight.detached || !flight.started.Load() && !flight.background) { + flight.cancelled = true flight.cancel() } } @@ -333,6 +339,11 @@ func (e *Executor) availablePanels(ctx context.Context, lookups []panelCacheLook // window is identical and every read snapshot is acquired after registration. // c.mu is held. func (e *Executor) joinable(flight *cacheFlight, stamp time.Time, bypass, fixedWindow bool) bool { + // A cancelled flight will end in context.Canceled; a new caller waits for + // it to clear and runs its own instead of adopting a dead run. + if flight.cancelled { + return false + } return !bypass || !flight.stamp.Before(stamp) || fixedWindow && flight.fixedWindow && !flight.registeredAt.Before(stamp) } diff --git a/internal/query/result_cache_test.go b/internal/query/result_cache_test.go index 81f7f3f5..ed812678 100644 --- a/internal/query/result_cache_test.go +++ b/internal/query/result_cache_test.go @@ -627,6 +627,49 @@ func TestResultCacheKeepsCancelledFlightUntilItExits(t *testing.T) { } } +// A non-bypass caller that arrives while a cancelled flight is still +// registered must not adopt it: the cancelled run ends in context.Canceled, +// which would surface as a failed panel for a live caller. The newcomer waits +// for the dead run to clear and then runs its own, getting a real answer. +func TestResultCacheRejectsCancelledFlight(t *testing.T) { + e, _, _ := cachedExecutor(t) + key := cacheKey{47} + gone, leave := context.WithCancel(t.Context()) + release := make(chan struct{}) + cancelled := make(chan struct{}) + left := make(chan struct{}) + go func() { + _, _, _ = loadCached(e, gone, key, e.now(), func(run context.Context) (int, int64, bool, error) { + <-run.Done() + close(cancelled) + <-release // slow to honor cancellation + return 0, 0, false, run.Err() + }) + close(left) + }() + waitForFlight(t, e, key) + leave() // the lone caller leaves, cancelling the flight + <-left + <-cancelled // the flight is cancelled but still registered + result := make(chan error, 1) + var calls atomic.Int64 + go func() { + _, _, err := loadCached(e, t.Context(), key, e.now(), func(context.Context) (int, int64, bool, error) { + calls.Add(1) + return 1, 8, true, nil + }) + result <- err + }() + waitForWaiters(t, e, key, 1) + close(release) // the cancelled run returns context.Canceled + if err := <-result; err != nil { + t.Fatalf("a live caller adopted a cancelled flight: %v", err) + } + if calls.Load() != 1 { + t.Fatalf("the newcomer did not run its own query: calls=%d", calls.Load()) + } +} + // A run nobody will read later, because the cache is off or only no-cache // callers wait for it, is cancelled once its last caller leaves, and not // stored. From 8292ff2f77e4e043580e099ac911b6ab1566077b Mon Sep 17 00:00:00 2001 From: Vishal Rana Date: Sat, 14 Nov 2026 19:23:04 -0800 Subject: [PATCH 2/5] fix(engine): key-filter service-rollup log and metric aggregates The service rollup derives one event-time hull [min(bucket), max(bucket)+1m) from the affected service-minutes and every statement scans it. The span path was already filtered to the affected keys, but log_agg and metric_agg scanned and grouped that whole hull with no key filter. A single late or clock-skewed span pulled the hull back by days, so the pass aggregated every service-minute of dense history, blew memory, held the write gate, and left the watermark frozen because the failed pass kept re-including the same span. Materialize the log and metric aggregates into temp tables the same way the span path does: probe the constant affected-key list over the same bounded event-time range, so one old bucket can no longer force a full-hull scan and group. The final join now reads the three materialized aggregates. Aggregates stay exact (keys are distinct, so each service-minute is in one chunk) and the serialized-delete, grouped-insert uniqueness invariant is unchanged. A new late-signal memory test over a dense 20-day hull drops peak RSS growth from ~305 MiB to ~8 MiB. --- internal/engine/duck.go | 127 ++++++++++++------ internal/engine/duck_test.go | 8 ++ internal/engine/rollup_keys_test.go | 60 ++++++++- .../engine/rollup_late_signal_memory_test.go | 81 +++++++++++ 4 files changed, 235 insertions(+), 41 deletions(-) create mode 100644 internal/engine/rollup_late_signal_memory_test.go diff --git a/internal/engine/duck.go b/internal/engine/duck.go index 8bb93d7a..9876a64d 100644 --- a/internal/engine/duck.go +++ b/internal/engine/duck.go @@ -1613,24 +1613,31 @@ WHERE bucket >= ?::TIMESTAMP_NS::TIMESTAMPTZ_NS // Aggregate each bounded signal scan before the final affected-bucket joins. // Schema-first Parquet scans can have a zero cardinality estimate in DuckDB 2; // joining raw events to the affected table can then build a hash table of every -// event. Logs and metrics aggregate before joining. Spans must be filtered to -// the affected keys first: each unaffected service-minute otherwise allocates -// its own t-digest states. A semi-join against service_pass_affected reverses -// into a build of every raw span, so the keys are bound as constants instead. +// event. Spans, logs and metrics are all filtered to the affected keys first, +// bound as a constant list rather than a semi-join against +// service_pass_affected, which reverses into a build of every raw event. A late +// or clock-skewed event pulls the event-time hull back by days; without the key +// filter the log and metric aggregates then scanned and grouped every +// service-minute of that whole hull, blowing memory and freezing the watermark. +// For spans each unaffected service-minute otherwise also allocates its own +// t-digest states. The final join only reads the materialized aggregates. func insertServiceRollup(ctx context.Context, tx rollupSQL, lo, hi time.Time) (sql.Result, error) { - // Materialize only bounded span aggregates. Logs, metrics and the final - // affected-key join run once, preserving empty and NULL-key output rows. - _, err := tx.ExecContext(ctx, `CREATE OR REPLACE TEMP TABLE service_pass_spans ( + for _, create := range []string{ + `CREATE OR REPLACE TEMP TABLE service_pass_spans ( namespace VARCHAR, bucket TIMESTAMP_NS, service VARCHAR, spans BIGINT, served_spans BIGINT, - p50_ms DOUBLE, p95_ms DOUBLE, error_rate DOUBLE)`) - if err != nil { - return nil, err + p50_ms DOUBLE, p95_ms DOUBLE, error_rate DOUBLE)`, + `CREATE OR REPLACE TEMP TABLE service_pass_logs ( + namespace VARCHAR, bucket TIMESTAMP_NS, service VARCHAR, log_count BIGINT)`, + `CREATE OR REPLACE TEMP TABLE service_pass_metrics ( + namespace VARCHAR, bucket TIMESTAMP_NS, service VARCHAR, metric_count BIGINT)`, + } { + if _, err := tx.ExecContext(ctx, create); err != nil { + return nil, err + } } - err = forAffectedKeys(ctx, tx, - `SELECT DISTINCT namespace, epoch_ns(bucket), service FROM service_pass_affected - WHERE namespace IS NOT NULL AND bucket IS NOT NULL AND service IS NOT NULL`, nil, + err := forAffectedKeys(ctx, tx, serviceRollupAffectedKeysSQL, nil, `struct_pack(namespace := s.namespace, bucket := date_trunc('minute', s.start_time::TIMESTAMP_NS), service := s.service)`, - `struct_pack(namespace := ?::VARCHAR, bucket := make_timestamp_ns(?::BIGINT), service := ?::VARCHAR)`, + serviceRollupAffectedKeyExpr, func(keys string, args []any) error { _, err := tx.ExecContext(ctx, "INSERT INTO service_pass_spans "+serviceRollupSpanSQL(keys)+" SELECT * FROM span_agg", append(args, lo, hi)...) if err != nil { @@ -1642,13 +1649,37 @@ func insertServiceRollup(ctx context.Context, tx rollupSQL, lo, hi time.Time) (s if err != nil { return nil, err } - res, err := tx.ExecContext(ctx, "WITH span_agg AS (SELECT * FROM service_pass_spans)"+serviceRollupJoinSQL, lo, hi, lo, hi) + // Logs and metrics aggregate the same affected keys over the same bounded + // event-time range, so one old bucket cannot force a full-hull scan. + err = forAffectedKeys(ctx, tx, serviceRollupAffectedKeysSQL, nil, + `struct_pack(namespace := l.namespace, bucket := date_trunc('minute', l.time::TIMESTAMP_NS), service := l.service)`, + serviceRollupAffectedKeyExpr, + func(keys string, args []any) error { + _, err := tx.ExecContext(ctx, serviceRollupLogInsertSQL(keys), append(args, lo, hi)...) + return err + }) + if err != nil { + return nil, err + } + err = forAffectedKeys(ctx, tx, serviceRollupAffectedKeysSQL, nil, + `struct_pack(namespace := m.namespace, bucket := date_trunc('minute', m.time::TIMESTAMP_NS), service := m.service)`, + serviceRollupAffectedKeyExpr, + func(keys string, args []any) error { + _, err := tx.ExecContext(ctx, serviceRollupMetricInsertSQL(keys), append(args, lo, hi)...) + return err + }) if err != nil { return nil, err } - if _, err := tx.ExecContext(ctx, "DROP TABLE service_pass_spans"); err != nil { + res, err := tx.ExecContext(ctx, serviceRollupJoinSQL) + if err != nil { return nil, err } + for _, table := range []string{"service_pass_spans", "service_pass_logs", "service_pass_metrics"} { + if _, err := tx.ExecContext(ctx, "DROP TABLE "+table); err != nil { + return nil, err + } + } return res, nil } @@ -1710,6 +1741,40 @@ func forAffectedKeys(ctx context.Context, tx rollupSQL, query string, queryArgs return nil } +// serviceRollupAffectedKeysSQL lists the distinct affected service-minutes, and +// serviceRollupAffectedKeyExpr is the constant each scan probes against. Both +// are shared by the span, log and metric aggregates so every signal is filtered +// to the same keys. +const serviceRollupAffectedKeysSQL = `SELECT DISTINCT namespace, epoch_ns(bucket), service FROM service_pass_affected + WHERE namespace IS NOT NULL AND bucket IS NOT NULL AND service IS NOT NULL` + +const serviceRollupAffectedKeyExpr = `struct_pack(namespace := ?::VARCHAR, bucket := make_timestamp_ns(?::BIGINT), service := ?::VARCHAR)` + +// Log and metric aggregates probe the same constant affected-key list and the +// same bounded event-time range as the span path, so one old or clock-skewed +// bucket cannot widen them into a scan and group of the whole hull. Keys are +// distinct, so each affected service-minute lands in exactly one chunk and its +// COUNT and COUNT(DISTINCT name) are exact. +func serviceRollupLogInsertSQL(keys string) string { + return `INSERT INTO service_pass_logs + SELECT l.namespace, date_trunc('minute', l.time::TIMESTAMP_NS), l.service, COUNT(*) + FROM logs l + WHERE ` + keys + ` + AND l.time >= ?::TIMESTAMP_NS::TIMESTAMPTZ_NS + AND l.time < ?::TIMESTAMP_NS::TIMESTAMPTZ_NS + GROUP BY l.namespace, date_trunc('minute', l.time::TIMESTAMP_NS), l.service` +} + +func serviceRollupMetricInsertSQL(keys string) string { + return `INSERT INTO service_pass_metrics + SELECT m.namespace, date_trunc('minute', m.time::TIMESTAMP_NS), m.service, COUNT(DISTINCT m.name) + FROM metrics m + WHERE ` + keys + ` + AND m.time >= ?::TIMESTAMP_NS::TIMESTAMPTZ_NS + AND m.time < ?::TIMESTAMP_NS::TIMESTAMPTZ_NS + GROUP BY m.namespace, date_trunc('minute', m.time::TIMESTAMP_NS), m.service` +} + // Aggregate after probing the constant affected-key list, never by hashing raw // spans into a join against service_pass_affected. The histogram is O(buckets). func spanRollupInsertSQL(keys string) string { @@ -1762,29 +1827,13 @@ WITH span_agg AS ( )` } -const serviceRollupJoinSQL = `, -log_agg AS ( - SELECT - l.namespace, - date_trunc('minute', l.time::TIMESTAMP_NS) AS bucket, - l.service, - COUNT(*) AS log_count - FROM logs l - WHERE l.time >= ?::TIMESTAMP_NS::TIMESTAMPTZ_NS - AND l.time < ?::TIMESTAMP_NS::TIMESTAMPTZ_NS - GROUP BY l.namespace, date_trunc('minute', l.time::TIMESTAMP_NS), l.service -), -metric_agg AS ( - SELECT - m.namespace, - date_trunc('minute', m.time::TIMESTAMP_NS) AS bucket, - m.service, - COUNT(DISTINCT m.name) AS metric_count - FROM metrics m - WHERE m.time >= ?::TIMESTAMP_NS::TIMESTAMPTZ_NS - AND m.time < ?::TIMESTAMP_NS::TIMESTAMPTZ_NS - GROUP BY m.namespace, date_trunc('minute', m.time::TIMESTAMP_NS), m.service -) +// serviceRollupJoinSQL reads the materialized span, log and metric aggregates, +// each already filtered to the affected keys, and joins them to every affected +// service-minute so empty signals still produce a zeroed row. +const serviceRollupJoinSQL = ` +WITH span_agg AS (SELECT * FROM service_pass_spans), +log_agg AS (SELECT * FROM service_pass_logs), +metric_agg AS (SELECT * FROM service_pass_metrics) INSERT INTO service_rollup ( namespace, bucket, diff --git a/internal/engine/duck_test.go b/internal/engine/duck_test.go index 2ed3bc87..c71537ed 100644 --- a/internal/engine/duck_test.go +++ b/internal/engine/duck_test.go @@ -283,11 +283,19 @@ func TestRollupAdmissionDoesNotCancelAdmittedWork(t *testing.T) { mock.ExpectExec("DELETE FROM resource_rollup").WillReturnResult(sqlmock.NewResult(0, 0)) mock.ExpectExec("DELETE FROM receiver_entity_rollup").WillReturnResult(sqlmock.NewResult(0, 0)) mock.ExpectExec("CREATE OR REPLACE TEMP TABLE service_pass_spans").WillReturnResult(sqlmock.NewResult(0, 0)) + mock.ExpectExec("CREATE OR REPLACE TEMP TABLE service_pass_logs").WillReturnResult(sqlmock.NewResult(0, 0)) + mock.ExpectExec("CREATE OR REPLACE TEMP TABLE service_pass_metrics").WillReturnResult(sqlmock.NewResult(0, 0)) mock.ExpectQuery("FROM service_pass_affected").WillReturnRows(sqlmock.NewRows([]string{"namespace", "bucket", "service"}).AddRow("ns", int64(0), "api")) mock.ExpectExec("INSERT INTO service_pass_spans").WillReturnResult(sqlmock.NewResult(0, 0)) mock.ExpectExec("INSERT INTO span_rollup").WillReturnResult(sqlmock.NewResult(0, 0)) + mock.ExpectQuery("FROM service_pass_affected").WillReturnRows(sqlmock.NewRows([]string{"namespace", "bucket", "service"}).AddRow("ns", int64(0), "api")) + mock.ExpectExec("INSERT INTO service_pass_logs").WillReturnResult(sqlmock.NewResult(0, 0)) + mock.ExpectQuery("FROM service_pass_affected").WillReturnRows(sqlmock.NewRows([]string{"namespace", "bucket", "service"}).AddRow("ns", int64(0), "api")) + mock.ExpectExec("INSERT INTO service_pass_metrics").WillReturnResult(sqlmock.NewResult(0, 0)) mock.ExpectExec("WITH span_agg").WillReturnResult(sqlmock.NewResult(0, 0)) mock.ExpectExec("DROP TABLE service_pass_spans").WillReturnResult(sqlmock.NewResult(0, 0)) + mock.ExpectExec("DROP TABLE service_pass_logs").WillReturnResult(sqlmock.NewResult(0, 0)) + mock.ExpectExec("DROP TABLE service_pass_metrics").WillReturnResult(sqlmock.NewResult(0, 0)) mock.ExpectQuery("FROM service_pass_affected").WillReturnRows(sqlmock.NewRows([]string{"namespace", "bucket", "service"}).AddRow("ns", int64(0), "api")) mock.ExpectExec("INSERT INTO resource_rollup").WillReturnResult(sqlmock.NewResult(0, 0)) mock.ExpectExec("INSERT INTO endpoint_rollup").WillReturnResult(sqlmock.NewResult(0, 0)) diff --git a/internal/engine/rollup_keys_test.go b/internal/engine/rollup_keys_test.go index def53490..74c960ad 100644 --- a/internal/engine/rollup_keys_test.go +++ b/internal/engine/rollup_keys_test.go @@ -408,7 +408,63 @@ func TestEdgeRollupKeysMatchConstantList(t *testing.T) { } } -// Unchunked reference for differential tests of the bounded implementation. +// Unchunked, full-scan reference for differential tests of the bounded +// implementation. span_agg comes from one bounded span scan; log_agg and +// metric_agg scan logs and metrics directly over the same event-time range. +// Production insertServiceRollup instead materializes all three per +// affected-key chunk; these tests pin that both produce identical rows. func serviceRollupInsertSQL(keys string) string { - return serviceRollupSpanSQL(keys) + serviceRollupJoinSQL + return serviceRollupSpanSQL(keys) + serviceRollupReferenceJoinSQL } + +const serviceRollupReferenceJoinSQL = `, +log_agg AS ( + SELECT + l.namespace, + date_trunc('minute', l.time::TIMESTAMP_NS) AS bucket, + l.service, + COUNT(*) AS log_count + FROM logs l + WHERE l.time >= ?::TIMESTAMP_NS::TIMESTAMPTZ_NS + AND l.time < ?::TIMESTAMP_NS::TIMESTAMPTZ_NS + GROUP BY l.namespace, date_trunc('minute', l.time::TIMESTAMP_NS), l.service +), +metric_agg AS ( + SELECT + m.namespace, + date_trunc('minute', m.time::TIMESTAMP_NS) AS bucket, + m.service, + COUNT(DISTINCT m.name) AS metric_count + FROM metrics m + WHERE m.time >= ?::TIMESTAMP_NS::TIMESTAMPTZ_NS + AND m.time < ?::TIMESTAMP_NS::TIMESTAMPTZ_NS + GROUP BY m.namespace, date_trunc('minute', m.time::TIMESTAMP_NS), m.service +) +INSERT INTO service_rollup ( + namespace, + bucket, + service, + spans, + served_spans, + p50_ms, + p95_ms, + error_rate, + log_count, + metric_count +) +SELECT + a.namespace, + a.bucket, + a.service, + COALESCE(s.spans, 0), + COALESCE(s.served_spans, 0), + COALESCE(s.p50_ms, 0), + COALESCE(s.p95_ms, 0), + COALESCE(s.error_rate, 0), + COALESCE(l.log_count, 0), + COALESCE(m.metric_count, 0) +FROM service_pass_affected a +LEFT JOIN span_agg s USING (namespace, bucket, service) +LEFT JOIN log_agg l USING (namespace, bucket, service) +LEFT JOIN metric_agg m USING (namespace, bucket, service) +WHERE a.service != '';` diff --git a/internal/engine/rollup_late_signal_memory_test.go b/internal/engine/rollup_late_signal_memory_test.go new file mode 100644 index 00000000..f47c285c --- /dev/null +++ b/internal/engine/rollup_late_signal_memory_test.go @@ -0,0 +1,81 @@ +package engine + +import ( + "fmt" + "path/filepath" + "testing" + "time" +) + +// One late span (or one skewed clock) pulls the affected hull back many days. +// The span path already filters its scan and aggregation to the affected keys, +// but the log and metric aggregates used to run over the whole hull with no +// key filter: a single old bucket then forced a hash aggregate over every +// service-minute of dense history, which blew memory and froze the watermark. +// This pins that the log/metric aggregates stay bounded to the affected keys. +func TestServiceLateSignalLogMetricMemory(t *testing.T) { + if !rollupMemoryProcess(t) { + return + } + db := openTestDuck(t) + if err := CreateTables(db); err != nil { + t.Fatal(err) + } + if err := CreateViews(db); err != nil { + t.Fatal(err) + } + dir := t.TempDir() + logSchema, logData := filepath.Join(dir, "log_schema.parquet"), filepath.Join(dir, "log_data.parquet") + metricSchema, metricData := filepath.Join(dir, "metric_schema.parquet"), filepath.Join(dir, "metric_data.parquet") + + // A dense 20-day hull: one log and one metric for every service-minute. The + // affected set is five recent service-minutes plus one 20-day-old bucket, + // which is all the pass should aggregate. + const minutes, services, oldMinute = 28800, 40, 0 + recent := minutes - 1 + affected := fmt.Sprintf(`SELECT 'ns' AS namespace,make_timestamp_ns(%d::BIGINT*60000000000)::TIMESTAMP_NS AS bucket,'svc-'||i::VARCHAR AS service FROM range(5) t(i) + UNION ALL SELECT 'ns',make_timestamp_ns(%d::BIGINT*60000000000)::TIMESTAMP_NS,'svc-0'`, recent, oldMinute) + + for _, q := range []string{ + "SET memory_limit='2GB'", "SET threads=4", + fmt.Sprintf(`CREATE VIEW log_fixture AS SELECT 'ns' AS namespace,'svc-'||(i%%%d)::VARCHAR AS service, + make_timestamp_ns((i//%d)::BIGINT*60000000000)::TIMESTAMPTZ_NS AS time + FROM range(%d) t(i)`, services, services, minutes*services), + fmt.Sprintf(`CREATE VIEW metric_fixture AS SELECT 'ns' AS namespace,'svc-'||(i%%%d)::VARCHAR AS service, + make_timestamp_ns((i//%d)::BIGINT*60000000000)::TIMESTAMPTZ_NS AS time,'m-0' AS name + FROM range(%d) t(i)`, services, services, minutes*services), + "COPY (SELECT * FROM log_fixture WHERE false) TO " + sqlLiteral(logSchema) + " (FORMAT PARQUET)", + "COPY log_fixture TO " + sqlLiteral(logData) + " (FORMAT PARQUET)", + "COPY (SELECT * FROM metric_fixture WHERE false) TO " + sqlLiteral(metricSchema) + " (FORMAT PARQUET)", + "COPY metric_fixture TO " + sqlLiteral(metricData) + " (FORMAT PARQUET)", + "CREATE OR REPLACE VIEW logs AS SELECT * FROM read_parquet([" + sqlLiteral(logSchema) + "," + sqlLiteral(logData) + "],union_by_name=false,hive_partitioning=false)", + "CREATE OR REPLACE VIEW metrics AS SELECT * FROM read_parquet([" + sqlLiteral(metricSchema) + "," + sqlLiteral(metricData) + "],union_by_name=false,hive_partitioning=false)", + "CREATE TEMP TABLE service_pass_affected AS " + affected, + } { + if _, err := db.Exec(q); err != nil { + t.Fatal(err) + } + } + + before := peakRSS(t) + started := time.Now() + lo, hi := time.Unix(0, 0).UTC(), time.Unix(int64(minutes*60), 0).UTC() + if _, err := insertServiceRollup(t.Context(), db, lo, hi); err != nil { + t.Fatal(err) + } + growth := peakRSS(t) - before + t.Logf("service log/metric rollup: %s, peak RSS growth %.1f MiB", time.Since(started), float64(growth)/(1<<20)) + if growth > 96<<20 { + t.Errorf("log/metric aggregates over %d service-minutes grew RSS by %d, want <=96 MiB; a late bucket must not force a full-hull aggregate", minutes*services, growth) + } + + // Only the six affected service-minutes are summarized, each with its one + // log and one metric. The dense history behind the hull is never counted. + var rows, logCount, metricCount int + if err := db.QueryRow("SELECT count(*),sum(log_count),sum(metric_count) FROM service_rollup").Scan(&rows, &logCount, &metricCount); err != nil { + t.Fatal(err) + } + if rows != 6 || logCount != 6 || metricCount != 6 { + t.Fatalf("service_rollup rows=%d log_count=%d metric_count=%d, want 6/6/6", rows, logCount, metricCount) + } +} From 2a9a01e0f7ac2053d3b641ad8c7164f080150c0e Mon Sep 17 00:00:00 2001 From: Vishal Rana Date: Sat, 14 Nov 2026 19:29:03 -0800 Subject: [PATCH 3/5] perf(telemetry): publish a pre-sorted BatchMetadata snapshot BatchMetadata() allocated and sorted the entire catalog under p.mu.RLock() on every call. It is called up to ~8 times per panel and by compaction selection, so readers serialized on the lock and a pending writer queued behind them, all to recompute an order that only changes when the catalog changes. Rebuild the canonically sorted slice once in setBatchLocked, the sole catalog mutation (already under mu), and publish it through an atomic pointer. BatchMetadata() now returns that shared snapshot with no lock, allocation, or sort. The order (generation, min ingest time, ID) and pin/capture semantics are unchanged. The returned slice is now shared and read-only. Two callers sorted it in place (batch_cache and annotations); both now clone before sorting, and the contract is documented on BatchMetadata. --- internal/engine/annotations.go | 4 +- internal/engine/batch_cache.go | 4 +- .../telemetry/batch_metadata_snapshot_test.go | 61 +++++++++++++++++++ internal/telemetry/compaction_debt.go | 1 + internal/telemetry/parquet.go | 23 ++++++- 5 files changed, 88 insertions(+), 5 deletions(-) create mode 100644 internal/telemetry/batch_metadata_snapshot_test.go diff --git a/internal/engine/annotations.go b/internal/engine/annotations.go index 85f672ee..1cf9df3e 100644 --- a/internal/engine/annotations.go +++ b/internal/engine/annotations.go @@ -6,6 +6,7 @@ import ( "errors" "fmt" "log/slog" + "slices" "sort" "strings" "time" @@ -142,7 +143,8 @@ func (d *Duck) refreshVersionRollup(ctx context.Context, waitBudget time.Duratio if err != nil { return 0, false, err } - batches := d.repository.Parquet.BatchMetadata() + // BatchMetadata returns a shared, read-only snapshot; clone it before sorting. + batches := slices.Clone(d.repository.Parquet.BatchMetadata()) // Compaction generations must not put old unprocessed outputs behind new ingest. sort.Slice(batches, func(i, j int) bool { if batches[i].MinIngestedNanos != batches[j].MinIngestedNanos { diff --git a/internal/engine/batch_cache.go b/internal/engine/batch_cache.go index 855f8ede..e22b8234 100644 --- a/internal/engine/batch_cache.go +++ b/internal/engine/batch_cache.go @@ -5,6 +5,7 @@ import ( "database/sql" "fmt" "log/slog" + "slices" "sort" "strings" "time" @@ -193,7 +194,8 @@ func (d *Duck) refreshReadCaches(ctx context.Context) (rows int64, backlog bool, if err := d.applyCacheTransfers(ctx); err != nil { return 0, true, err } - batches := d.repository.Parquet.BatchMetadata() + // BatchMetadata returns a shared, read-only snapshot; clone it before sorting. + batches := slices.Clone(d.repository.Parquet.BatchMetadata()) markers, err := batchMarkers(ctx, d.writer()) if err != nil { return 0, false, err diff --git a/internal/telemetry/batch_metadata_snapshot_test.go b/internal/telemetry/batch_metadata_snapshot_test.go new file mode 100644 index 00000000..926e895a --- /dev/null +++ b/internal/telemetry/batch_metadata_snapshot_test.go @@ -0,0 +1,61 @@ +package telemetry + +import ( + "slices" + "sort" + "strings" + "testing" +) + +func orderedBatchIDs(batches []BatchMetadata) string { + ids := make([]string, len(batches)) + for i, b := range batches { + ids[i] = b.ID + } + return strings.Join(ids, ",") +} + +// BatchMetadata publishes an immutable, pre-sorted snapshot rebuilt on each +// catalog mutation, rather than allocating and sorting the whole catalog on +// every call under the lock. The order is canonical (generation, then min +// ingest time, then ID), the same snapshot is returned until the catalog +// changes, callers must not mutate it, and a mutation shows on the next call. +func TestBatchMetadataSharesSortedSnapshot(t *testing.T) { + p := &ParquetStore{batches: make(map[string]*storedBatch)} + install := func(id string, generation uint32, minIngest int64) { + p.installBatch(&storedBatch{metadata: BatchMetadata{ID: id, Generation: generation, MinIngestedNanos: minIngest, Spans: 1}}) + } + // Installed out of canonical order. + install("z", 1, 10) + install("a", 0, 30) + install("b", 0, 20) + install("c", 0, 20) + + if got := orderedBatchIDs(p.BatchMetadata()); got != "b,c,a,z" { + t.Fatalf("canonical order = %q, want \"b,c,a,z\"", got) + } + + // No per-call allocation or sort: the same snapshot backs each call until + // the catalog mutates. + first, second := p.BatchMetadata(), p.BatchMetadata() + if len(first) == 0 || &first[0] != &second[0] { + t.Fatal("BatchMetadata reallocated the catalog without a mutation") + } + + // The snapshot is shared and read-only: a caller sorting a clone must not + // disturb what the next reader sees. + clone := slices.Clone(first) + sort.Slice(clone, func(i, j int) bool { return clone[i].ID > clone[j].ID }) + if got := orderedBatchIDs(p.BatchMetadata()); got != "b,c,a,z" { + t.Fatalf("shared snapshot was mutated: %q", got) + } + + // A prune and a commit are both reflected on the next call, re-sorted. + p.mu.Lock() + p.setBatchLocked("a", nil) // prune + p.mu.Unlock() + install("m", 0, 5) // commit + if got := orderedBatchIDs(p.BatchMetadata()); got != "m,b,c,z" { + t.Fatalf("after prune+commit = %q, want \"m,b,c,z\"", got) + } +} diff --git a/internal/telemetry/compaction_debt.go b/internal/telemetry/compaction_debt.go index eee8890d..3342a959 100644 --- a/internal/telemetry/compaction_debt.go +++ b/internal/telemetry/compaction_debt.go @@ -78,6 +78,7 @@ func (p *ParquetStore) setBatchLocked(id string, batch *storedBatch) { p.changeGeneration0Locked(batch.metadata, 1) } p.generation0Snapshot.Store(int64(p.generation0Batches)) + p.publishMetadataLocked() if p.compactionDebt >= CompactionDebtLimit && p.debtLimitReached != nil { close(p.debtLimitReached) p.debtLimitReached = nil diff --git a/internal/telemetry/parquet.go b/internal/telemetry/parquet.go index 69723db3..65752521 100644 --- a/internal/telemetry/parquet.go +++ b/internal/telemetry/parquet.go @@ -124,6 +124,10 @@ type ParquetStore struct { generation0Batches int // Published after each catalog mutation; scrapes never take mu. generation0Snapshot atomic.Int64 + // The canonically sorted catalog, republished after each mutation so + // BatchMetadata reads it without taking mu or sorting per call. The slice + // is shared and immutable; callers must not mutate it. + metadataSnapshot atomic.Pointer[[]BatchMetadata] generation0ByWindow map[int64]int compactionDebt int debtLimitReached chan struct{} @@ -215,9 +219,22 @@ func (p *ParquetStore) MergeParquet(ctx context.Context, signal string, inputs [ return stats, err } +// BatchMetadata returns the installed batches in canonical order: generation, +// then earliest ingest time, then ID. The result is a shared, immutable +// snapshot republished on every catalog mutation, so this call takes no lock +// and allocates nothing. Callers MUST NOT mutate it — no sorting, appending in +// place, or assigning elements; clone it first (slices.Clone) if they need to. func (p *ParquetStore) BatchMetadata() []BatchMetadata { - p.mu.RLock() - defer p.mu.RUnlock() + if snapshot := p.metadataSnapshot.Load(); snapshot != nil { + return *snapshot + } + return nil +} + +// publishMetadataLocked rebuilds the canonically sorted catalog snapshot. It +// runs under mu from setBatchLocked, the sole catalog mutation, so every commit, +// replacement, restoration, rollback, startup load and retention refreshes it. +func (p *ParquetStore) publishMetadataLocked() { out := make([]BatchMetadata, 0, len(p.batches)) for _, batch := range p.batches { out = append(out, batch.metadata) @@ -231,7 +248,7 @@ func (p *ParquetStore) BatchMetadata() []BatchMetadata { } return out[i].ID < out[j].ID }) - return out + p.metadataSnapshot.Store(&out) } // batchMetadataByID returns one installed batch without scanning the catalog. From 16d5f8ec69d88effcac4608e57efff18bbeb0d3c Mon Sep 17 00:00:00 2001 From: Vishal Rana Date: Sat, 14 Nov 2026 21:06:09 -0800 Subject: [PATCH 4/5] test(engine): clone BatchMetadata before mutating in inventory pruning test BatchMetadata now returns a shared, immutable snapshot. The resource inventory pruning test mutated batches[0].SpanTime in place to exercise the unknown-bounds path, which corrupted the shared catalog for the bindSnapshot call later in the test and broke signal-specific pruning. Clone the slice first so the mutation stays local, matching the contract the production callers follow. --- internal/engine/resource_inventory_test.go | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/internal/engine/resource_inventory_test.go b/internal/engine/resource_inventory_test.go index 409c1da2..67e216fd 100644 --- a/internal/engine/resource_inventory_test.go +++ b/internal/engine/resource_inventory_test.go @@ -7,6 +7,7 @@ import ( "io" "os" "path/filepath" + "slices" "strings" "testing" "time" @@ -154,7 +155,9 @@ func TestResourceInventorySignalSpecificUnknownBoundsAndPruning(t *testing.T) { } w := queryrows.Window{Start: at, End: time.Unix(0, 120)} assertInventoryOracle(t, d, w, 3) - batches := repo.Parquet.BatchMetadata() + // BatchMetadata returns a shared, read-only snapshot; clone before mutating + // SpanTime below so the mutation stays local to this test. + batches := slices.Clone(repo.Parquet.BatchMetadata()) source := d.resourceInventorySource(batches, map[string]bool{"mixed": true}, w) if strings.Contains(source, "spans.parquet") || !strings.Contains(source, "logs.parquet") || strings.Contains(source, "metrics.parquet") { t.Fatal(source) From aeb9b4dccde520f4d5c714995e8ff7fca5119c0d Mon Sep 17 00:00:00 2001 From: Vishal Rana Date: Sat, 14 Nov 2026 21:34:12 -0800 Subject: [PATCH 5/5] style: gofmt struct field alignment in parquet.go --- internal/telemetry/parquet.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/internal/telemetry/parquet.go b/internal/telemetry/parquet.go index 65752521..3fa74b99 100644 --- a/internal/telemetry/parquet.go +++ b/internal/telemetry/parquet.go @@ -127,7 +127,7 @@ type ParquetStore struct { // The canonically sorted catalog, republished after each mutation so // BatchMetadata reads it without taking mu or sorting per call. The slice // is shared and immutable; callers must not mutate it. - metadataSnapshot atomic.Pointer[[]BatchMetadata] + metadataSnapshot atomic.Pointer[[]BatchMetadata] generation0ByWindow map[int64]int compactionDebt int debtLimitReached chan struct{}