Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 3 additions & 1 deletion internal/engine/annotations.go
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,7 @@ import (
"errors"
"fmt"
"log/slog"
"slices"
"sort"
"strings"
"time"
Expand Down Expand Up @@ -142,7 +143,8 @@ func (d *Duck) refreshVersionRollup(ctx context.Context, waitBudget time.Duratio
if err != nil {
return 0, false, err
}
batches := d.repository.Parquet.BatchMetadata()
// BatchMetadata returns a shared, read-only snapshot; clone it before sorting.
batches := slices.Clone(d.repository.Parquet.BatchMetadata())
// Compaction generations must not put old unprocessed outputs behind new ingest.
sort.Slice(batches, func(i, j int) bool {
if batches[i].MinIngestedNanos != batches[j].MinIngestedNanos {
Expand Down
4 changes: 3 additions & 1 deletion internal/engine/batch_cache.go
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,7 @@ import (
"database/sql"
"fmt"
"log/slog"
"slices"
"sort"
"strings"
"time"
Expand Down Expand Up @@ -193,7 +194,8 @@ func (d *Duck) refreshReadCaches(ctx context.Context) (rows int64, backlog bool,
if err := d.applyCacheTransfers(ctx); err != nil {
return 0, true, err
}
batches := d.repository.Parquet.BatchMetadata()
// BatchMetadata returns a shared, read-only snapshot; clone it before sorting.
batches := slices.Clone(d.repository.Parquet.BatchMetadata())
markers, err := batchMarkers(ctx, d.writer())
if err != nil {
return 0, false, err
Expand Down
127 changes: 88 additions & 39 deletions internal/engine/duck.go
Original file line number Diff line number Diff line change
Expand Up @@ -1613,24 +1613,31 @@ WHERE bucket >= ?::TIMESTAMP_NS::TIMESTAMPTZ_NS
// Aggregate each bounded signal scan before the final affected-bucket joins.
// Schema-first Parquet scans can have a zero cardinality estimate in DuckDB 2;
// joining raw events to the affected table can then build a hash table of every
// event. Logs and metrics aggregate before joining. Spans must be filtered to
// the affected keys first: each unaffected service-minute otherwise allocates
// its own t-digest states. A semi-join against service_pass_affected reverses
// into a build of every raw span, so the keys are bound as constants instead.
// event. Spans, logs and metrics are all filtered to the affected keys first,
// bound as a constant list rather than a semi-join against
// service_pass_affected, which reverses into a build of every raw event. A late
// or clock-skewed event pulls the event-time hull back by days; without the key
// filter the log and metric aggregates then scanned and grouped every
// service-minute of that whole hull, blowing memory and freezing the watermark.
// For spans each unaffected service-minute otherwise also allocates its own
// t-digest states. The final join only reads the materialized aggregates.
func insertServiceRollup(ctx context.Context, tx rollupSQL, lo, hi time.Time) (sql.Result, error) {
// Materialize only bounded span aggregates. Logs, metrics and the final
// affected-key join run once, preserving empty and NULL-key output rows.
_, err := tx.ExecContext(ctx, `CREATE OR REPLACE TEMP TABLE service_pass_spans (
for _, create := range []string{
`CREATE OR REPLACE TEMP TABLE service_pass_spans (
namespace VARCHAR, bucket TIMESTAMP_NS, service VARCHAR, spans BIGINT, served_spans BIGINT,
p50_ms DOUBLE, p95_ms DOUBLE, error_rate DOUBLE)`)
if err != nil {
return nil, err
p50_ms DOUBLE, p95_ms DOUBLE, error_rate DOUBLE)`,
`CREATE OR REPLACE TEMP TABLE service_pass_logs (
namespace VARCHAR, bucket TIMESTAMP_NS, service VARCHAR, log_count BIGINT)`,
`CREATE OR REPLACE TEMP TABLE service_pass_metrics (
namespace VARCHAR, bucket TIMESTAMP_NS, service VARCHAR, metric_count BIGINT)`,
} {
if _, err := tx.ExecContext(ctx, create); err != nil {
return nil, err
}
}
err = forAffectedKeys(ctx, tx,
`SELECT DISTINCT namespace, epoch_ns(bucket), service FROM service_pass_affected
WHERE namespace IS NOT NULL AND bucket IS NOT NULL AND service IS NOT NULL`, nil,
err := forAffectedKeys(ctx, tx, serviceRollupAffectedKeysSQL, nil,
`struct_pack(namespace := s.namespace, bucket := date_trunc('minute', s.start_time::TIMESTAMP_NS), service := s.service)`,
`struct_pack(namespace := ?::VARCHAR, bucket := make_timestamp_ns(?::BIGINT), service := ?::VARCHAR)`,
serviceRollupAffectedKeyExpr,
func(keys string, args []any) error {
_, err := tx.ExecContext(ctx, "INSERT INTO service_pass_spans "+serviceRollupSpanSQL(keys)+" SELECT * FROM span_agg", append(args, lo, hi)...)
if err != nil {
Expand All @@ -1642,13 +1649,37 @@ func insertServiceRollup(ctx context.Context, tx rollupSQL, lo, hi time.Time) (s
if err != nil {
return nil, err
}
res, err := tx.ExecContext(ctx, "WITH span_agg AS (SELECT * FROM service_pass_spans)"+serviceRollupJoinSQL, lo, hi, lo, hi)
// Logs and metrics aggregate the same affected keys over the same bounded
// event-time range, so one old bucket cannot force a full-hull scan.
err = forAffectedKeys(ctx, tx, serviceRollupAffectedKeysSQL, nil,
`struct_pack(namespace := l.namespace, bucket := date_trunc('minute', l.time::TIMESTAMP_NS), service := l.service)`,
serviceRollupAffectedKeyExpr,
func(keys string, args []any) error {
_, err := tx.ExecContext(ctx, serviceRollupLogInsertSQL(keys), append(args, lo, hi)...)
return err
})
if err != nil {
return nil, err
}
err = forAffectedKeys(ctx, tx, serviceRollupAffectedKeysSQL, nil,
`struct_pack(namespace := m.namespace, bucket := date_trunc('minute', m.time::TIMESTAMP_NS), service := m.service)`,
serviceRollupAffectedKeyExpr,
func(keys string, args []any) error {
_, err := tx.ExecContext(ctx, serviceRollupMetricInsertSQL(keys), append(args, lo, hi)...)
return err
})
if err != nil {
return nil, err
}
if _, err := tx.ExecContext(ctx, "DROP TABLE service_pass_spans"); err != nil {
res, err := tx.ExecContext(ctx, serviceRollupJoinSQL)
if err != nil {
return nil, err
}
for _, table := range []string{"service_pass_spans", "service_pass_logs", "service_pass_metrics"} {
if _, err := tx.ExecContext(ctx, "DROP TABLE "+table); err != nil {
return nil, err
}
}
return res, nil
}

Expand Down Expand Up @@ -1710,6 +1741,40 @@ func forAffectedKeys(ctx context.Context, tx rollupSQL, query string, queryArgs
return nil
}

// serviceRollupAffectedKeysSQL lists the distinct affected service-minutes, and
// serviceRollupAffectedKeyExpr is the constant each scan probes against. Both
// are shared by the span, log and metric aggregates so every signal is filtered
// to the same keys.
const serviceRollupAffectedKeysSQL = `SELECT DISTINCT namespace, epoch_ns(bucket), service FROM service_pass_affected
WHERE namespace IS NOT NULL AND bucket IS NOT NULL AND service IS NOT NULL`

const serviceRollupAffectedKeyExpr = `struct_pack(namespace := ?::VARCHAR, bucket := make_timestamp_ns(?::BIGINT), service := ?::VARCHAR)`

// Log and metric aggregates probe the same constant affected-key list and the
// same bounded event-time range as the span path, so one old or clock-skewed
// bucket cannot widen them into a scan and group of the whole hull. Keys are
// distinct, so each affected service-minute lands in exactly one chunk and its
// COUNT and COUNT(DISTINCT name) are exact.
func serviceRollupLogInsertSQL(keys string) string {
return `INSERT INTO service_pass_logs
SELECT l.namespace, date_trunc('minute', l.time::TIMESTAMP_NS), l.service, COUNT(*)
FROM logs l
WHERE ` + keys + `
AND l.time >= ?::TIMESTAMP_NS::TIMESTAMPTZ_NS
AND l.time < ?::TIMESTAMP_NS::TIMESTAMPTZ_NS
GROUP BY l.namespace, date_trunc('minute', l.time::TIMESTAMP_NS), l.service`
}

func serviceRollupMetricInsertSQL(keys string) string {
return `INSERT INTO service_pass_metrics
SELECT m.namespace, date_trunc('minute', m.time::TIMESTAMP_NS), m.service, COUNT(DISTINCT m.name)
FROM metrics m
WHERE ` + keys + `
AND m.time >= ?::TIMESTAMP_NS::TIMESTAMPTZ_NS
AND m.time < ?::TIMESTAMP_NS::TIMESTAMPTZ_NS
GROUP BY m.namespace, date_trunc('minute', m.time::TIMESTAMP_NS), m.service`
}

// Aggregate after probing the constant affected-key list, never by hashing raw
// spans into a join against service_pass_affected. The histogram is O(buckets).
func spanRollupInsertSQL(keys string) string {
Expand Down Expand Up @@ -1762,29 +1827,13 @@ WITH span_agg AS (
)`
}

const serviceRollupJoinSQL = `,
log_agg AS (
SELECT
l.namespace,
date_trunc('minute', l.time::TIMESTAMP_NS) AS bucket,
l.service,
COUNT(*) AS log_count
FROM logs l
WHERE l.time >= ?::TIMESTAMP_NS::TIMESTAMPTZ_NS
AND l.time < ?::TIMESTAMP_NS::TIMESTAMPTZ_NS
GROUP BY l.namespace, date_trunc('minute', l.time::TIMESTAMP_NS), l.service
),
metric_agg AS (
SELECT
m.namespace,
date_trunc('minute', m.time::TIMESTAMP_NS) AS bucket,
m.service,
COUNT(DISTINCT m.name) AS metric_count
FROM metrics m
WHERE m.time >= ?::TIMESTAMP_NS::TIMESTAMPTZ_NS
AND m.time < ?::TIMESTAMP_NS::TIMESTAMPTZ_NS
GROUP BY m.namespace, date_trunc('minute', m.time::TIMESTAMP_NS), m.service
)
// serviceRollupJoinSQL reads the materialized span, log and metric aggregates,
// each already filtered to the affected keys, and joins them to every affected
// service-minute so empty signals still produce a zeroed row.
const serviceRollupJoinSQL = `
WITH span_agg AS (SELECT * FROM service_pass_spans),
log_agg AS (SELECT * FROM service_pass_logs),
metric_agg AS (SELECT * FROM service_pass_metrics)
INSERT INTO service_rollup (
namespace,
bucket,
Expand Down
8 changes: 8 additions & 0 deletions internal/engine/duck_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -283,11 +283,19 @@ func TestRollupAdmissionDoesNotCancelAdmittedWork(t *testing.T) {
mock.ExpectExec("DELETE FROM resource_rollup").WillReturnResult(sqlmock.NewResult(0, 0))
mock.ExpectExec("DELETE FROM receiver_entity_rollup").WillReturnResult(sqlmock.NewResult(0, 0))
mock.ExpectExec("CREATE OR REPLACE TEMP TABLE service_pass_spans").WillReturnResult(sqlmock.NewResult(0, 0))
mock.ExpectExec("CREATE OR REPLACE TEMP TABLE service_pass_logs").WillReturnResult(sqlmock.NewResult(0, 0))
mock.ExpectExec("CREATE OR REPLACE TEMP TABLE service_pass_metrics").WillReturnResult(sqlmock.NewResult(0, 0))
mock.ExpectQuery("FROM service_pass_affected").WillReturnRows(sqlmock.NewRows([]string{"namespace", "bucket", "service"}).AddRow("ns", int64(0), "api"))
mock.ExpectExec("INSERT INTO service_pass_spans").WillReturnResult(sqlmock.NewResult(0, 0))
mock.ExpectExec("INSERT INTO span_rollup").WillReturnResult(sqlmock.NewResult(0, 0))
mock.ExpectQuery("FROM service_pass_affected").WillReturnRows(sqlmock.NewRows([]string{"namespace", "bucket", "service"}).AddRow("ns", int64(0), "api"))
mock.ExpectExec("INSERT INTO service_pass_logs").WillReturnResult(sqlmock.NewResult(0, 0))
mock.ExpectQuery("FROM service_pass_affected").WillReturnRows(sqlmock.NewRows([]string{"namespace", "bucket", "service"}).AddRow("ns", int64(0), "api"))
mock.ExpectExec("INSERT INTO service_pass_metrics").WillReturnResult(sqlmock.NewResult(0, 0))
mock.ExpectExec("WITH span_agg").WillReturnResult(sqlmock.NewResult(0, 0))
mock.ExpectExec("DROP TABLE service_pass_spans").WillReturnResult(sqlmock.NewResult(0, 0))
mock.ExpectExec("DROP TABLE service_pass_logs").WillReturnResult(sqlmock.NewResult(0, 0))
mock.ExpectExec("DROP TABLE service_pass_metrics").WillReturnResult(sqlmock.NewResult(0, 0))
mock.ExpectQuery("FROM service_pass_affected").WillReturnRows(sqlmock.NewRows([]string{"namespace", "bucket", "service"}).AddRow("ns", int64(0), "api"))
mock.ExpectExec("INSERT INTO resource_rollup").WillReturnResult(sqlmock.NewResult(0, 0))
mock.ExpectExec("INSERT INTO endpoint_rollup").WillReturnResult(sqlmock.NewResult(0, 0))
Expand Down
5 changes: 4 additions & 1 deletion internal/engine/resource_inventory_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,7 @@ import (
"io"
"os"
"path/filepath"
"slices"
"strings"
"testing"
"time"
Expand Down Expand Up @@ -154,7 +155,9 @@ func TestResourceInventorySignalSpecificUnknownBoundsAndPruning(t *testing.T) {
}
w := queryrows.Window{Start: at, End: time.Unix(0, 120)}
assertInventoryOracle(t, d, w, 3)
batches := repo.Parquet.BatchMetadata()
// BatchMetadata returns a shared, read-only snapshot; clone before mutating
// SpanTime below so the mutation stays local to this test.
batches := slices.Clone(repo.Parquet.BatchMetadata())
source := d.resourceInventorySource(batches, map[string]bool{"mixed": true}, w)
if strings.Contains(source, "spans.parquet") || !strings.Contains(source, "logs.parquet") || strings.Contains(source, "metrics.parquet") {
t.Fatal(source)
Expand Down
60 changes: 58 additions & 2 deletions internal/engine/rollup_keys_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -408,7 +408,63 @@ func TestEdgeRollupKeysMatchConstantList(t *testing.T) {
}
}

// Unchunked reference for differential tests of the bounded implementation.
// Unchunked, full-scan reference for differential tests of the bounded
// implementation. span_agg comes from one bounded span scan; log_agg and
// metric_agg scan logs and metrics directly over the same event-time range.
// Production insertServiceRollup instead materializes all three per
// affected-key chunk; these tests pin that both produce identical rows.
func serviceRollupInsertSQL(keys string) string {
return serviceRollupSpanSQL(keys) + serviceRollupJoinSQL
return serviceRollupSpanSQL(keys) + serviceRollupReferenceJoinSQL
}

const serviceRollupReferenceJoinSQL = `,
log_agg AS (
SELECT
l.namespace,
date_trunc('minute', l.time::TIMESTAMP_NS) AS bucket,
l.service,
COUNT(*) AS log_count
FROM logs l
WHERE l.time >= ?::TIMESTAMP_NS::TIMESTAMPTZ_NS
AND l.time < ?::TIMESTAMP_NS::TIMESTAMPTZ_NS
GROUP BY l.namespace, date_trunc('minute', l.time::TIMESTAMP_NS), l.service
),
metric_agg AS (
SELECT
m.namespace,
date_trunc('minute', m.time::TIMESTAMP_NS) AS bucket,
m.service,
COUNT(DISTINCT m.name) AS metric_count
FROM metrics m
WHERE m.time >= ?::TIMESTAMP_NS::TIMESTAMPTZ_NS
AND m.time < ?::TIMESTAMP_NS::TIMESTAMPTZ_NS
GROUP BY m.namespace, date_trunc('minute', m.time::TIMESTAMP_NS), m.service
)
INSERT INTO service_rollup (
namespace,
bucket,
service,
spans,
served_spans,
p50_ms,
p95_ms,
error_rate,
log_count,
metric_count
)
SELECT
a.namespace,
a.bucket,
a.service,
COALESCE(s.spans, 0),
COALESCE(s.served_spans, 0),
COALESCE(s.p50_ms, 0),
COALESCE(s.p95_ms, 0),
COALESCE(s.error_rate, 0),
COALESCE(l.log_count, 0),
COALESCE(m.metric_count, 0)
FROM service_pass_affected a
LEFT JOIN span_agg s USING (namespace, bucket, service)
LEFT JOIN log_agg l USING (namespace, bucket, service)
LEFT JOIN metric_agg m USING (namespace, bucket, service)
WHERE a.service != '';`
81 changes: 81 additions & 0 deletions internal/engine/rollup_late_signal_memory_test.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,81 @@
package engine

import (
"fmt"
"path/filepath"
"testing"
"time"
)

// One late span (or one skewed clock) pulls the affected hull back many days.
// The span path already filters its scan and aggregation to the affected keys,
// but the log and metric aggregates used to run over the whole hull with no
// key filter: a single old bucket then forced a hash aggregate over every
// service-minute of dense history, which blew memory and froze the watermark.
// This pins that the log/metric aggregates stay bounded to the affected keys.
func TestServiceLateSignalLogMetricMemory(t *testing.T) {
if !rollupMemoryProcess(t) {
return
}
db := openTestDuck(t)
if err := CreateTables(db); err != nil {
t.Fatal(err)
}
if err := CreateViews(db); err != nil {
t.Fatal(err)
}
dir := t.TempDir()
logSchema, logData := filepath.Join(dir, "log_schema.parquet"), filepath.Join(dir, "log_data.parquet")
metricSchema, metricData := filepath.Join(dir, "metric_schema.parquet"), filepath.Join(dir, "metric_data.parquet")

// A dense 20-day hull: one log and one metric for every service-minute. The
// affected set is five recent service-minutes plus one 20-day-old bucket,
// which is all the pass should aggregate.
const minutes, services, oldMinute = 28800, 40, 0
recent := minutes - 1
affected := fmt.Sprintf(`SELECT 'ns' AS namespace,make_timestamp_ns(%d::BIGINT*60000000000)::TIMESTAMP_NS AS bucket,'svc-'||i::VARCHAR AS service FROM range(5) t(i)
UNION ALL SELECT 'ns',make_timestamp_ns(%d::BIGINT*60000000000)::TIMESTAMP_NS,'svc-0'`, recent, oldMinute)

for _, q := range []string{
"SET memory_limit='2GB'", "SET threads=4",
fmt.Sprintf(`CREATE VIEW log_fixture AS SELECT 'ns' AS namespace,'svc-'||(i%%%d)::VARCHAR AS service,
make_timestamp_ns((i//%d)::BIGINT*60000000000)::TIMESTAMPTZ_NS AS time
FROM range(%d) t(i)`, services, services, minutes*services),
fmt.Sprintf(`CREATE VIEW metric_fixture AS SELECT 'ns' AS namespace,'svc-'||(i%%%d)::VARCHAR AS service,
make_timestamp_ns((i//%d)::BIGINT*60000000000)::TIMESTAMPTZ_NS AS time,'m-0' AS name
FROM range(%d) t(i)`, services, services, minutes*services),
"COPY (SELECT * FROM log_fixture WHERE false) TO " + sqlLiteral(logSchema) + " (FORMAT PARQUET)",
"COPY log_fixture TO " + sqlLiteral(logData) + " (FORMAT PARQUET)",
"COPY (SELECT * FROM metric_fixture WHERE false) TO " + sqlLiteral(metricSchema) + " (FORMAT PARQUET)",
"COPY metric_fixture TO " + sqlLiteral(metricData) + " (FORMAT PARQUET)",
"CREATE OR REPLACE VIEW logs AS SELECT * FROM read_parquet([" + sqlLiteral(logSchema) + "," + sqlLiteral(logData) + "],union_by_name=false,hive_partitioning=false)",
"CREATE OR REPLACE VIEW metrics AS SELECT * FROM read_parquet([" + sqlLiteral(metricSchema) + "," + sqlLiteral(metricData) + "],union_by_name=false,hive_partitioning=false)",
"CREATE TEMP TABLE service_pass_affected AS " + affected,
} {
if _, err := db.Exec(q); err != nil {
t.Fatal(err)
}
}

before := peakRSS(t)
started := time.Now()
lo, hi := time.Unix(0, 0).UTC(), time.Unix(int64(minutes*60), 0).UTC()
if _, err := insertServiceRollup(t.Context(), db, lo, hi); err != nil {
t.Fatal(err)
}
growth := peakRSS(t) - before
t.Logf("service log/metric rollup: %s, peak RSS growth %.1f MiB", time.Since(started), float64(growth)/(1<<20))
if growth > 96<<20 {
t.Errorf("log/metric aggregates over %d service-minutes grew RSS by %d, want <=96 MiB; a late bucket must not force a full-hull aggregate", minutes*services, growth)
}

// Only the six affected service-minutes are summarized, each with its one
// log and one metric. The dense history behind the hull is never counted.
var rows, logCount, metricCount int
if err := db.QueryRow("SELECT count(*),sum(log_count),sum(metric_count) FROM service_rollup").Scan(&rows, &logCount, &metricCount); err != nil {
t.Fatal(err)
}
if rows != 6 || logCount != 6 || metricCount != 6 {
t.Fatalf("service_rollup rows=%d log_count=%d metric_count=%d, want 6/6/6", rows, logCount, metricCount)
}
}
Loading
Loading