diff --git a/internal/engine/compile_cache_lru_test.go b/internal/engine/compile_cache_lru_test.go new file mode 100644 index 00000000..93e27483 --- /dev/null +++ b/internal/engine/compile_cache_lru_test.go @@ -0,0 +1,53 @@ +package engine + +import ( + "fmt" + "testing" +) + +// The snapshot-compile cache is an LRU: reaching capacity evicts exactly one +// entry, the least recently used, rather than clearing the whole cache. A +// wholesale clear would throw away a dashboard's stable, repeatedly-run +// statements every time a burst of one-off queries crossed the bound. +func TestCompileCacheEvictsOneEntryNotAll(t *testing.T) { + d, _ := newBatchCacheTest(t) + ctx := t.Context() + // Distinct statement text per index keys a distinct cache entry. + query := func(i int) string { return fmt.Sprintf("SELECT %d FROM spans", i) } + + // Fill the cache to capacity; query(0) is the least recently used. + for i := 0; i < snapshotCacheCapacity; i++ { + if _, err := d.compileSnapshot(ctx, d.DB, query(i)); err != nil { + t.Fatal(err) + } + } + // One more entry overflows the cache, evicting exactly the one LRU entry. + if _, err := d.compileSnapshot(ctx, d.DB, query(snapshotCacheCapacity)); err != nil { + t.Fatal(err) + } + + // Every entry except the evicted query(0) must still be cached. Re-reading + // them is a hit and never evicts, so order among survivors is irrelevant. + hits0, misses0 := d.SnapshotCacheStats() + for i := 1; i <= snapshotCacheCapacity; i++ { + if _, err := d.compileSnapshot(ctx, d.DB, query(i)); err != nil { + t.Fatal(err) + } + } + hits1, misses1 := d.SnapshotCacheStats() + if misses1 != misses0 { + t.Fatalf("surviving entries should all hit, but %d missed (wholesale clear?)", misses1-misses0) + } + if hits1-hits0 != int64(snapshotCacheCapacity) { + t.Fatalf("expected %d hits from survivors, got %d", snapshotCacheCapacity, hits1-hits0) + } + + // The single oldest entry, and only it, was evicted: a fresh miss. + if _, err := d.compileSnapshot(ctx, d.DB, query(0)); err != nil { + t.Fatal(err) + } + _, misses2 := d.SnapshotCacheStats() + if misses2 != misses1+1 { + t.Fatalf("query(0) should have been the one evicted; misses %d->%d", misses1, misses2) + } +} diff --git a/internal/engine/duck.go b/internal/engine/duck.go index 9fcaed39..d4915836 100644 --- a/internal/engine/duck.go +++ b/internal/engine/duck.go @@ -1,6 +1,7 @@ package engine import ( + "container/list" "context" "database/sql" "database/sql/driver" @@ -39,8 +40,17 @@ type Duck struct { cachePublishMu sync.Mutex cachePublishing map[string]bool cacheTransfers []*batchCacheTransfer + // snapshotMu guards the snapshot-compile cache: the most-recently-used list + // and its key index. Snapshot compilation is costly (serialize/rewrite/ + // deserialize through the engine), so compiled relation aliases are kept by + // their now window-stable statement text and evicted one entry at a time. snapshotMu sync.Mutex - snapshotQueries map[string]compiledSnapshot + snapshotLRU *list.List // of *snapshotCacheEntry, most recently used first + snapshotQueries map[string]*list.Element + // snapshotHits and snapshotMisses count the compile cache's lookups so a + // benchmark or test can observe its effectiveness; see SnapshotCacheStats. + snapshotHits atomic.Int64 + snapshotMisses atomic.Int64 // DB is the read pool. Reads scan immutable Parquet concurrently, so it is // sized from the machine. DB *sql.DB @@ -238,6 +248,14 @@ func (d *Duck) runPendingCheckpoint(ctx context.Context) error { return nil } +// SnapshotCacheStats reports the cumulative snapshot-compile cache hits and +// misses since start. A miss is a statement whose compiled relation aliases +// had to be built; a hit reused them. It lets a benchmark or test measure the +// cache hit rate, which window-stable statement text is meant to raise. +func (d *Duck) SnapshotCacheStats() (hits, misses int64) { + return d.snapshotHits.Load(), d.snapshotMisses.Load() +} + const ( // Rollup progress keys in rollup_state. readCacheVersion drops that table // with the rollups, so a version change also restarts their backfill; the diff --git a/internal/engine/file_snapshot.go b/internal/engine/file_snapshot.go index c985e020..1533f6a3 100644 --- a/internal/engine/file_snapshot.go +++ b/internal/engine/file_snapshot.go @@ -1,6 +1,7 @@ package engine import ( + "container/list" "context" "database/sql" "encoding/json" @@ -344,43 +345,87 @@ type compiledSnapshot struct { metricNames []string } +// snapshotCacheCapacity bounds the compile cache. Reaching it evicts the single +// least-recently-used entry, so a burst of one-off statements cannot discard +// the dashboard's stable, repeatedly-run ones. +const snapshotCacheCapacity = 128 + +// snapshotCacheEntry is one cached compilation, held in the LRU list under its +// statement text. +type snapshotCacheEntry struct { + key string + compiled compiledSnapshot +} + +// cachedSnapshot returns a compilation for query and moves it to the front of +// the LRU when present. +func (d *Duck) cachedSnapshot(query string) (compiledSnapshot, bool) { + d.snapshotMu.Lock() + defer d.snapshotMu.Unlock() + element, ok := d.snapshotQueries[query] + if !ok { + return compiledSnapshot{}, false + } + d.snapshotLRU.MoveToFront(element) + return element.Value.(*snapshotCacheEntry).compiled, true +} + +// storeSnapshot records compiled under query at the front of the LRU, evicting +// the least-recently-used entry once over capacity. +func (d *Duck) storeSnapshot(query string, compiled compiledSnapshot) { + d.snapshotMu.Lock() + defer d.snapshotMu.Unlock() + if d.snapshotQueries == nil { + d.snapshotQueries = map[string]*list.Element{} + d.snapshotLRU = list.New() + } + if element, ok := d.snapshotQueries[query]; ok { + element.Value.(*snapshotCacheEntry).compiled = compiled + d.snapshotLRU.MoveToFront(element) + return + } + d.snapshotQueries[query] = d.snapshotLRU.PushFront(&snapshotCacheEntry{key: query, compiled: compiled}) + for d.snapshotLRU.Len() > snapshotCacheCapacity { + oldest := d.snapshotLRU.Back() + d.snapshotLRU.Remove(oldest) + delete(d.snapshotQueries, oldest.Value.(*snapshotCacheEntry).key) + } +} + // compileSnapshot compiles relation aliases once with the native parser. Per-read // scopes become ordinary CTEs, avoiding repeated JSON SQL parse/deserialize work. +// The window bounds ride as bind parameters, so a panel's statement text is +// identical across time windows and this cache hits instead of recompiling. func (d *Duck) compileSnapshot(ctx context.Context, db snapshotSQL, query string) (compiledSnapshot, error) { - d.snapshotMu.Lock() - compiled, ok := d.snapshotQueries[query] - d.snapshotMu.Unlock() - if !ok { - aliases := map[string]string{} - names := []string{"trace_candidates", "trace_tail", "resource_inventory", "receiver_rollup", "receiver_points", "resource_rollup", "receiver_entity_rollup"} - for _, s := range telemetry.Signals { - names = append(names, "telemetry."+s.Name, s.Name) - } - for _, name := range names { - alias := "__pitot_snapshot_" + strings.ReplaceAll(name, ".", "_") - aliases[name] = "SELECT * FROM " + alias - } - used := map[string]string{} - rewritten, err := compileRelations(ctx, db, query, aliases, used) + if compiled, ok := d.cachedSnapshot(query); ok { + d.snapshotHits.Add(1) + return compiled, nil + } + d.snapshotMisses.Add(1) + aliases := map[string]string{} + names := []string{"trace_candidates", "trace_tail", "resource_inventory", "receiver_rollup", "receiver_points", "resource_rollup", "receiver_entity_rollup"} + for _, s := range telemetry.Signals { + names = append(names, "telemetry."+s.Name, s.Name) + } + for _, name := range names { + alias := "__pitot_snapshot_" + strings.ReplaceAll(name, ".", "_") + aliases[name] = "SELECT * FROM " + alias + } + used := map[string]string{} + rewritten, err := compileRelations(ctx, db, query, aliases, used) + if err != nil { + return compiledSnapshot{}, err + } + // The AST records only rewritten table references, never SQL literals. + compiled := compiledSnapshot{query: rewritten, names: used} + if _, ok := used["metrics"]; ok { + root, err := parseSQLTree(ctx, db, query) if err != nil { return compiledSnapshot{}, err } - // The AST records only rewritten table references, never SQL literals. - compiled = compiledSnapshot{query: rewritten, names: used} - if _, ok := used["metrics"]; ok { - root, err := parseSQLTree(ctx, db, query) - if err != nil { - return compiledSnapshot{}, err - } - compiled.metricNames = singleMetricNames(root) - } - d.snapshotMu.Lock() - if d.snapshotQueries == nil || len(d.snapshotQueries) >= 128 { - d.snapshotQueries = map[string]compiledSnapshot{} - } - d.snapshotQueries[query] = compiled - d.snapshotMu.Unlock() + compiled.metricNames = singleMetricNames(root) } + d.storeSnapshot(query, compiled) return compiled, nil } diff --git a/internal/query/compile.go b/internal/query/compile.go index f9da7418..64b1ef75 100644 --- a/internal/query/compile.go +++ b/internal/query/compile.go @@ -254,10 +254,20 @@ func compileAggregateQuery(p *Panel, measures []Measure, filters []Filter, scope } hasTotals := bucket != "" && (p.Viz == "stat" || p.Viz == "gauge") quantiles := map[int]Measure{} + // Rolled rate measures read the covered-seconds window bounds as bind + // parameters. They appear in the measure projection, which the engine + // renders after the base CTE's own window placeholders and before no other + // placeholder, so these arrive after args in textual order. Only rate + // embeds the expression, so its parameters ride along exactly when the + // placeholder survived into the measure text. + var measureArgs []any for _, m := range measures { expr := measureExprSQL(m, sig, seconds, partition) + var coveredArgs []any if rolled { - expr = rollupMeasureSQL(m, coveredSecondsSQL(scope, bucket, hasTotals), partition) + var covered string + covered, coveredArgs = coveredSecondsSQL(scope, bucket, hasTotals) + expr = rollupMeasureSQL(m, covered, partition) } else if hasTotals { expr = strings.ReplaceAll(expr, "count(*) / "+sqlFloat(seconds), "count(*) / (CASE WHEN grouping("+bucket+")=1 THEN "+sqlFloat(scope.End.Sub(scope.Start).Seconds())+" ELSE "+sqlFloat(seconds)+" END)") } @@ -266,12 +276,16 @@ func compileAggregateQuery(p *Panel, measures []Measure, filters []Filter, scope expr = "NULL::DOUBLE" } selects = append(selects, expr+" AS "+quoteIdent(m.Alias)) + if rolled && strings.Contains(expr, "?") { + measureArgs = append(measureArgs, coveredArgs...) + } unit := m.Unit if unit == "none" { unit = "" // The catalog cannot infer a physical unit for this field. } columns = append(columns, Column{Name: m.Alias, Type: "number", Role: "measure", Unit: unit}) } + args = append(args, measureArgs...) var b strings.Builder var ranking *Ranking rankingFloor := false diff --git a/internal/query/compile_cache_params_test.go b/internal/query/compile_cache_params_test.go new file mode 100644 index 00000000..baa245b5 --- /dev/null +++ b/internal/query/compile_cache_params_test.go @@ -0,0 +1,145 @@ +package query + +import ( + "reflect" + "strings" + "testing" + "time" + + "github.com/labstack/pitot/internal/config" + "github.com/labstack/pitot/internal/engine" + "github.com/labstack/pitot/internal/queryrows" + "github.com/labstack/pitot/internal/telemetry" + telemetrystore "github.com/labstack/pitot/internal/telemetry/store" +) + +// Rate and gauge panels read their window bounds as bind parameters, so the +// same panel compiles to identical statement text across different time +// windows. That text stability is what lets the engine's snapshot-compile +// cache hit instead of reparsing a per-request string on every refresh. +func TestRatePanelCompiledTextIsWindowStable(t *testing.T) { + const interval = time.Minute + windowA := Scope{Start: fixtureStart, End: fixtureStart.Add(time.Hour), Interval: interval} + windowB := Scope{Start: fixtureStart.Add(30 * time.Minute), End: fixtureStart.Add(90 * time.Minute), Interval: interval} + // FreshTo slides with the window so the rollup freshness clamp is exercised + // and proven to ride as a parameter rather than a literal. + windowA.FreshTo = windowA.End.Add(-90 * time.Second) + windowB.FreshTo = windowB.End.Add(-90 * time.Second) + + for _, tc := range []struct { + name, from, measure, viz string + }{ + {"rollup rate", "spans", "rate()", "timeseries"}, + {"metric rate", "metrics", "rate()", "timeseries"}, + {"metric gauge avg", "metrics", "avg(value)", "timeseries"}, + } { + t.Run(tc.name, func(t *testing.T) { + p := &Panel{Viz: tc.viz, Query: &Spec{From: tc.from}} + measures := measuresOf(t, tc.from, tc.measure) + c1, err := compileQuery(p, measures, nil, windowA) + if err != nil { + t.Fatal(err) + } + c2, err := compileQuery(p, measures, nil, windowB) + if err != nil { + t.Fatal(err) + } + if c1.SQL != c2.SQL { + t.Fatalf("statement text changed with the window:\nA: %s\nB: %s", c1.SQL, c2.SQL) + } + // Every placeholder must have exactly one argument, or DuckDB would + // reject the statement; a mismatch here means the window values were + // not threaded in positional order. + for _, c := range []Compiled{c1, c2} { + if n := strings.Count(c.SQL, "?"); n != len(c.Args) { + t.Fatalf("%d placeholders but %d args:\n%s", n, len(c.Args), c.SQL) + } + } + if len(c1.Args) == 0 { + t.Fatalf("expected window bounds to bind as parameters, got none:\n%s", c1.SQL) + } + if reflect.DeepEqual(c1.Args, c2.Args) { + t.Fatalf("different windows must bind different args: %v", c1.Args) + } + }) + } +} + +// A rate panel refreshed across sliding windows now reuses one compiled +// snapshot instead of recompiling each window. The window_stable arm issues the +// real compiled text; window_varying appends a unique per-window token, which +// faithfully reproduces the pre-fix behaviour where embedded nanosecond +// literals made every window's text unique. Compare their reported hit_rate_% +// and ns/op: the stable arm both hits the cache and runs faster. +func BenchmarkRatePanelSnapshotCompileCacheHitRate(b *testing.B) { + duck := benchMetricEngine(b) + p := &Panel{Viz: "timeseries", Query: &Spec{From: "metrics", Where: []string{"name='net'"}}} + var problems Problems + measures := parseMeasures(signals["metrics"], []string{"rate()"}, "m", &problems) + if len(problems) > 0 { + b.Fatal(problems) + } + // A window wide enough to pin every seeded file; the compiled statement's + // own ? bounds do the time filtering. + wide := queryrows.Window{Start: fixtureStart.Add(-time.Hour), End: fixtureStart.Add(6 * time.Hour)} + + run := func(b *testing.B, vary bool) { + hits0, misses0 := duck.SnapshotCacheStats() + b.ResetTimer() + for i := 0; i < b.N; i++ { + // Each iteration is a different window, as an auto-refreshing + // dashboard produces. + shift := time.Duration(i) * time.Minute + scope := Scope{Start: fixtureStart.Add(shift), End: fixtureStart.Add(shift + time.Hour), Interval: time.Minute} + c, err := compileQuery(p, measures, nil, scope) + if err != nil { + b.Fatal(err) + } + sql := c.SQL + if vary { + sql += " /* window " + scope.Start.String() + " */" + } + rows, err := duck.QueryContext(queryrows.WithWindow(b.Context(), wide), sql, c.Args...) + if err != nil { + b.Fatal(err) + } + for rows.Next() { + } + rows.Close() + } + b.StopTimer() + hits, misses := duck.SnapshotCacheStats() + dh, dm := hits-hits0, misses-misses0 + rate := 0.0 + if dh+dm > 0 { + rate = float64(dh) / float64(dh+dm) + } + b.ReportMetric(rate*100, "hit_rate_%") + } + b.Run("window_stable", func(b *testing.B) { run(b, false) }) + b.Run("window_varying", func(b *testing.B) { run(b, true) }) +} + +// benchMetricEngine seeds a single counter series so the rate panel returns +// rows while the benchmark measures the read path end to end. +func benchMetricEngine(b *testing.B) *engine.Duck { + b.Helper() + cfg := config.Config{DataDir: b.TempDir(), DuckDBMemory: "256MB", DuckDBThreads: 2, DuckDBMaxConns: 4, RollupInterval: time.Hour} + repo, err := telemetrystore.Open(cfg.TelemetryDir()) + if err != nil { + b.Fatal(err) + } + d, err := engine.NewDuck(b.Context(), cfg, repo) + if err != nil { + b.Fatal(err) + } + b.Cleanup(func() { _ = d.Close(); _ = repo.Close() }) + var series []telemetry.Metric + for i := -2; i < 300; i++ { + series = append(series, metricCounter(fixtureStart.Add(time.Duration(i)*time.Minute), float64(100+60*(i+2)))) + } + if err := repo.Commit(b.Context(), telemetrystore.Batch{ID: "bench-metrics", Metrics: series}); err != nil { + b.Fatal(err) + } + return d +} diff --git a/internal/query/metric_buckets.go b/internal/query/metric_buckets.go index 275d8b39..5ff7ba83 100644 --- a/internal/query/metric_buckets.go +++ b/internal/query/metric_buckets.go @@ -36,7 +36,12 @@ func metricBucketWindowCTE(scope Scope) (string, []any) { // expansion to at most ceil(metricLookback/interval)+1 bucket rows. func metricSpreadCTE(scope Scope) (string, []any) { cte, args := metricBucketWindowCTE(scope) - start, end := fmt.Sprint(scope.Start.UnixNano()), fmt.Sprint(scope.End.UnixNano()) + // The window start and end bind as parameters so metric_intervals' text does + // not change with the time window; the bucket width stays a literal because + // it is constant for a fixed relative range. Each placeholder is appended in + // the order it appears in the clause: _lo (start), _hi (end), then the two + // lower-bound predicates (start). + startNS, endNS := scope.Start.UnixNano(), scope.End.UnixNano() bucket, lo, hi := "0::BIGINT", "_lo", "_hi" if scope.Interval > 0 { bucket = fmt.Sprintf("i*%d", scope.Interval.Milliseconds()) @@ -53,10 +58,10 @@ func metricSpreadCTE(scope Scope) (string, []any) { lookback := fmt.Sprint(metricLookback.Seconds()) cte += `, metric_intervals AS (SELECT *,elapsed_s>` + lookback + ` AS _gap, - greatest(epoch_ns(_elapsed_from::TIMESTAMP_NS),` + start + `) AS _lo, - least(epoch_ns(time::TIMESTAMP_NS),` + end + `) AS _hi - FROM metric_deltas WHERE (elapsed_s>0 AND elapsed_s<=` + lookback + ` AND time>make_timestamp_ns(` + start + `)::TIMESTAMPTZ_NS) - OR ((elapsed_s IS NULL OR elapsed_s>` + lookback + `) AND time>=make_timestamp_ns(` + start + `)::TIMESTAMPTZ_NS)), + greatest(epoch_ns(_elapsed_from::TIMESTAMP_NS),?::BIGINT) AS _lo, + least(epoch_ns(time::TIMESTAMP_NS),?::BIGINT) AS _hi + FROM metric_deltas WHERE (elapsed_s>0 AND elapsed_s<=` + lookback + ` AND time>make_timestamp_ns(?::BIGINT)::TIMESTAMPTZ_NS) + OR ((elapsed_s IS NULL OR elapsed_s>` + lookback + `) AND time>=make_timestamp_ns(?::BIGINT)::TIMESTAMPTZ_NS)), metric_interval_buckets AS (SELECT *` + expansion + ` FROM metric_intervals WHERE elapsed_s IS NULL OR _gap OR _hi>_lo), metric_overlaps AS (SELECT *,` + bucket + ` AS _spread_bucket, CASE WHEN _gap THEN elapsed_s WHEN elapsed_s>0 THEN (` + hi + `-` + lo + `)::DOUBLE/1e9 END AS _covered @@ -68,6 +73,7 @@ func metricSpreadCTE(scope Scope) (string, []any) { CASE WHEN elapsed_s IS NULL OR _gap THEN list_transform(bound_increases,lambda n: n::DOUBLE) ELSE list_transform(bound_increases,lambda n: n::DOUBLE*_covered/elapsed_s) END AS bound_increases, _covered AS elapsed_s FROM metric_overlaps WHERE elapsed_s IS NULL OR _covered>0)` + args = append(args, startNS, endNS, startNS, startNS) return cte, args } @@ -75,9 +81,15 @@ func metricSpreadCTE(scope Scope) (string, []any) { // those anchors for at most five minutes, then remove covered keys. The work // follows gaps in coverage, never a series-by-window grid. Multiple nearby // gaps choose the latest completed point for each uncovered key. -func metricCarryCandidatesCTE(scope Scope, points, covered, eligible string, inclusive bool) string { - start := fmt.Sprint(scope.Start.UnixNano()) - end := fmt.Sprint(scope.End.UnixNano()) +func metricCarryCandidatesCTE(scope Scope, points, covered, eligible string, inclusive bool) (string, []any) { + // The window start and end bind as parameters so the carry text is identical + // across time windows; bucket math stays literal because the interval is + // constant for a fixed relative range. The placeholder string is used + // wherever a nanosecond bound was rendered, including inside the bucket-index + // helper, which embeds its argument twice. Args are appended in the order the + // placeholders appear in the returned text; a non-bucketed scope drops the + // two index helpers, so those bounds are omitted then. + startP, endP := "?::BIGINT", "?::BIGINT" index, bucket, step := "0::BIGINT", "0::BIGINT", "1" if scope.Interval > 0 { index = fmt.Sprintf("_bucket//%d", scope.Interval.Milliseconds()) @@ -88,17 +100,17 @@ func metricCarryCandidatesCTE(scope Scope, points, covered, eligible string, inc if inclusive { comparison = "<=" } - return `, metric_covered AS MATERIALIZED (SELECT series_id,_bucket,` + index + ` AS _index, + cte := `, metric_covered AS MATERIALIZED (SELECT series_id,_bucket,` + index + ` AS _index, max(epoch_ns(time::TIMESTAMP_NS)) AS _last_ns FROM ` + covered + ` WHERE ` + eligible + ` GROUP BY series_id,_bucket), metric_carry_anchors AS (SELECT series_id,_index,_last_ns FROM metric_covered c WHERE NOT EXISTS (SELECT 1 FROM metric_covered n WHERE n.series_id=c.series_id AND n._bucket=c._bucket+` + step + `) - UNION ALL SELECT series_id,` + metricBucketIndexSQL(start, scope.Interval) + `-1 AS _index, + UNION ALL SELECT series_id,` + metricBucketIndexSQL(startP, scope.Interval) + `-1 AS _index, max(epoch_ns(time::TIMESTAMP_NS)) AS _last_ns FROM ` + points + ` - WHERE ` + eligible + ` AND time` + comparison + `make_timestamp_ns(` + start + `)::TIMESTAMPTZ_NS - AND time>=make_timestamp_ns(` + start + `-300000000000)::TIMESTAMPTZ_NS + WHERE ` + eligible + ` AND time` + comparison + `make_timestamp_ns(` + startP + `)::TIMESTAMPTZ_NS + AND time>=make_timestamp_ns(` + startP + `-300000000000)::TIMESTAMPTZ_NS GROUP BY series_id), metric_carry_indexes AS (SELECT series_id,_last_ns,unnest(range(_index+1, - least(` + metricBucketIndexSQL(end+"-1", scope.Interval) + `,` + + least(` + metricBucketIndexSQL(endP+"-1", scope.Interval) + `,` + metricBucketIndexSQL("_last_ns+300000000000", scope.Interval) + `)+1)) AS i FROM metric_carry_anchors), metric_carry_candidates AS (SELECT c.series_id AS _sid,` + bucket + ` AS _bucket,max(_last_ns) AS _last_ns @@ -106,12 +118,27 @@ func metricCarryCandidatesCTE(scope Scope, points, covered, eligible string, inc WHERE _last_ns` + comparison + `b._start_ns AND _last_ns>=b._start_ns-300000000000 AND NOT EXISTS (SELECT 1 FROM metric_covered p WHERE p.series_id=c.series_id AND p._bucket=b._bucket) GROUP BY c.series_id,b._bucket,i)` + startNS, endNS := scope.Start.UnixNano(), scope.End.UnixNano() + var args []any + if scope.Interval > 0 { + // metricBucketIndexSQL(startP, …) embeds the start bound twice. + args = append(args, startNS, startNS) + } + // The two anchor predicates always read the start bound. + args = append(args, startNS, startNS) + if scope.Interval > 0 { + // metricBucketIndexSQL(endP-1, …) embeds the end bound twice. + args = append(args, endNS, endNS) + } + return cte, args } // Rates alone may extend a completed delta. Covered buckets win outright; // a spread delta ending exactly on the boundary covers no new duration. -func metricRateCarryCTE(scope Scope) string { - return metricCarryCandidatesCTE(scope, "metric_deltas", "metric_spread", "elapsed_s>0", true) + `, +func metricRateCarryCTE(scope Scope) (string, []any) { + carry, args := metricCarryCandidatesCTE(scope, "metric_deltas", "metric_spread", "elapsed_s>0", true) + // The rate rows below read only columns, so they add no window parameters. + return carry + `, metric_rate_carry AS (SELECT p.* EXCLUDE (_bucket,increase,count_increase,sum_increase,bound_increases,elapsed_s), g._bucket,NULL::DOUBLE AS increase,NULL::DOUBLE AS count_increase,NULL::DOUBLE AS sum_increase, NULL::DOUBLE[] AS bound_increases,NULL::DOUBLE AS elapsed_s, @@ -121,7 +148,7 @@ func metricRateCarryCTE(scope Scope) string { ON g._sid=p.series_id AND make_timestamp_ns(g._last_ns)::TIMESTAMPTZ_NS=p.time), metric_rate_rows AS (SELECT *,CASE WHEN type='histogram' THEN count_increase ELSE increase END AS _rate_share, elapsed_s AS _rate_covered,false AS _carry FROM metric_spread - UNION ALL BY NAME SELECT * FROM metric_rate_carry)` + UNION ALL BY NAME SELECT * FROM metric_rate_carry)`, args } func metricGaugeCTE(source metricRead, scope Scope, kinds []string) (string, []any, error) { @@ -130,13 +157,19 @@ func metricGaugeCTE(source metricRead, scope Scope, kinds []string) (string, []a return "", nil, err } window, windowArgs := metricBucketWindowCTE(scope) - cte += window + fmt.Sprintf(`, metric_visible_points AS (SELECT p.* FROM metric_points p - WHERE time>=make_timestamp_ns(%d)::TIMESTAMPTZ_NS)`, scope.Start.UnixNano()) - cte += metricCarryCandidatesCTE(scope, "metric_points", "metric_visible_points", "true", false) + `, + // The visible-points lower bound binds the window start as a parameter so + // this text does not change with the window. Args follow the text order: + // the points scan, then the window CTE, then this bound, then the carry. + cte += window + `, metric_visible_points AS (SELECT p.* FROM metric_points p + WHERE time>=make_timestamp_ns(?::BIGINT)::TIMESTAMPTZ_NS)` + carry, carryArgs := metricCarryCandidatesCTE(scope, "metric_points", "metric_visible_points", "true", false) + cte += carry + `, metric_gauge_carry AS (SELECT p.* EXCLUDE (_bucket),g._bucket,true AS _carry FROM metric_carry_candidates g JOIN metric_points p ON g._sid=p.series_id AND make_timestamp_ns(g._last_ns)::TIMESTAMPTZ_NS=p.time), metric_gauge_rows AS (SELECT *,false AS _carry FROM metric_visible_points UNION ALL BY NAME SELECT * FROM metric_gauge_carry)` args = append(args, windowArgs...) + args = append(args, scope.Start.UnixNano()) + args = append(args, carryArgs...) return cte, args, nil } diff --git a/internal/query/metric_intervals_test.go b/internal/query/metric_intervals_test.go index 7ad7a6f3..0cb83fcf 100644 --- a/internal/query/metric_intervals_test.go +++ b/internal/query/metric_intervals_test.go @@ -78,10 +78,11 @@ func TestMetricGapDeltasAttributeWholeIncreaseAndBoundExpansion(t *testing.T) { // execution may automatically widen buckets to respect its limit. var rows, carried, firstBucket, lastBucket int64 var minRate, maxRate, totalIncrease float64 - if err := db.db().QueryRowContext(t.Context(), cte+metricRateCarryCTE(scope)+` SELECT + carry, carryArgs := metricRateCarryCTE(scope) + if err := db.db().QueryRowContext(t.Context(), cte+carry+` SELECT count(*),count(*) FILTER (WHERE _carry),min(_bucket),max(_bucket), min(_rate_share/_rate_covered),max(_rate_share/_rate_covered), - sum(coalesce(increase,count_increase)) FROM metric_rate_rows`, args...).Scan( + sum(coalesce(increase,count_increase)) FROM metric_rate_rows`, append(args, carryArgs...)...).Scan( &rows, &carried, &firstBucket, &lastBucket, &minRate, &maxRate, &totalIncrease); err != nil { t.Fatal(err) } diff --git a/internal/query/metric_measures.go b/internal/query/metric_measures.go index 9e80fc63..2a36abed 100644 --- a/internal/query/metric_measures.go +++ b/internal/query/metric_measures.go @@ -170,7 +170,9 @@ func compileMetricQuery(p *Panel, measures []Measure, filters []Filter, scope Sc cte += spread builderArgs = append(builderArgs, spreadArgs...) if needRate { - cte += metricRateCarryCTE(scope) + carry, carryArgs := metricRateCarryCTE(scope) + cte += carry + builderArgs = append(builderArgs, carryArgs...) plan = "metric_rate_rows" } else { cte += ", metric_spread_rows AS (SELECT *,false AS _carry FROM metric_spread)" diff --git a/internal/query/rollup_compile.go b/internal/query/rollup_compile.go index 5afc6d9f..02b24409 100644 --- a/internal/query/rollup_compile.go +++ b/internal/query/rollup_compile.go @@ -111,21 +111,30 @@ func rollupRankingSQL(extra string, m Measure, seconds float64) (string, string, return value, samples, extra } -func coveredSecondsSQL(scope Scope, bucket string, totals bool) string { +// coveredSecondsSQL returns the covered-seconds expression and the window +// bound values it reads, in the left-to-right order of the ? placeholders it +// emits. The start, FreshTo-clamped end and whole-window duration ride as bind +// parameters rather than rendered literals so the compiled statement text is +// identical across time windows of the same panel, which lets the snapshot +// compile cache hit instead of reparsing a per-request string. The bucket +// width stays a literal: it is constant for a fixed relative range, so it does +// not vary the text. Every returned value corresponds to exactly one emitted +// placeholder, so callers append them positionally and the arg count matches. +func coveredSecondsSQL(scope Scope, bucket string, totals bool) (string, []any) { end := scope.End if !scope.FreshTo.IsZero() && scope.FreshTo.Before(end) { end = scope.FreshTo } startNS, endNS := scope.Start.UnixNano(), end.UnixNano() - full := sqlFloat(max(0, end.Sub(scope.Start).Seconds())) + full := max(0, end.Sub(scope.Start).Seconds()) if bucket == "" { - return full + return "?::DOUBLE", []any{full} } - covered := fmt.Sprintf("greatest(0.0, (least(%d::BIGINT, epoch_ns(%s)+%d::BIGINT)-greatest(%d::BIGINT,epoch_ns(%s)))/1000000000.0)", endNS, bucket, scope.Interval.Nanoseconds(), startNS, bucket) + covered := fmt.Sprintf("greatest(0.0, (least(?::BIGINT, epoch_ns(%s)+%d::BIGINT)-greatest(?::BIGINT,epoch_ns(%s)))/1000000000.0)", bucket, scope.Interval.Nanoseconds(), bucket) if totals { - return "CASE WHEN grouping(" + bucket + ")=1 THEN " + full + " ELSE " + covered + " END" + return "CASE WHEN grouping(" + bucket + ")=1 THEN ?::DOUBLE ELSE " + covered + " END", []any{full, endNS, startNS} } - return covered + return covered, []any{endNS, startNS} } // Endpoint routing deliberately accepts only the nonempty-route population, diff --git a/ui/host/tests/go-fragments/performance.json b/ui/host/tests/go-fragments/performance.json index 79f27066..9a98d838 100644 --- a/ui/host/tests/go-fragments/performance.json +++ b/ui/host/tests/go-fragments/performance.json @@ -220,7 +220,7 @@ ], "rows": 1 }, - "sql": "WITH base AS (SELECT * FROM span_rollup WHERE \"bucket\" \u003e= ?::TIMESTAMP_NS AND \"bucket\" \u003c ?::TIMESTAMP_NS AND ((namespace = 'shop'))) SELECT epoch_ms(time_bucket(INTERVAL '60 seconds', \"bucket\"::TIMESTAMP_NS))::BIGINT AS \"_t\", coalesce(sum(spans),0) / nullif(greatest(0.0, (least(1791402300000000000::BIGINT, epoch_ns(time_bucket(INTERVAL '60 seconds', \"bucket\"::TIMESTAMP_NS))+60000000000::BIGINT)-greatest(1791398700000000000::BIGINT,epoch_ns(time_bucket(INTERVAL '60 seconds', \"bucket\"::TIMESTAMP_NS))))/1000000000.0),0) AS \"rate\" FROM base GROUP BY time_bucket(INTERVAL '60 seconds', \"bucket\"::TIMESTAMP_NS) ORDER BY \"_t\" LIMIT 14000", + "sql": "WITH base AS (SELECT * FROM span_rollup WHERE \"bucket\" \u003e= ?::TIMESTAMP_NS AND \"bucket\" \u003c ?::TIMESTAMP_NS AND ((namespace = 'shop'))) SELECT epoch_ms(time_bucket(INTERVAL '60 seconds', \"bucket\"::TIMESTAMP_NS))::BIGINT AS \"_t\", coalesce(sum(spans),0) / nullif(greatest(0.0, (least(?::BIGINT, epoch_ns(time_bucket(INTERVAL '60 seconds', \"bucket\"::TIMESTAMP_NS))+60000000000::BIGINT)-greatest(?::BIGINT,epoch_ns(time_bucket(INTERVAL '60 seconds', \"bucket\"::TIMESTAMP_NS))))/1000000000.0),0) AS \"rate\" FROM base GROUP BY time_bucket(INTERVAL '60 seconds', \"bucket\"::TIMESTAMP_NS) ORDER BY \"_t\" LIMIT 14000", "interval": "1m", "elapsed_ms": 0, "from_ms": 1791398700000,