Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
39 changes: 36 additions & 3 deletions internal/engine/file_snapshot.go
Original file line number Diff line number Diff line change
Expand Up @@ -29,6 +29,28 @@ func overlapping(b telemetry.BatchMetadata, signal string, w queryrows.Window) b
return count > 0 && (!bounds.Known || bounds.MaxNanos >= w.Start.UnixNano() && bounds.MinNanos < w.End.UnixNano())
}

// signalWindowBounded reports whether a windowed column from a trusted SQL
// panel bounds this signal's own event-time clock, so that pruning files to the
// read window is sound. A bare column must equal the clock; a qualified column
// must name the signal's own relation (alias qualifiers we cannot resolve stay
// unbounded, which binds every file — correct, if slower). Columns come only
// from the validated macro expansion, never from unvalidated statement text.
func signalWindowBounded(columns []string, s *telemetry.Signal) bool {
clock := s.Clock()
for _, column := range columns {
if dot := strings.LastIndexByte(column, '.'); dot >= 0 {
if column[dot+1:] == clock && strings.EqualFold(column[:dot], s.Name) {
return true
}
continue
}
if column == clock {
return true
}
}
return false
}

func (d *Duck) physicalSource(signal string, batches []telemetry.BatchMetadata) string {
s, _ := telemetry.LookupSignal(signal)
sort.SliceStable(batches, func(i, j int) bool {
Expand Down Expand Up @@ -148,22 +170,33 @@ func (d *Duck) bindSnapshot(ctx context.Context, db snapshotSQL, query string, w
}
queryrows.RecordCoverage(ctx, fresh)
}
windowColumns, scopedColumns := queryrows.ReadWindowColumns(ctx)
for _, s := range telemetry.Signals {
signal := s.Name
if _, raw := compiled.names["telemetry."+signal]; !raw {
if _, clean := compiled.names[signal]; !clean {
continue
}
}
// A raw signal is event-time pruned only when the statement actually
// bounds its own clock with the window. When a trusted SQL panel windows
// on another column (or joins this signal unwindowed), pruning by the
// clock would drop rows the statement keeps, so bind every captured file.
selectWindow := w
if scopedColumns && !signalWindowBounded(windowColumns, s) {
selectWindow = allTime
}
selected := []telemetry.BatchMetadata{}
for _, b := range batches {
if overlapping(b, signal, w) {
if overlapping(b, signal, selectWindow) {
selected = append(selected, b)
}
}
// Only an explicit internal window may prune. Public arbitrary SQL
// uses allTime and always binds every captured file.
if signal == "metrics" && len(compiled.metricNames) > 0 && w != allTime {
// uses allTime and always binds every captured file. Metric-name
// pruning likewise requires the metrics signal to be genuinely
// window-bounded.
if signal == "metrics" && len(compiled.metricNames) > 0 && selectWindow != allTime {
var err error
selected, err = pruneMetricNames(ctx, db, selected, compiled.metricNames)
if err != nil {
Expand Down
63 changes: 63 additions & 0 deletions internal/engine/file_snapshot_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -94,6 +94,69 @@ func TestSnapshotUsesActualEventBoundsAndPreservesParameters(t *testing.T) {
}

}

// A trusted SQL panel may window on a column other than a signal's
// footer-sorted event-time clock. Pruning raw files by the clock then drops
// rows the panel's own predicate keeps. When the windowed column is not the
// signal's clock, every captured file for that signal must bind.
func TestSQLPanelBindsAllFilesWhenWindowColumnIsNotTheFooterClock(t *testing.T) {
d, repo := newBatchCacheTest(t)
at := time.Date(2026, 10, 1, 12, 0, 0, 0, time.UTC)
// Spans are footer-sorted on start_time, but this panel windows on end_time.
commit := func(id string, start, end time.Time) {
t.Helper()
if err := repo.Commit(t.Context(), telemetrystore.Batch{ID: id, Spans: []telemetry.Span{{
TraceID: "trace", SpanID: id, StartUnixNanos: start.UnixNano(), EndUnixNanos: end.UnixNano(), IngestedAt: at.UnixNano(),
}}}); err != nil {
t.Fatal(err)
}
}
commit("before", at.Add(-time.Hour), at.Add(30*time.Second)) // starts before the window, ends inside it
commit("inside", at, at.Add(time.Second))
commit("after", at.Add(time.Hour), at.Add(time.Hour+time.Second))

w := queryrows.Window{Start: at, End: at.Add(time.Minute)}
ctx := queryrows.WithWindowColumns(queryrows.WithWindow(t.Context(), w), []string{"end_time"})
var count int
q := `SELECT count(*) FROM spans WHERE end_time>=?::TIMESTAMP_NS::TIMESTAMPTZ_NS AND end_time<?::TIMESTAMP_NS::TIMESTAMPTZ_NS`
if err := d.QueryRowScan(ctx, []any{&count}, q, w.Start, w.End); err != nil {
t.Fatal(err)
}
if count != 2 {
t.Fatalf("end_time window pruned by the start_time footer dropped rows: count=%d want 2", count)
}
}

// The default dashboard windows spans on their clock (start_time). That common
// case must still prune to the overlapping files, with no file-selection
// regression from the non-clock fix above.
func TestSQLPanelPrunesFilesWhenWindowColumnIsTheFooterClock(t *testing.T) {
d, repo := newBatchCacheTest(t)
at := time.Date(2026, 10, 1, 12, 0, 0, 0, time.UTC)
for i := range 3 {
start := at.Add(time.Duration(i-1) * time.Hour)
if err := repo.Commit(t.Context(), telemetrystore.Batch{ID: fmt.Sprint("batch", i), Spans: []telemetry.Span{{
TraceID: "trace", SpanID: fmt.Sprint(i), StartUnixNanos: start.UnixNano(), EndUnixNanos: start.UnixNano() + 1, IngestedAt: at.UnixNano(),
}}}); err != nil {
t.Fatal(err)
}
}
w := queryrows.Window{Start: at, End: at.Add(time.Minute)}
conn, err := d.DB.Conn(t.Context())
if err != nil {
t.Fatal(err)
}
defer conn.Close()
ctx := queryrows.WithWindowColumns(t.Context(), []string{"start_time"})
rewritten, err := d.bindSnapshot(ctx, conn, `SELECT * FROM spans`, w)
if err != nil {
t.Fatal(err)
}
if strings.Contains(rewritten, "batch0.batch") || strings.Contains(rewritten, "batch2.batch") || !strings.Contains(rewritten, "batch1.batch") {
t.Fatalf("clock window did not prune to the overlapping file: %s", rewritten)
}
}

func TestCompletedBatchCachesIncludeLatePublicationAndPartialMinutes(t *testing.T) {
d, repo := newBatchCacheTest(t)
at := time.Date(2026, 10, 1, 12, 0, 0, 0, time.UTC)
Expand Down
2 changes: 1 addition & 1 deletion internal/query/check.go
Original file line number Diff line number Diff line change
Expand Up @@ -271,7 +271,7 @@ func checkValidated(ctx context.Context, parser Parser, d *Dashboard, memo *chec
}

func checkSQLPanel(ctx context.Context, parser Parser, p *Panel, vars map[string]Variable) error {
expanded, err := expandMacros(p.SQL, time.Minute)
expanded, _, err := expandMacros(p.SQL, time.Minute)
if err != nil {
return err
}
Expand Down
3 changes: 3 additions & 0 deletions internal/query/exec.go
Original file line number Diff line number Diff line change
Expand Up @@ -952,6 +952,9 @@ func (e *Executor) runScope(ctx context.Context, p *Panel, checked *Checked, sco
if err != nil {
return nil, "", err
}
// Carry the panel's windowed columns so the engine prunes a raw signal's
// files only when the statement bounds that signal's own event-time clock.
ctx = queryrows.WithWindowColumns(ctx, plan.windowColumns)
patterns := plan.patterns
query, _, args, err := bindParams(plan.canonical, func(name string) ([]any, bool, error) {
switch name {
Expand Down
2 changes: 1 addition & 1 deletion internal/query/log_redaction_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -18,7 +18,7 @@ func TestSQLPanelExecutionMatchesDescribedStructBinding(t *testing.T) {
e := NewExecutor(engine, 30)
e.now = func() time.Time { return fixtureStart.Add(time.Hour) }
text := "SELECT telemetry.logs.body AS b FROM (SELECT {'body': 42} AS logs) AS telemetry, logs WHERE $__window(logs.time)"
expanded, err := expandMacros(text, time.Minute)
expanded, _, err := expandMacros(text, time.Minute)
if err != nil {
t.Fatal(err)
}
Expand Down
9 changes: 6 additions & 3 deletions internal/query/params_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -43,18 +43,21 @@ func TestBindParams(t *testing.T) {
}

func TestExpandMacros(t *testing.T) {
out, err := expandMacros("SELECT $__bucket(s.start_time) AS t, count(*) FROM spans s WHERE $__window(s.start_time) GROUP BY 1", 5*time.Minute)
out, columns, err := expandMacros("SELECT $__bucket(s.start_time) AS t, count(*) FROM spans s WHERE $__window(s.start_time) GROUP BY 1", 5*time.Minute)
if err != nil {
t.Fatal(err)
}
want := "SELECT time_bucket(INTERVAL '300 seconds', s.start_time::TIMESTAMP_NS) AS t, count(*) FROM spans s WHERE s.start_time >= $__from::TIMESTAMP_NS::TIMESTAMPTZ_NS AND s.start_time < $__to::TIMESTAMP_NS::TIMESTAMPTZ_NS GROUP BY 1"
if out != want {
t.Fatalf("got %s\nwant %s", out, want)
}
if _, err := expandMacros("SELECT count(*) FROM spans", time.Minute); err == nil || !strings.Contains(err.Error(), "$__window") {
if len(columns) != 1 || columns[0] != "s.start_time" {
t.Fatalf("windowed columns = %v, want [s.start_time]", columns)
}
if _, _, err := expandMacros("SELECT count(*) FROM spans", time.Minute); err == nil || !strings.Contains(err.Error(), "$__window") {
t.Fatalf("missing window macro err = %v", err)
}
if _, err := expandMacros("SELECT $__interval FROM spans WHERE $__window(start_time)", time.Minute); err == nil {
if _, _, err := expandMacros("SELECT $__interval FROM spans WHERE $__window(start_time)", time.Minute); err == nil {
t.Fatal("unknown macro accepted")
}
}
Expand Down
13 changes: 9 additions & 4 deletions internal/query/sqlpanel.go
Original file line number Diff line number Diff line change
Expand Up @@ -23,12 +23,17 @@ var (
// never carry SQL. Every SQL panel must declare its selected time window.
// Its snapshot covers that window plus the metric lookback before it, so an
// explicit lookback predicate of up to ten minutes stays visible.
func expandMacros(sql string, interval time.Duration) (string, error) {
func expandMacros(sql string, interval time.Duration) (string, []string, error) {
if !windowMacro.MatchString(sql) {
return "", errors.New("SQL panels must filter time with $__window(time_column)")
return "", nil, errors.New("SQL panels must filter time with $__window(time_column)")
}
var columns []string
out := windowMacro.ReplaceAllStringFunc(sql, func(macro string) string {
column := windowMacro.FindStringSubmatch(macro)[1]
// The macro argument is the only legitimate prune key, and it is
// unambiguous here. Record it so file pruning can later check it against
// each raw signal's event-time clock.
columns = append(columns, column)
// Analytical bucket columns can be TIMESTAMP_NS. Make their UTC
// instant explicit; raw signal clocks already carry TIMESTAMPTZ_NS.
if column == "bucket" || strings.HasSuffix(column, ".bucket") {
Expand All @@ -43,10 +48,10 @@ func expandMacros(sql string, interval time.Duration) (string, error) {
out = bucketMacro.ReplaceAllString(out, fmt.Sprintf(`time_bucket(INTERVAL '%d seconds', $1::TIMESTAMP_NS)`, seconds))
for _, m := range macroToken.FindAllStringSubmatch(stripQuoted(out), -1) {
if m[1] != "from" && m[1] != "to" {
return "", fmt.Errorf("unknown macro $__%s; use $__window(column) or $__bucket(column)", m[1])
return "", nil, fmt.Errorf("unknown macro $__%s; use $__window(column) or $__bucket(column)", m[1])
}
}
return out, nil
return out, columns, nil
}

// canonicalSQL parses macro-expanded panel SQL, renders it back with DuckDB,
Expand Down
8 changes: 6 additions & 2 deletions internal/query/sqlpanel_plan.go
Original file line number Diff line number Diff line change
Expand Up @@ -15,10 +15,14 @@ type sqlPanelPlan struct {
canonical, prepared string
serialized []bool
patterns map[string]bool
// windowColumns are the columns the panel's $__window macros bound, in
// expansion order. File pruning binds every captured file for any raw
// signal whose event-time clock is absent from this set.
windowColumns []string
}

func (e *Executor) sqlPanel(ctx context.Context, source string, interval time.Duration) (sqlPanelPlan, error) {
expanded, err := expandMacros(source, interval)
expanded, windowColumns, err := expandMacros(source, interval)
if err != nil {
return sqlPanelPlan{}, err
}
Expand Down Expand Up @@ -49,7 +53,7 @@ func (e *Executor) sqlPanel(ctx context.Context, source string, interval time.Du
if err != nil {
return sqlPanelPlan{}, err
}
plan = sqlPanelPlan{canonical: canonical, prepared: prepared, serialized: serialized, patterns: engine.PatternParams(node)}
plan = sqlPanelPlan{canonical: canonical, prepared: prepared, serialized: serialized, patterns: engine.PatternParams(node), windowColumns: windowColumns}
e.mu.Lock()
if e.sqlPanels == nil || len(e.sqlPanels) >= 128 {
e.sqlPanels = map[string]sqlPanelPlan{}
Expand Down
17 changes: 17 additions & 0 deletions internal/queryrows/window.go
Original file line number Diff line number Diff line change
Expand Up @@ -20,6 +20,23 @@ type Window struct {

type windowKey struct{}
type inventoryWindowKey struct{}
type windowColumnsKey struct{}

// WithWindowColumns records the exact columns a trusted SQL panel bounds with
// the read window, captured at macro expansion where each column is
// unambiguous. A raw signal is event-time pruned by the window only when one of
// these columns is that signal's own event-time clock; a window on any other
// column (or on another relation joined to it) binds every captured file for
// the signal instead of silently dropping rows. Absent entirely, every raw
// signal keeps its default clock-based pruning.
func WithWindowColumns(ctx context.Context, columns []string) context.Context {
return context.WithValue(ctx, windowColumnsKey{}, columns)
}

func ReadWindowColumns(ctx context.Context) ([]string, bool) {
columns, ok := ctx.Value(windowColumnsKey{}).([]string)
return columns, ok
}

// WithInventoryWindow records the logical panel interval before file lookback
// expansion. It is independent of the raw-file read window and read kind.
Expand Down
Loading