From d2f21a5a6192ff9edb257a746ef48dc3b830954a Mon Sep 17 00:00:00 2001 From: Vishal Rana Date: Mon, 7 Sep 2026 10:11:17 -0700 Subject: [PATCH 1/3] perf: measure the stall, and stop making reads pay for writes MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Three changes for #241, none of which assumes a mechanism, because the first diagnosis assumed one that does not exist. **The reader's side of Parquet publication is measured.** The publisher's wait was already a histogram; the reader's was nothing at all, which is why a twelve-second query could not be attributed to anything and was blamed on a lock removed by #207. `fanout_parquet_read_wait_seconds` is labelled by reader class — a query that waits and a rollup that waits are different problems with the same symptom — and a reader that gives up is counted separately from one that waited, because a refusal and a long wait are different failures. The exclusive half of a publication is measured too, and an explicit checkpoint now reports both its duration and whether it failed: the non-FORCE form fails outright while any transaction is open, which on a live instance is most of the time. **Rollups and maintenance no longer draw from the read pool.** They take a dedicated single connection. The pool is sized one per core, so on the four-core reference host it holds four, shared by every dashboard widget, the detector, the alert engine, the readiness probe — and, until now, whichever rollup happened to be running, for as long as it ran. A request that waits for a connection waits invisibly; database/sql counts that wait and nothing exported it, so the pool's own gauges are exported now as well. One connection is all the writers can use, because the write gate already lets one through at a time. **The edge rollup yields to a queued publication.** A pass holds the Parquet snapshot until it commits, and a publication that queues behind it makes every query arriving after the grace period wait out the rest of that pass — up to eight sub-windows of work for a request that needed none of it. The pass now stops at the first sub-window boundary once a publisher is waiting. That costs nothing to implement: the resume cursor already exists for the per-pass budget, and the ingested watermark already stays behind an unfinished window. It never yields before completing one sub-window, so a busy compaction cycle cannot starve the cache the whole product reads from. What is deliberately not here: the publication budgets are untouched. They are the stall's ceiling, not its cause, and tuning them would hide the measurement that is the point of this change. Verified against a local instance: the new series appear with a query/rollup/ stats split and an idle baseline of about two microseconds per admission, against which a nine-second wait is unmistakable. The yield is pinned by a test that fails when the condition is removed. Claude-Session: https://claude.ai/code/session_017ovmTezqzvxVP8P9MGDQNk --- internal/metrics/metrics.go | 97 ++++++++++ internal/query/duck.go | 132 +++++++++++--- internal/query/parquet_gate.go | 11 ++ internal/query/rollup_yield_test.go | 180 +++++++++++++++++++ internal/query/stall_instrumentation_test.go | 128 +++++++++++++ 5 files changed, 521 insertions(+), 27 deletions(-) create mode 100644 internal/query/rollup_yield_test.go create mode 100644 internal/query/stall_instrumentation_test.go diff --git a/internal/metrics/metrics.go b/internal/metrics/metrics.go index 07e1c85af..512d637cb 100644 --- a/internal/metrics/metrics.go +++ b/internal/metrics/metrics.go @@ -1,7 +1,9 @@ package metrics import ( + "database/sql" "math" + "sync/atomic" "github.com/prometheus/client_golang/prometheus" "github.com/prometheus/client_golang/prometheus/promauto" @@ -204,6 +206,41 @@ var ( Help: "Parquet publications abandoned after reader wait timeout", }) + // The publisher's side of Parquet publication was measured and the + // reader's side was not, which left the one question a stalled query asks + // — "how long did I wait to enter the snapshot?" — unanswerable. These + // three complete the pair. + ParquetPublishHold = promauto.NewHistogram(prometheus.HistogramOpts{ + Name: "fanout_parquet_publish_hold_seconds", + Help: "Time readers were excluded while a new Parquet file set was swapped in", + Buckets: []float64{.0001, .0005, .001, .005, .01, .025, .05, .1, .25, .5, 1, 2.5, 5, 10, 30, 60}, + }) + + ParquetReadWait = promauto.NewHistogramVec(prometheus.HistogramOpts{ + Name: "fanout_parquet_read_wait_seconds", + Help: "Time a reader spent waiting to enter the Parquet snapshot", + Buckets: []float64{.0001, .0005, .001, .005, .01, .025, .05, .1, .25, .5, 1, 2.5, 5, 10, 30, 60}, + }, []string{"reader"}) + + ParquetReadRefusals = promauto.NewCounterVec(prometheus.CounterOpts{ + Name: "fanout_parquet_read_refusals_total", + Help: "Readers that gave up waiting to enter the Parquet snapshot", + }, []string{"reader"}) + + // An explicit CHECKPOINT blocks new transactions for its duration, and the + // non-FORCE form fails outright while any transaction is open — so both + // how long it takes and how often it does not happen are worth knowing. + DuckDBCheckpoint = promauto.NewHistogram(prometheus.HistogramOpts{ + Name: "fanout_duckdb_checkpoint_seconds", + Help: "Duration of an explicit DuckDB checkpoint on the rollup cache", + Buckets: []float64{.001, .005, .01, .05, .1, .25, .5, 1, 2.5, 5, 10, 30}, + }) + + DuckDBCheckpointFailures = promauto.NewCounter(prometheus.CounterOpts{ + Name: "fanout_duckdb_checkpoint_failures_total", + Help: "Explicit DuckDB checkpoints that returned an error", + }) + // HTTP metrics HTTPRequestsTotal = promauto.NewCounterVec(prometheus.CounterOpts{ Name: "fanout_http_requests_total", @@ -239,6 +276,66 @@ func RecordFlush(signal string, durationSec float64) { FlushDuration.WithLabelValues(signal).Observe(durationSec) } +// duckDBPoolStats is the live source for the connection-pool gauges below. +// +// A holder rather than a promauto.NewGaugeFunc closure over the pool: the +// gauges are declared once for the process, while a *Duck is constructed many +// times in a test binary, and registering a collector per construction panics +// on the second one. Unset, every gauge reads zero. +var duckDBPoolStats atomic.Pointer[func() sql.DBStats] + +// SetDuckDBPoolSource points the connection-pool gauges at a pool. The last +// caller wins, which is what a test binary that builds several wants. +func SetDuckDBPoolSource(stats func() sql.DBStats) { + if stats == nil { + duckDBPoolStats.Store(nil) + return + } + duckDBPoolStats.Store(&stats) +} + +func poolStats() sql.DBStats { + if fn := duckDBPoolStats.Load(); fn != nil { + return (*fn)() + } + return sql.DBStats{} +} + +// The read pool is the one that can starve: background writers, the detector, +// the alert engine and every dashboard widget draw from it, and a request that +// waits for a connection waits invisibly — database/sql counts that wait and +// nothing exported it. +var ( + _ = promauto.NewGaugeFunc(prometheus.GaugeOpts{ + Name: "fanout_duckdb_pool_in_use", + Help: "DuckDB read connections currently in use", + }, func() float64 { return float64(poolStats().InUse) }) + + _ = promauto.NewGaugeFunc(prometheus.GaugeOpts{ + Name: "fanout_duckdb_pool_idle", + Help: "DuckDB read connections currently idle", + }, func() float64 { return float64(poolStats().Idle) }) + + _ = promauto.NewGaugeFunc(prometheus.GaugeOpts{ + Name: "fanout_duckdb_pool_max_open", + Help: "Configured DuckDB read connection limit", + }, func() float64 { return float64(poolStats().MaxOpenConnections) }) + + _ = promauto.NewCounterFunc(prometheus.CounterOpts{ + Name: "fanout_duckdb_pool_waits_total", + Help: "Times a caller waited for a free DuckDB read connection", + }, func() float64 { return float64(poolStats().WaitCount) }) + + _ = promauto.NewCounterFunc(prometheus.CounterOpts{ + Name: "fanout_duckdb_pool_wait_seconds_total", + Help: "Total time callers spent waiting for a free DuckDB read connection", + }, func() float64 { return poolStats().WaitDuration.Seconds() }) +) + +// DuckDBPoolStatsForTest exposes what the pool gauges are reading, so a test +// can prove they point at the read pool rather than the write handle. +func DuckDBPoolStatsForTest() sql.DBStats { return poolStats() } + // RecordWriteGate records one complete Telemetry catalog write critical section. // Callers constrain operation to the fixed writegate.WriteOperation set so this // metric cannot grow with tenant or telemetry cardinality. diff --git a/internal/query/duck.go b/internal/query/duck.go index c5405e4ce..deff05842 100644 --- a/internal/query/duck.go +++ b/internal/query/duck.go @@ -24,7 +24,17 @@ import ( ) type Duck struct { - DB *sql.DB + // DB is the read pool. Reads scan immutable Parquet concurrently, so it is + // sized from the machine. + DB *sql.DB + // writeDB is the single connection every rollup and maintenance write uses. + // + // Those writes are serialized by writeGate anyway, so one connection is all + // they can use — and taking it from the read pool is what let a background + // pass occupy a quarter of a four-core host's read capacity for the length + // of a rollup. A request that then waits for a connection waits invisibly: + // database/sql counts that wait, and nothing charged it to anything. + writeDB *sql.DB cfg config.Config repository *telemetrystore.Repository // rollupLagNanos holds the rollup watermark back from the max ingested @@ -231,12 +241,14 @@ func NewDuck(ctx context.Context, cfg config.Config, repository *telemetrystore. return nil, err } - db, err := openDuckDB(ctx, dsn, tempDir, duckDBPoolSize(cfg)) + db, writeDB, err := openDuckDB(ctx, dsn, tempDir, duckDBPoolSize(cfg)) if err != nil { return nil, fmt.Errorf("open DuckDB query cache: %w (the cache at %s is rebuildable from Parquet)", err, dbPath) } + // Reads are the pool that can starve, so it is the one worth watching. + metrics.SetDuckDBPoolSource(db.Stats) - d := &Duck{DB: db, cfg: cfg, repository: repository, rollupLagNanos: int64(rollupPublicationSafetyLag)} + d := &Duck{DB: db, writeDB: writeDB, cfg: cfg, repository: repository, rollupLagNanos: int64(rollupPublicationSafetyLag)} if cfg.DuckDBMemory == "" { // Only when the operator hasn't pinned storage.duckdb.memory: keep DuckDB's // cgroup-aware auto limit on big boxes but leave absolute RAM headroom on @@ -245,16 +257,19 @@ func NewDuck(ctx context.Context, cfg config.Config, repository *telemetrystore. slog.Warn("apply memory headroom failed; using DuckDB default memory_limit", "err", err) } } - if err := CreateCacheTables(db); err != nil { - _ = db.Close() + // DDL goes through the write handle for the same reason every other write + // does; the two handles share one DuckDB instance, so the tables and views + // are the same ones the read pool sees. + if err := CreateCacheTables(writeDB); err != nil { + _ = d.Close() return nil, err } - if err := CreateParquetViews(db, repository.Parquet.Dir()); err != nil { - _ = db.Close() + if err := CreateParquetViews(writeDB, repository.Parquet.Dir()); err != nil { + _ = d.Close() return nil, err } - if err := CreateViews(db); err != nil { - _ = db.Close() + if err := CreateViews(writeDB); err != nil { + _ = d.Close() return nil, fmt.Errorf("create views: %w", err) } if cfg.RollupSkipToLatest { @@ -278,7 +293,7 @@ func (d *Duck) skipRollupToLatest(ctx context.Context) error { unlock := d.writeGate.Lock(writegate.WriteRollupSkip) defer unlock() - tx, err := d.DB.BeginTx(ctx, nil) + tx, err := d.writer().BeginTx(ctx, nil) if err != nil { return err } @@ -323,7 +338,10 @@ func (d *Duck) skipRollupToLatest(ctx context.Context) error { return tx.Commit() } -func openDuckDB(ctx context.Context, dsn, tempDir string, maxConns int) (*sql.DB, error) { +// openDuckDB returns the read pool and the single-connection write handle. +// Both come from one connector: DuckDB allows a file one read-write instance +// per process, so a second connector to the same DSN would fail on its lock. +func openDuckDB(ctx context.Context, dsn, tempDir string, maxConns int) (*sql.DB, *sql.DB, error) { // Every pooled connection must use UTC. DuckDB otherwise inherits the host // timezone and casts TIMESTAMPTZ rollup buckets into local wall-clock // TIMESTAMP values, while API windows arrive in UTC. @@ -352,7 +370,7 @@ func openDuckDB(ctx context.Context, dsn, tempDir string, maxConns int) (*sql.DB return nil }) if err != nil { - return nil, err + return nil, nil, err } db := sql.OpenDB(connector) @@ -363,7 +381,13 @@ func openDuckDB(ctx context.Context, dsn, tempDir string, maxConns int) (*sql.DB } db.SetMaxOpenConns(maxConns) db.SetMaxIdleConns(maxConns) - return db, nil + + // One connection is all the writers can use — writeGate already lets one + // through at a time — and keeping it out of the read pool is the point. + writeDB := sql.OpenDB(connector) + writeDB.SetMaxOpenConns(1) + writeDB.SetMaxIdleConns(1) + return db, writeDB, nil } func sqlLiteral(v string) string { @@ -371,7 +395,38 @@ func sqlLiteral(v string) string { } func (d *Duck) Close() error { - return d.DB.Close() + var writeErr error + if d.writeDB != nil { + writeErr = d.writeDB.Close() + } + metrics.SetDuckDBPoolSource(nil) + return errors.Join(writeErr, d.DB.Close()) +} + +// writer returns the handle every write goes through. NewDuck always sets one; +// the fallback is for a Duck assembled field-by-field in a test, where one +// handle is the whole point. +func (d *Duck) writer() *sql.DB { + if d.writeDB != nil { + return d.writeDB + } + return d.DB +} + +// checkpoint folds the rollup cache's WAL back into its file, and says how long +// that took. A checkpoint blocks new transactions for its duration, so it is a +// candidate explanation for a stalled query and has to be measurable. The +// non-FORCE form also fails outright while any transaction is open, which on a +// live instance is most of the time — the failure counter is how often that +// happens rather than a guess. +func (d *Duck) checkpoint(ctx context.Context) error { + start := time.Now() + _, err := d.writer().ExecContext(ctx, "CHECKPOINT") + metrics.DuckDBCheckpoint.Observe(time.Since(start).Seconds()) + if err != nil { + metrics.DuckDBCheckpointFailures.Inc() + } + return err } // DefaultNamespace returns empty string so queries search all namespaces. @@ -428,7 +483,7 @@ func (d *Duck) RunRollups(ctx context.Context) { func (d *Duck) updateParquetStats(ctx context.Context) { statsCtx, cancel := context.WithTimeout(ctx, parquetStatsWait) defer cancel() - if err := d.lockParquetRead(statsCtx); err != nil { + if err := d.lockParquetRead(statsCtx, readerStats); err != nil { slog.Warn("parquet stats skipped", "err", err) return } @@ -556,13 +611,12 @@ func (d *Duck) runRepositoryMaintenance(ctx context.Context) error { var errs []error if d.cfg.RetentionDays > 0 { for _, table := range []string{"service_rollup", "endpoint_rollup", "edge_rollup"} { - if _, err := d.DB.ExecContext(ctx, fmt.Sprintf("DELETE FROM %s WHERE bucket < now() - INTERVAL %d DAY", table, d.cfg.RetentionDays)); err != nil { + if _, err := d.writer().ExecContext(ctx, fmt.Sprintf("DELETE FROM %s WHERE bucket < now() - INTERVAL %d DAY", table, d.cfg.RetentionDays)); err != nil { errs = append(errs, fmt.Errorf("prune %s: %w", table, err)) } } } - _, checkpointErr := d.DB.ExecContext(ctx, "CHECKPOINT") - return errors.Join(errors.Join(errs...), checkpointErr) + return errors.Join(errors.Join(errs...), d.checkpoint(ctx)) }() err := errors.Join(pruneErr, cacheErr) maintenanceResult := metrics.TelemetrySuccess @@ -612,7 +666,7 @@ func (d *Duck) refreshServiceRollup(ctx context.Context) (int64, error) { } defer d.parquetMu.RUnlock() - tx, err := d.DB.BeginTx(ctx, nil) + tx, err := d.writer().BeginTx(ctx, nil) if err != nil { return 0, err } @@ -733,7 +787,7 @@ func (d *Duck) refreshEndpointRollup(ctx context.Context) (int64, error) { } defer d.parquetMu.RUnlock() - tx, err := d.DB.BeginTx(ctx, nil) + tx, err := d.writer().BeginTx(ctx, nil) if err != nil { return 0, err } @@ -856,7 +910,7 @@ func (d *Duck) refreshEdgeRollup(ctx context.Context) (int64, error) { } defer d.parquetMu.RUnlock() - tx, err := d.DB.BeginTx(ctx, nil) + tx, err := d.writer().BeginTx(ctx, nil) if err != nil { return 0, err } @@ -960,7 +1014,14 @@ WHERE ingested_unix_nano > ? maxT := maxStartT.Time processed := 0 for !subLo.After(maxT) { - if processed == maxEdgeSubWindowsPerPass { + // Stop at a resumable point when the budget is spent, and also when + // a publication is waiting: this pass holds the Parquet snapshot + // until it commits, and every query that arrives after the + // publisher's grace expires waits out whatever is left of it. One + // sub-window is the unit; the cursor below is what makes stopping + // free. Never before the first, so a pass always makes progress and + // a busy compaction cycle cannot starve the rollup. + if processed == maxEdgeSubWindowsPerPass || (processed > 0 && d.parquetMu.publisherQueued()) { completed = false nextCursor = subLo.UnixNano() break @@ -1600,7 +1661,7 @@ FROM messaging_edges;` // QueryContext executes a read against immutable Parquet files and DuckDB's // local rollup cache. func (d *Duck) QueryContext(ctx context.Context, query string, args ...any) (queryrows.Rows, error) { - if err := d.lockParquetRead(ctx); err != nil { + if err := d.lockParquetRead(ctx, readerQuery); err != nil { return nil, err } rows, err := d.DB.QueryContext(ctx, query, args...) @@ -1636,7 +1697,7 @@ func (r *lockedRows) release() { r.unlockOnce.Do(r.unlock) } // QueryRowScan executes a single-row query against immutable Parquet files and // DuckDB's local rollup cache. func (d *Duck) QueryRowScan(ctx context.Context, dest []any, query string, args ...any) error { - if err := d.lockParquetRead(ctx); err != nil { + if err := d.lockParquetRead(ctx, readerQuery); err != nil { return err } defer d.parquetMu.RUnlock() @@ -1645,7 +1706,7 @@ func (d *Duck) QueryRowScan(ctx context.Context, dest []any, query string, args // Trace pins the immutable Parquet snapshot for the full indexed-file read. func (d *Duck) Trace(ctx context.Context, query telemetry.TraceQuery) ([]telemetry.IndexedSpan, error) { - if err := d.lockParquetRead(ctx); err != nil { + if err := d.lockParquetRead(ctx, readerQuery); err != nil { return nil, err } defer d.parquetMu.RUnlock() @@ -1673,22 +1734,39 @@ func (d *Duck) PublishParquet(ctx context.Context, publish func(context.Context) return fmt.Errorf("wait for Parquet readers: %w", err) } defer d.parquetMu.Unlock() + swapStarted := time.Now() + defer func() { metrics.ParquetPublishHold.Observe(time.Since(swapStarted).Seconds()) }() swapCtx, cancelSwap := context.WithTimeout(ctx, parquetSwapBudget) defer cancelSwap() return publish(swapCtx) } -func (d *Duck) lockParquetRead(ctx context.Context) error { +// Reader classes for the snapshot-wait metric. A query that waits and a rollup +// that waits are different problems with the same symptom, and the whole point +// of measuring here is to tell them apart. +const ( + readerQuery = "query" + readerRollup = "rollup" + readerStats = "stats" +) + +// lockParquetRead admits a reader to the Parquet snapshot and records what the +// wait cost. The publisher's wait was already measured and the reader's was +// not, which is why a multi-second query could not be attributed to anything. +func (d *Duck) lockParquetRead(ctx context.Context, reader string) error { + start := time.Now() if err := d.parquetMu.RLockContext(ctx); err != nil { + metrics.ParquetReadRefusals.WithLabelValues(reader).Inc() return errors.Join(ErrParquetReadWait, err) } + metrics.ParquetReadWait.WithLabelValues(reader).Observe(time.Since(start).Seconds()) return nil } func (d *Duck) lockRollupParquetRead(ctx context.Context) error { waitCtx, cancel := context.WithTimeout(ctx, rollupReaderLease) defer cancel() - return d.lockParquetRead(waitCtx) + return d.lockParquetRead(waitCtx, readerRollup) } // ---- Queries for API ---- diff --git a/internal/query/parquet_gate.go b/internal/query/parquet_gate.go index 9945de1aa..0028ab248 100644 --- a/internal/query/parquet_gate.go +++ b/internal/query/parquet_gate.go @@ -80,6 +80,17 @@ func (g *parquetReadGate) admitsReaderLocked() bool { return g.clock().Sub(g.waiting[0].queuedAt) < g.grace() } +// publisherQueued reports whether a publication is waiting for readers to +// leave. A long-running internal reader asks between units of work so it can +// stop at a resumable point instead of making every query behind it wait out +// the rest of its pass. +func (g *parquetReadGate) publisherQueued() bool { + g.init() + g.mu.Lock() + defer g.mu.Unlock() + return g.writer || len(g.waiting) > 0 +} + func (g *parquetReadGate) TryRLock() bool { g.init() g.mu.Lock() diff --git a/internal/query/rollup_yield_test.go b/internal/query/rollup_yield_test.go new file mode 100644 index 000000000..1742cd21f --- /dev/null +++ b/internal/query/rollup_yield_test.go @@ -0,0 +1,180 @@ +package query + +import ( + "context" + "fmt" + "testing" + "time" + + "github.com/labstack/fanout/internal/config" +) + +// seedEdgeSpread writes SERVER/CLIENT pairs that share one ingested instant but +// spread their start_time over spreadMins, which is what forces the edge rollup +// to sub-window: a seed load or a backfill compresses a wide start_time range +// into a narrow ingested window. +func seedEdgeSpread(t *testing.T, d *Duck, rows, spreadMins int) { + t.Helper() + ctx := context.Background() + baseTime := time.Now().UTC().Truncate(time.Minute) + for _, spec := range []struct{ kind, span, parent, service string }{ + {"SPAN_KIND_SERVER", "parent-%d", "", "svc-a"}, + {"SPAN_KIND_CLIENT", "child-%d", "parent-%d", "svc-b"}, + } { + parent := "NULL" + if spec.parent != "" { + parent = fmt.Sprintf("printf('%s', i)", spec.parent) + } + _, err := d.DB.ExecContext(ctx, fmt.Sprintf(` +WITH input AS (SELECT CAST(? AS TIMESTAMP) AS base_time) +INSERT INTO telemetry.spans ( + namespace, trace_id, span_id, parent_span_id, service, operation, kind, + start_time, end_time, start_unix_nano, end_unix_nano, duration_ms, + status, ingested_at, ingested_unix_nano +) +SELECT 'default', printf('trace-%%d', i %% 2000), printf('%s', i), %s, '%s', 'op', '%s', + base_time - ((i %% %d) * INTERVAL '1' MINUTE), + base_time - ((i %% %d) * INTERVAL '1' MINUTE) + INTERVAL '10' MILLISECOND, + epoch_ns(base_time - ((i %% %d) * INTERVAL '1' MINUTE)), + epoch_ns(base_time - ((i %% %d) * INTERVAL '1' MINUTE)) + 10000000, + 10.0, 'STATUS_CODE_OK', base_time, ? +FROM range(?, ?) t(i), input`, + spec.span, parent, spec.service, spec.kind, spreadMins, spreadMins, spreadMins, spreadMins), + baseTime, baseTime.UnixNano(), 0, rows) + if err != nil { + t.Fatalf("insert %s: %v", spec.kind, err) + } + } +} + +func newSpreadDuck(t *testing.T, rows, spreadMins int) *Duck { + t.Helper() + db := openTestDuck(t) + if err := CreateTables(db); err != nil { + t.Fatalf("CreateTables: %v", err) + } + if err := CreateViews(db); err != nil { + t.Fatalf("CreateViews: %v", err) + } + d := &Duck{DB: db, cfg: config.Config{RetentionDays: 30, DuckDBMemory: "1GB"}} + seedEdgeSpread(t, d, rows, spreadMins) + return d +} + +// A rollup pass holds the Parquet snapshot until it commits, so a publication +// that queues behind it makes every query arriving after the grace period wait +// out the rest of that pass — up to eight sub-windows of work for a request +// that needed none of it. The pass now stops at the first sub-window boundary +// once a publisher is waiting, which is free: the resume cursor already exists, +// and the ingested watermark already stays behind an unfinished window. +func TestEdgeRollupYieldsToAQueuedPublisher(t *testing.T) { + const ( + rows = 6_000 + spreadMins = 300 + ) + ctx := context.Background() + + undisturbed := newSpreadDuck(t, rows, spreadMins) + if _, err := undisturbed.refreshEdgeRollup(ctx); err != nil { + t.Fatalf("undisturbed pass: %v", err) + } + _, freeCursor := readEdgeRollupState(t, undisturbed) + if freeCursor == 0 { + t.Fatal("the undisturbed pass consumed the whole window; this seed no longer exercises sub-windowing") + } + + yielding := newSpreadDuck(t, rows, spreadMins) + // A publisher can only queue while something is reading, so the test holds + // a reader of its own. The publisher then stays queued for the whole pass, + // which is what the rollup checks at each sub-window boundary. The grace + // period has not expired, so the pass is still admitted. + if err := yielding.parquetMu.RLockContext(ctx); err != nil { + t.Fatalf("hold a reader: %v", err) + } + queued := queuePublisher(t, yielding) + + if _, err := yielding.refreshEdgeRollup(ctx); err != nil { + t.Fatalf("yielding pass: %v", err) + } + watermark, yieldCursor := readEdgeRollupState(t, yielding) + if yieldCursor == 0 { + t.Fatal("the pass ran the whole window with a publisher waiting on it") + } + if watermark != 0 { + t.Fatalf("ingested watermark advanced to %d over a window the pass did not finish", watermark) + } + if yieldCursor >= freeCursor { + t.Fatalf("yielding pass reached cursor %d, no earlier than the undisturbed %d", yieldCursor, freeCursor) + } + + // Yielding must not stall the rollup: with the publisher gone the same + // instance finishes the window. + yielding.parquetMu.RUnlock() + queued.done() + for passes := 0; ; passes++ { + if passes > spreadMins/30+4 { + t.Fatal("edge rollup did not converge after yielding") + } + if _, err := yielding.refreshEdgeRollup(ctx); err != nil { + t.Fatalf("catch-up pass: %v", err) + } + if _, cursor := readEdgeRollupState(t, yielding); cursor == 0 { + break + } + } + if watermark, _ = readEdgeRollupState(t, yielding); watermark == 0 { + t.Fatal("watermark never advanced after the window completed") + } +} + +// A publisher that is always queued must not stop the rollup dead: every pass +// has to complete at least one sub-window, or a busy compaction cycle starves +// the cache the whole product reads from. +func TestEdgeRollupAlwaysCompletesOneSubWindow(t *testing.T) { + ctx := context.Background() + d := newSpreadDuck(t, 4_000, 180) + if err := d.parquetMu.RLockContext(ctx); err != nil { + t.Fatalf("hold a reader: %v", err) + } + queued := queuePublisher(t, d) + + rows, err := d.refreshEdgeRollup(ctx) + if err != nil { + t.Fatalf("pass with a publisher queued: %v", err) + } + if rows == 0 { + t.Fatal("a pass yielded before doing any work; a busy compaction cycle would starve the cache") + } + d.parquetMu.RUnlock() + queued.done() +} + +// queuePublisher parks a publication in the gate's wait list and returns a +// handle that releases it. It requires a reader to already hold the snapshot, +// which is the only state in which a publisher waits rather than enters. +func queuePublisher(t *testing.T, d *Duck) (handle struct{ done func() }) { + t.Helper() + entered := make(chan struct{}) + release := make(chan struct{}) + go func() { + if err := d.parquetMu.LockContext(context.Background()); err != nil { + close(entered) + return + } + close(entered) + <-release + d.parquetMu.Unlock() + }() + deadline := time.Now().Add(2 * time.Second) + for !d.parquetMu.publisherQueued() { + if time.Now().After(deadline) { + t.Fatal("publisher never queued") + } + time.Sleep(time.Millisecond) + } + handle.done = func() { + <-entered + close(release) + } + return handle +} diff --git a/internal/query/stall_instrumentation_test.go b/internal/query/stall_instrumentation_test.go new file mode 100644 index 000000000..8a222e2df --- /dev/null +++ b/internal/query/stall_instrumentation_test.go @@ -0,0 +1,128 @@ +package query + +import ( + "context" + "testing" + "time" + + "github.com/labstack/fanout/internal/config" + "github.com/labstack/fanout/internal/metrics" + telemetrystore "github.com/labstack/fanout/internal/telemetry/store" + "github.com/prometheus/client_golang/prometheus/testutil" +) + +// A rollup or a maintenance pass used to take its connection from the same +// pool every read draws from. On a four-core host that pool holds four +// connections, so one background pass occupied a quarter of the machine's read +// capacity for its whole duration — and a request that waits for a connection +// waits invisibly. +func TestWritesDoNotDrawFromTheReadPool(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + cfg := config.Config{DataDir: t.TempDir(), RollupInterval: time.Minute, DuckDBMemory: "128MB", DuckDBMaxConns: 4} + repository, err := telemetrystore.Open(cfg.TelemetryDir()) + if err != nil { + t.Fatalf("open telemetry repository: %v", err) + } + defer repository.Close() + d, err := NewDuck(ctx, cfg, repository) + if err != nil { + t.Fatalf("NewDuck: %v", err) + } + defer d.Close() + + if got := d.DB.Stats().MaxOpenConnections; got != 4 { + t.Fatalf("read pool MaxOpenConnections = %d, want 4", got) + } + if got := d.writeDB.Stats().MaxOpenConnections; got != 1 { + t.Fatalf("write handle MaxOpenConnections = %d, want 1 — writes are serialized by the gate anyway", got) + } + if d.writer() != d.writeDB { + t.Fatal("writes are not going through the write handle") + } + + // Hold a write transaction open and prove the read pool is untouched. + tx, err := d.writer().BeginTx(ctx, nil) + if err != nil { + t.Fatalf("begin write: %v", err) + } + defer func() { _ = tx.Rollback() }() + if _, err := tx.ExecContext(ctx, "CREATE OR REPLACE TABLE stall_probe (n INTEGER)"); err != nil { + t.Fatalf("write inside transaction: %v", err) + } + if inUse := d.DB.Stats().InUse; inUse != 0 { + t.Fatalf("read pool has %d connections in use while a write is open", inUse) + } + if inUse := d.writeDB.Stats().InUse; inUse != 1 { + t.Fatalf("write handle has %d connections in use, want 1", inUse) + } + + // The read pool is the one worth watching, so it is the one the gauges read. + if got := metrics.DuckDBPoolStatsForTest().MaxOpenConnections; got != 4 { + t.Fatalf("pool gauges read a pool with MaxOpenConnections = %d, want the read pool's 4", got) + } +} + +// A stalled query asks one question — how long did I wait to enter the +// snapshot? — and until this metric existed the answer was unavailable, which +// is why the first diagnosis of the stalls named a lock that does not exist. +func TestSnapshotWaitIsMeasuredForReaders(t *testing.T) { + // Label values of this test's own, so the counts are this test's alone + // however the package's tests are ordered. + const admitted, refused = "test-admitted", "test-refused" + d := &Duck{} + before := testutil.CollectAndCount(metrics.ParquetReadWait) + if err := d.lockParquetRead(context.Background(), admitted); err != nil { + t.Fatalf("lockParquetRead: %v", err) + } + d.parquetMu.RUnlock() + admittedCount := testutil.CollectAndCount(metrics.ParquetReadWait) + if admittedCount != before+1 { + t.Fatalf("an admitted reader added %d wait series, want 1", admittedCount-before) + } + + // A reader that gives up is counted separately: a refusal and a long wait + // are different failures. + if err := d.parquetMu.LockContext(context.Background()); err != nil { + t.Fatalf("take the publisher side: %v", err) + } + defer d.parquetMu.Unlock() + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Millisecond) + defer cancel() + if err := d.lockParquetRead(ctx, refused); err == nil { + t.Fatal("reader entered a snapshot held by a publisher") + } + if got := testutil.ToFloat64(metrics.ParquetReadRefusals.WithLabelValues(refused)); got != 1 { + t.Fatalf("refusals = %v, want 1", got) + } + // A refusal is not also counted as a wait: they are different failures. + if got := testutil.CollectAndCount(metrics.ParquetReadWait); got != admittedCount { + t.Fatalf("a refused reader added %d wait series, want 0", got-admittedCount) + } +} + +func TestPublisherQueuedReportsAWaitingSwap(t *testing.T) { + var gate parquetReadGate + if gate.publisherQueued() { + t.Fatal("an idle gate reports a queued publisher") + } + if err := gate.RLockContext(context.Background()); err != nil { + t.Fatalf("RLockContext: %v", err) + } + queued := make(chan struct{}) + go func() { + close(queued) + if err := gate.LockContext(context.Background()); err == nil { + gate.Unlock() + } + }() + <-queued + deadline := time.Now().Add(time.Second) + for !gate.publisherQueued() { + if time.Now().After(deadline) { + t.Fatal("a publisher waiting for readers is not reported as queued") + } + time.Sleep(time.Millisecond) + } + gate.RUnlock() +} From 8b65703462040cea1d9e56cd5900d6ccf082f455 Mon Sep 17 00:00:00 2001 From: Vishal Rana Date: Mon, 7 Sep 2026 10:35:09 -0700 Subject: [PATCH 2/3] fix: one owner for the DuckDB instance, and a yield that waits for the grace MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Review findings on this branch. Two of them were defects in the change itself. The write handle could free the DuckDB instance a second time. database/sql closes a connector that implements io.Closer when its DB is closed, and duckdb's Connector.Close calls duckdb_close behind an unsynchronized bool — so two handles over one connector meant two closes of one instance, which kills the process with a SIGTRAP out of cgo rather than returning an error. The read pool owns the instance; the write handle takes a connector wrapper that cannot close it, and Close runs once however many times it is called. The comment claiming a second connector would fail on a file lock was also wrong: DuckDB keeps an instance cache and would have shared happily. The yield gave up eight sub-windows to relieve congestion that had not happened. A publication queues every ten seconds under compaction, and readers keep being admitted for the whole thirty-second grace, so yielding the moment a publisher appeared cut a pass from eight sub-windows to one exactly when ingest was busiest — an eightfold slower catch-up, plus the fixed cost of the window scan on every resumed pass. The trigger is now the grace running out, which is the moment a query would actually start waiting behind the pass. The test drives the gate's clock rather than a timer, so it pins the ordering instead of hoping for it. Smaller ones from the same review: a refused checkpoint no longer lands in the duration histogram it would drag towards zero, and the comment saying refusals would be common was wrong — only a concurrent write transaction refuses one, and the write gate prevents those. Releasing the pool-gauge source now clears only the source it installed, so a displaced Duck closing cannot blank the gauges of the one still serving. The two Parquet histograms start a hundred times lower, because an uncontended admission costs about two microseconds and every healthy observation was landing in the first bucket. The reader-wait series go through a recorder for the reason the write gate's do — a label that must stay a fixed class should not be set through an exported vector. The pool test now uses the read pool before asserting nothing took a connection from it, so its failure message is about where the write went rather than about a pool nobody had touched, and the metrics test no longer asserts on process-global registry state, so -count works. Claude-Session: https://claude.ai/code/session_017ovmTezqzvxVP8P9MGDQNk --- internal/metrics/metrics.go | 53 ++++++-- internal/query/duck.go | 108 +++++++++++----- internal/query/parquet_gate.go | 29 ++++- internal/query/rollup_yield_test.go | 72 +++++++++-- internal/query/stall_instrumentation_test.go | 124 +++++++++++++++++-- 5 files changed, 319 insertions(+), 67 deletions(-) diff --git a/internal/metrics/metrics.go b/internal/metrics/metrics.go index 512d637cb..bfad175c3 100644 --- a/internal/metrics/metrics.go +++ b/internal/metrics/metrics.go @@ -210,19 +210,24 @@ var ( // reader's side was not, which left the one question a stalled query asks // — "how long did I wait to enter the snapshot?" — unanswerable. These // three complete the pair. + // An uncontended admission costs single-digit microseconds, so the ladder + // starts below that: a histogram whose healthy traffic all lands in the + // first bucket can tell you a stall happened but not what normal was. + snapshotBuckets = []float64{.000001, .000005, .00002, .0001, .0005, .001, .005, .025, .1, .5, 1, 2.5, 5, 10, 30, 60} + ParquetPublishHold = promauto.NewHistogram(prometheus.HistogramOpts{ Name: "fanout_parquet_publish_hold_seconds", Help: "Time readers were excluded while a new Parquet file set was swapped in", - Buckets: []float64{.0001, .0005, .001, .005, .01, .025, .05, .1, .25, .5, 1, 2.5, 5, 10, 30, 60}, + Buckets: snapshotBuckets, }) - ParquetReadWait = promauto.NewHistogramVec(prometheus.HistogramOpts{ + parquetReadWait = promauto.NewHistogramVec(prometheus.HistogramOpts{ Name: "fanout_parquet_read_wait_seconds", Help: "Time a reader spent waiting to enter the Parquet snapshot", - Buckets: []float64{.0001, .0005, .001, .005, .01, .025, .05, .1, .25, .5, 1, 2.5, 5, 10, 30, 60}, + Buckets: snapshotBuckets, }, []string{"reader"}) - ParquetReadRefusals = promauto.NewCounterVec(prometheus.CounterOpts{ + parquetReadRefusals = promauto.NewCounterVec(prometheus.CounterOpts{ Name: "fanout_parquet_read_refusals_total", Help: "Readers that gave up waiting to enter the Parquet snapshot", }, []string{"reader"}) @@ -284,14 +289,18 @@ func RecordFlush(signal string, durationSec float64) { // on the second one. Unset, every gauge reads zero. var duckDBPoolStats atomic.Pointer[func() sql.DBStats] -// SetDuckDBPoolSource points the connection-pool gauges at a pool. The last -// caller wins, which is what a test binary that builds several wants. -func SetDuckDBPoolSource(stats func() sql.DBStats) { +// SetDuckDBPoolSource points the connection-pool gauges at a pool and returns +// the function that stops reading it. The last caller wins; the returned +// release only clears the source if it is still the one it installed, so a +// second Duck closing in a test binary cannot blank the gauges of one that is +// still serving. +func SetDuckDBPoolSource(stats func() sql.DBStats) (release func()) { if stats == nil { - duckDBPoolStats.Store(nil) - return + return func() {} } - duckDBPoolStats.Store(&stats) + installed := &stats + duckDBPoolStats.Store(installed) + return func() { duckDBPoolStats.CompareAndSwap(installed, nil) } } func poolStats() sql.DBStats { @@ -332,6 +341,30 @@ var ( }, func() float64 { return poolStats().WaitDuration.Seconds() }) ) +// RecordParquetRead records one attempt to enter the Parquet snapshot. +// +// Both series are behind this function for the same reason RecordWriteGate +// exists: the label is a fixed reader class, and a vector left exported is a +// vector something eventually calls with a service name in it. +func RecordParquetRead(reader string, waitSec float64, admitted bool) { + if admitted { + parquetReadWait.WithLabelValues(reader).Observe(waitSec) + return + } + parquetReadRefusals.WithLabelValues(reader).Inc() +} + +// ParquetReadWaitForTest and ParquetReadRefusalsForTest hand the two series to +// a test in another package, which asserts on them rather than on the call — +// the call is the easy half. They return collectors rather than the vectors so +// production code still cannot set an arbitrary label, and they keep +// prometheus/testutil out of the release dependency graph. +func ParquetReadWaitForTest() prometheus.Collector { return parquetReadWait } + +func ParquetReadRefusalsForTest(reader string) prometheus.Counter { + return parquetReadRefusals.WithLabelValues(reader) +} + // DuckDBPoolStatsForTest exposes what the pool gauges are reading, so a test // can prove they point at the read pool rather than the write handle. func DuckDBPoolStatsForTest() sql.DBStats { return poolStats() } diff --git a/internal/query/duck.go b/internal/query/duck.go index deff05842..1713228e9 100644 --- a/internal/query/duck.go +++ b/internal/query/duck.go @@ -27,6 +27,15 @@ type Duck struct { // DB is the read pool. Reads scan immutable Parquet concurrently, so it is // sized from the machine. DB *sql.DB + // closeOnce makes Close idempotent. Both handles come from one connector, + // and database/sql closes a connector that implements io.Closer — so a + // second Close would call duckdb_close twice on the same instance, which + // is a double free the runtime cannot recover from. + closeOnce sync.Once + closeErr error + // releasePoolGauges stops the pool gauges reading this Duck's pool, and + // only if they are still reading it. + releasePoolGauges func() // writeDB is the single connection every rollup and maintenance write uses. // // Those writes are serialized by writeGate anyway, so one connection is all @@ -111,6 +120,13 @@ const ( // so the internal lease can be exercised without a five-second test. var parquetStatsWait = 5 * time.Second +// edgeYieldMargin is how much of a queued publication's grace has to be left +// for the edge rollup to take another sub-window. It is a rough estimate of +// what one more sub-window costs: enough that the pass usually finishes before +// readers start queuing, small enough that the eight-sub-window budget is not +// given up for a publication that has only just arrived. +const edgeYieldMargin = 10 * time.Second + // rollupPublicationSafetyLag covers the maximum public SQL hold, publication // grace, bounded commit retries, and queued Parquet encoding with headroom for // a busy disk. Rows stamped at request receipt remain inside the recomputed @@ -245,10 +261,9 @@ func NewDuck(ctx context.Context, cfg config.Config, repository *telemetrystore. if err != nil { return nil, fmt.Errorf("open DuckDB query cache: %w (the cache at %s is rebuildable from Parquet)", err, dbPath) } - // Reads are the pool that can starve, so it is the one worth watching. - metrics.SetDuckDBPoolSource(db.Stats) - d := &Duck{DB: db, writeDB: writeDB, cfg: cfg, repository: repository, rollupLagNanos: int64(rollupPublicationSafetyLag)} + // Reads are the pool that can starve, so it is the one worth watching. + d.releasePoolGauges = metrics.SetDuckDBPoolSource(db.Stats) if cfg.DuckDBMemory == "" { // Only when the operator hasn't pinned storage.duckdb.memory: keep DuckDB's // cgroup-aware auto limit on big boxes but leave absolute RAM headroom on @@ -339,8 +354,12 @@ func (d *Duck) skipRollupToLatest(ctx context.Context) error { } // openDuckDB returns the read pool and the single-connection write handle. -// Both come from one connector: DuckDB allows a file one read-write instance -// per process, so a second connector to the same DSN would fail on its lock. +// +// Both come from one connector so they share one DuckDB instance and therefore +// one catalog: a write committed on one handle is visible to the other, and +// DDL run at boot applies to both. (A second connector to the same file would +// also work — DuckDB keeps an instance cache — but two owners of one instance +// is exactly the hazard the wrapper below exists to avoid.) func openDuckDB(ctx context.Context, dsn, tempDir string, maxConns int) (*sql.DB, *sql.DB, error) { // Every pooled connection must use UTC. DuckDB otherwise inherits the host // timezone and casts TIMESTAMPTZ rollup buckets into local wall-clock @@ -384,23 +403,42 @@ func openDuckDB(ctx context.Context, dsn, tempDir string, maxConns int) (*sql.DB // One connection is all the writers can use — writeGate already lets one // through at a time — and keeping it out of the read pool is the point. - writeDB := sql.OpenDB(connector) + // + // The write handle gets a connector that cannot close the instance. + // database/sql closes a connector that implements io.Closer when its DB is + // closed, and duckdb's Connector.Close calls duckdb_close behind an + // unsynchronized bool — so two handles over one connector means two + // duckdb_close calls on one instance, which crashes the process rather + // than returning an error. The read pool owns the instance; the write + // handle borrows it. + writeDB := sql.OpenDB(borrowedConnector{connector}) writeDB.SetMaxOpenConns(1) writeDB.SetMaxIdleConns(1) return db, writeDB, nil } +// borrowedConnector hands out connections from a connector it does not own. +// It deliberately does not implement io.Closer: that is the whole point. +type borrowedConnector struct{ driver.Connector } + func sqlLiteral(v string) string { return "'" + strings.ReplaceAll(v, "'", "''") + "'" } func (d *Duck) Close() error { - var writeErr error - if d.writeDB != nil { - writeErr = d.writeDB.Close() - } - metrics.SetDuckDBPoolSource(nil) - return errors.Join(writeErr, d.DB.Close()) + d.closeOnce.Do(func() { + if d.releasePoolGauges != nil { + d.releasePoolGauges() + } + var writeErr error + if d.writeDB != nil { + // The write handle holds a connector that does not close the + // instance, so this releases its connection and nothing else. + writeErr = d.writeDB.Close() + } + d.closeErr = errors.Join(writeErr, d.DB.Close()) + }) + return d.closeErr } // writer returns the handle every write goes through. NewDuck always sets one; @@ -415,18 +453,22 @@ func (d *Duck) writer() *sql.DB { // checkpoint folds the rollup cache's WAL back into its file, and says how long // that took. A checkpoint blocks new transactions for its duration, so it is a -// candidate explanation for a stalled query and has to be measurable. The -// non-FORCE form also fails outright while any transaction is open, which on a -// live instance is most of the time — the failure counter is how often that -// happens rather than a guess. +// candidate explanation for a stalled query and has to be measurable. +// +// The non-FORCE form refuses while another *write* transaction is open. Every +// write here is serialized behind the write gate on one connection, and this +// runs under that gate, so a refusal should not happen — the counter is there +// to say so rather than to be assumed. Only a checkpoint that ran is timed: a +// refusal returns in microseconds and would drag the distribution towards zero, +// which is the opposite of what the histogram is for. func (d *Duck) checkpoint(ctx context.Context) error { start := time.Now() - _, err := d.writer().ExecContext(ctx, "CHECKPOINT") - metrics.DuckDBCheckpoint.Observe(time.Since(start).Seconds()) - if err != nil { + if _, err := d.writer().ExecContext(ctx, "CHECKPOINT"); err != nil { metrics.DuckDBCheckpointFailures.Inc() + return err } - return err + metrics.DuckDBCheckpoint.Observe(time.Since(start).Seconds()) + return nil } // DefaultNamespace returns empty string so queries search all namespaces. @@ -1015,13 +1057,21 @@ WHERE ingested_unix_nano > ? processed := 0 for !subLo.After(maxT) { // Stop at a resumable point when the budget is spent, and also when - // a publication is waiting: this pass holds the Parquet snapshot - // until it commits, and every query that arrives after the - // publisher's grace expires waits out whatever is left of it. One - // sub-window is the unit; the cursor below is what makes stopping - // free. Never before the first, so a pass always makes progress and - // a busy compaction cycle cannot starve the rollup. - if processed == maxEdgeSubWindowsPerPass || (processed > 0 && d.parquetMu.publisherQueued()) { + // a queued publication is about to start refusing readers: this + // pass holds the Parquet snapshot until it commits, so from that + // moment every arriving query waits out whatever is left of it. + // + // The trigger is the grace running out, not a publisher existing. + // A publication queues every ten seconds under compaction, and + // yielding on sight would cut a pass from eight sub-windows to one + // whenever ingest is busy — which is when the eight are needed. + // While the grace holds, readers are still being admitted and + // nothing is waiting on this pass. + // + // One sub-window is the unit, and the cursor below is what makes + // stopping free. Never before the first, so a pass always makes + // progress and a busy compaction cycle cannot starve the rollup. + if processed == maxEdgeSubWindowsPerPass || (processed > 0 && d.parquetMu.publisherGraceLeft() <= edgeYieldMargin) { completed = false nextCursor = subLo.UnixNano() break @@ -1756,10 +1806,10 @@ const ( func (d *Duck) lockParquetRead(ctx context.Context, reader string) error { start := time.Now() if err := d.parquetMu.RLockContext(ctx); err != nil { - metrics.ParquetReadRefusals.WithLabelValues(reader).Inc() + metrics.RecordParquetRead(reader, time.Since(start).Seconds(), false) return errors.Join(ErrParquetReadWait, err) } - metrics.ParquetReadWait.WithLabelValues(reader).Observe(time.Since(start).Seconds()) + metrics.RecordParquetRead(reader, time.Since(start).Seconds(), true) return nil } diff --git a/internal/query/parquet_gate.go b/internal/query/parquet_gate.go index 0028ab248..2ec1cb7c0 100644 --- a/internal/query/parquet_gate.go +++ b/internal/query/parquet_gate.go @@ -80,15 +80,32 @@ func (g *parquetReadGate) admitsReaderLocked() bool { return g.clock().Sub(g.waiting[0].queuedAt) < g.grace() } -// publisherQueued reports whether a publication is waiting for readers to -// leave. A long-running internal reader asks between units of work so it can -// stop at a resumable point instead of making every query behind it wait out -// the rest of its pass. -func (g *parquetReadGate) publisherQueued() bool { +// publisherGraceLeft reports how long the oldest queued publication may still +// be kept waiting before the gate starts refusing new readers. +// +// A long-running internal reader asks between units of work, so it can stop at +// a resumable point rather than make every query behind it wait out the rest +// of its pass. It is the remaining grace and not the mere presence of a +// publisher that matters: while the grace holds, readers are still being +// admitted and nothing is blocked, so yielding then would give up throughput +// to relieve congestion that does not exist yet. +// +// Zero means a publication is holding the gate, or has waited out its grace +// and new readers are already queuing. With nothing queued, the full grace. +func (g *parquetReadGate) publisherGraceLeft() time.Duration { g.init() g.mu.Lock() defer g.mu.Unlock() - return g.writer || len(g.waiting) > 0 + if g.writer { + return 0 + } + if len(g.waiting) == 0 { + return g.grace() + } + if left := g.grace() - g.clock().Sub(g.waiting[0].queuedAt); left > 0 { + return left + } + return 0 } func (g *parquetReadGate) TryRLock() bool { diff --git a/internal/query/rollup_yield_test.go b/internal/query/rollup_yield_test.go index 1742cd21f..f85530783 100644 --- a/internal/query/rollup_yield_test.go +++ b/internal/query/rollup_yield_test.go @@ -3,6 +3,7 @@ package query import ( "context" "fmt" + "sync/atomic" "testing" "time" @@ -62,11 +63,11 @@ func newSpreadDuck(t *testing.T, rows, spreadMins int) *Duck { } // A rollup pass holds the Parquet snapshot until it commits, so a publication -// that queues behind it makes every query arriving after the grace period wait -// out the rest of that pass — up to eight sub-windows of work for a request -// that needed none of it. The pass now stops at the first sub-window boundary -// once a publisher is waiting, which is free: the resume cursor already exists, -// and the ingested watermark already stays behind an unfinished window. +// whose grace has run out makes every arriving query wait out the rest of that +// pass — up to eight sub-windows of work for a request that needed none of it. +// The pass now stops at the first sub-window boundary once that happens, which +// costs nothing to arrange: the resume cursor already exists for the per-pass +// budget, and the ingested watermark already stays behind an unfinished window. func TestEdgeRollupYieldsToAQueuedPublisher(t *testing.T) { const ( rows = 6_000 @@ -85,13 +86,16 @@ func TestEdgeRollupYieldsToAQueuedPublisher(t *testing.T) { yielding := newSpreadDuck(t, rows, spreadMins) // A publisher can only queue while something is reading, so the test holds - // a reader of its own. The publisher then stays queued for the whole pass, - // which is what the rollup checks at each sub-window boundary. The grace - // period has not expired, so the pass is still admitted. + // a reader of its own. The gate's clock is the test's: the pass is admitted + // with the grace intact, and the clock jumps past it once the pass is + // under way — which is the state the rollup yields for, and the only one + // in which a query is actually blocked behind it. + spend := spendGraceOnCue(&yielding.parquetMu) if err := yielding.parquetMu.RLockContext(ctx); err != nil { t.Fatalf("hold a reader: %v", err) } queued := queuePublisher(t, yielding) + spendOnceAdmitted(&yielding.parquetMu, spend) if _, err := yielding.refreshEdgeRollup(ctx); err != nil { t.Fatalf("yielding pass: %v", err) @@ -133,10 +137,14 @@ func TestEdgeRollupYieldsToAQueuedPublisher(t *testing.T) { func TestEdgeRollupAlwaysCompletesOneSubWindow(t *testing.T) { ctx := context.Background() d := newSpreadDuck(t, 4_000, 180) + spend := spendGraceOnCue(&d.parquetMu) if err := d.parquetMu.RLockContext(ctx); err != nil { t.Fatalf("hold a reader: %v", err) } queued := queuePublisher(t, d) + // Overdue from the pass's very first boundary: it must still complete one + // sub-window, or a busy compaction cycle starves the cache. + spendOnceAdmitted(&d.parquetMu, spend) rows, err := d.refreshEdgeRollup(ctx) if err != nil { @@ -166,7 +174,7 @@ func queuePublisher(t *testing.T, d *Duck) (handle struct{ done func() }) { d.parquetMu.Unlock() }() deadline := time.Now().Add(2 * time.Second) - for !d.parquetMu.publisherQueued() { + for queuedPublishers(&d.parquetMu) == 0 { if time.Now().After(deadline) { t.Fatal("publisher never queued") } @@ -178,3 +186,49 @@ func queuePublisher(t *testing.T, d *Duck) (handle struct{ done func() }) { } return handle } + +// spendGraceOnCue replaces the gate's clock with one the test moves. Until the +// returned function is called the clock stands still, so a queued publication +// keeps its whole grace and readers are admitted; afterwards the clock is an +// hour later, so the grace is spent and every new reader would queue. +func spendGraceOnCue(gate *parquetReadGate) (spend func()) { + base := time.Now() + var spent atomic.Bool + gate.now = func() time.Time { + if spent.Load() { + return base.Add(time.Hour) + } + return base + } + return func() { spent.Store(true) } +} + +// queuedPublishers reads the gate's wait list. The grace-based predicate the +// rollup uses cannot serve here: a publication that has only just queued still +// has its whole grace, which is exactly the state this waits to observe. +func queuedPublishers(gate *parquetReadGate) int { + gate.init() + gate.mu.Lock() + defer gate.mu.Unlock() + return len(gate.waiting) +} + +// spendOnceAdmitted spends the queued publication's grace the moment the rollup +// has entered the snapshot, so the pass is admitted with the grace intact and +// finds it gone at its first sub-window boundary. Waiting on the reader count +// rather than on a duration keeps that ordering exact. +func spendOnceAdmitted(gate *parquetReadGate, spend func()) { + go func() { + for readers(gate) < 2 { + time.Sleep(time.Millisecond) + } + spend() + }() +} + +func readers(gate *parquetReadGate) int { + gate.init() + gate.mu.Lock() + defer gate.mu.Unlock() + return gate.readers +} diff --git a/internal/query/stall_instrumentation_test.go b/internal/query/stall_instrumentation_test.go index 8a222e2df..0a984182f 100644 --- a/internal/query/stall_instrumentation_test.go +++ b/internal/query/stall_instrumentation_test.go @@ -2,6 +2,8 @@ package query import ( "context" + "fmt" + "sync" "testing" "time" @@ -41,6 +43,16 @@ func TestWritesDoNotDrawFromTheReadPool(t *testing.T) { t.Fatal("writes are not going through the write handle") } + // Use the read pool first, so its idle count is non-zero and the assertion + // below is about where the write went rather than about a pool nobody has + // touched. + if err := d.DB.QueryRowContext(ctx, "SELECT 1").Scan(new(int)); err != nil { + t.Fatalf("read: %v", err) + } + if idle := d.DB.Stats().Idle; idle == 0 { + t.Fatal("the read pool holds no connection after a read; the assertion below would prove nothing") + } + // Hold a write transaction open and prove the read pool is untouched. tx, err := d.writer().BeginTx(ctx, nil) if err != nil { @@ -69,14 +81,15 @@ func TestWritesDoNotDrawFromTheReadPool(t *testing.T) { func TestSnapshotWaitIsMeasuredForReaders(t *testing.T) { // Label values of this test's own, so the counts are this test's alone // however the package's tests are ordered. - const admitted, refused = "test-admitted", "test-refused" + admitted := fmt.Sprintf("test-admitted-%d", time.Now().UnixNano()) + refused := fmt.Sprintf("test-refused-%d", time.Now().UnixNano()) d := &Duck{} - before := testutil.CollectAndCount(metrics.ParquetReadWait) + before := testutil.CollectAndCount(metrics.ParquetReadWaitForTest()) if err := d.lockParquetRead(context.Background(), admitted); err != nil { t.Fatalf("lockParquetRead: %v", err) } d.parquetMu.RUnlock() - admittedCount := testutil.CollectAndCount(metrics.ParquetReadWait) + admittedCount := testutil.CollectAndCount(metrics.ParquetReadWaitForTest()) if admittedCount != before+1 { t.Fatalf("an admitted reader added %d wait series, want 1", admittedCount-before) } @@ -92,19 +105,19 @@ func TestSnapshotWaitIsMeasuredForReaders(t *testing.T) { if err := d.lockParquetRead(ctx, refused); err == nil { t.Fatal("reader entered a snapshot held by a publisher") } - if got := testutil.ToFloat64(metrics.ParquetReadRefusals.WithLabelValues(refused)); got != 1 { - t.Fatalf("refusals = %v, want 1", got) + if refusals := testutil.ToFloat64(metrics.ParquetReadRefusalsForTest(refused)); refusals != 1 { + t.Fatalf("refusals = %v, want 1", refusals) } // A refusal is not also counted as a wait: they are different failures. - if got := testutil.CollectAndCount(metrics.ParquetReadWait); got != admittedCount { - t.Fatalf("a refused reader added %d wait series, want 0", got-admittedCount) + if series := testutil.CollectAndCount(metrics.ParquetReadWaitForTest()); series != admittedCount { + t.Fatalf("a refused reader added %d wait series, want 0", series-admittedCount) } } -func TestPublisherQueuedReportsAWaitingSwap(t *testing.T) { - var gate parquetReadGate - if gate.publisherQueued() { - t.Fatal("an idle gate reports a queued publisher") +func TestPublisherGraceLeftTracksAWaitingSwap(t *testing.T) { + gate := parquetReadGate{writerGrace: time.Second} + if left := gate.publisherGraceLeft(); left != time.Second { + t.Fatalf("an idle gate reports %v of grace left, want the whole %v", left, time.Second) } if err := gate.RLockContext(context.Background()); err != nil { t.Fatalf("RLockContext: %v", err) @@ -118,11 +131,96 @@ func TestPublisherQueuedReportsAWaitingSwap(t *testing.T) { }() <-queued deadline := time.Now().Add(time.Second) - for !gate.publisherQueued() { + for gate.publisherGraceLeft() == time.Second { if time.Now().After(deadline) { - t.Fatal("a publisher waiting for readers is not reported as queued") + t.Fatal("a publisher waiting for readers did not start consuming its grace") } time.Sleep(time.Millisecond) } + // The grace runs down rather than vanishing: a reader admitted now is not + // blocking anything yet, which is what stops the rollup yielding on sight. + if left := gate.publisherGraceLeft(); left <= 0 || left > time.Second { + t.Fatalf("grace left = %v, want a positive remainder under %v", left, time.Second) + } + deadline = time.Now().Add(3 * time.Second) + for gate.publisherGraceLeft() > 0 { + if time.Now().After(deadline) { + t.Fatal("grace never ran out") + } + time.Sleep(5 * time.Millisecond) + } gate.RUnlock() } + +// database/sql closes a connector that implements io.Closer when its DB is +// closed, and duckdb's connector closes the instance behind an unsynchronized +// bool. Two handles over one connector therefore meant two duckdb_close calls +// on one instance — a double free that kills the process rather than returning +// an error. The write handle borrows the instance; the read pool owns it. +func TestClosingTheWriteHandleLeavesTheInstanceAlive(t *testing.T) { + ctx := context.Background() + cfg := config.Config{DataDir: t.TempDir(), RollupInterval: time.Minute, DuckDBMemory: "128MB"} + repository, err := telemetrystore.Open(cfg.TelemetryDir()) + if err != nil { + t.Fatalf("open telemetry repository: %v", err) + } + defer repository.Close() + d, err := NewDuck(ctx, cfg, repository) + if err != nil { + t.Fatalf("NewDuck: %v", err) + } + + if err := d.writeDB.Close(); err != nil { + t.Fatalf("close write handle: %v", err) + } + var one int + if err := d.DB.QueryRowContext(ctx, "SELECT 1").Scan(&one); err != nil { + t.Fatalf("read pool died with the write handle: %v", err) + } + + // Close is idempotent, including under a second shutdown path arriving at + // the same time. + var wg sync.WaitGroup + for range 8 { + wg.Add(1) + go func() { + defer wg.Done() + _ = d.Close() + }() + } + wg.Wait() + if err := d.Close(); err != nil { + t.Fatalf("closing an already-closed Duck: %v", err) + } +} + +// Installing a gauge source is last-caller-wins; removing one is not. A Duck +// that closes after another has taken over must clear nothing, or the gauges +// go blank while a pool is still serving. +func TestClosingADisplacedDuckLeavesTheGaugeSourceAlone(t *testing.T) { + ctx := context.Background() + open := func(conns int) *Duck { + t.Helper() + cfg := config.Config{DataDir: t.TempDir(), RollupInterval: time.Minute, DuckDBMemory: "128MB", DuckDBMaxConns: conns} + repository, err := telemetrystore.Open(cfg.TelemetryDir()) + if err != nil { + t.Fatalf("open telemetry repository: %v", err) + } + t.Cleanup(func() { _ = repository.Close() }) + d, err := NewDuck(ctx, cfg, repository) + if err != nil { + t.Fatalf("NewDuck: %v", err) + } + return d + } + first := open(2) + second := open(3) + defer second.Close() + // second installed itself over first. first closing must leave it alone. + if err := first.Close(); err != nil { + t.Fatalf("close first: %v", err) + } + if got := metrics.DuckDBPoolStatsForTest().MaxOpenConnections; got != 3 { + t.Fatalf("pool gauges read MaxOpenConnections = %d after a displaced Duck closed, want the live pool's 3", got) + } +} From 403ff694e9fa8a69412f36b025ebc4e61c7d214f Mon Sep 17 00:00:00 2001 From: Vishal Rana Date: Mon, 7 Sep 2026 22:29:36 -0700 Subject: [PATCH 3/3] perf: bound the two statements whose cost grew with the data MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Measurement said the stall was pool contention, so the first instinct was to make the pool bigger. That was wrong: the requests waiting for a connection were waiting behind two statements doing far more work than they needed. Log redaction ran on every row in the window. Four chained regexp_replace sat in the same SELECT as ORDER BY time DESC LIMIT, and a projection there is evaluated below the top-N — so returning a hundred entries redacted every row the window held. Measured at 140ms for 100 rows out of 372,000 against 8ms once the top-N runs first, and independently at 449ms against 11ms over 400,000 rows with more realistic bodies — the plan confirming the projection moved above the top-N and EXCEPT ALL confirming the rows are identical either way. It was the one statement whose cost grew with retention rather than with the answer — with no search term; a search still has to redact every row it tests, because it must match the text the reader is shown rather than the secret underneath it. The endpoint boundary scan read the whole window. Its bounds came from a CTE joined against the scan, and a bound derived from a joined row is not a constant DuckDB can push into a Parquet scan — so a query that wants the two partial minutes at the window's edges read everything between them: 799,333 rows of 800,000 to return 7,333, and 240,000 to return 667 in a smaller independent reproduction. The bounds are computed in Go and bound as parameters now; the same queries scan 11,333 and 35,200. Dashboard queries get a twenty-second deadline. The SQL API has always had one; these endpoints, which every widget calls, had none, so a pathological query held its connection until the client gave up. A query that exceeds it answers 503: running out of time under load is not an internal error, and reporting it as one logs at ERROR and counts towards the 5xx rate that pages someone. And the half of the picture the last commit lacked: how long a statement takes to execute. The connection is acquired before the timer starts, because DB.QueryContext would fold the wait for a free connection into the statement's duration — and telling queueing from execution is the entire reason to measure it. A test holds the pool's only connection to prove the two do not overlap. Measured across the same seed and load — 96 batches of 4,000 spans over ten hours, continuous ingest, eight concurrent clients, 150s, four-connection pool in both arms: req p50 p95 p99 cheap p99 before 11,970 45ms 275ms 498ms 178ms after 15,642 31ms 191ms 358ms 122ms Throughput up 31%, p95 down 31%. The max column is deliberately absent: with a closed-loop generator one server-side stall appears as eight slow requests, so a single run cannot separate a shifted distribution from one bad event. No pool-sizing change. An earlier version of this commit rewrote resolveDuckDBMaxConns, and the arm that appeared to justify it compared a hand-pinned pool of four against an auto-sized seven — which is not what the rule would have done on any host: unchanged on the four-core reference host, and a reduction on larger ones that nothing here measured. The evidence says a cheap query should not queue behind an expensive one; it does not say what the pool should be. The statement histogram and the pool gauges are what should answer that, on a real host. Claude-Session: https://claude.ai/code/session_017ovmTezqzvxVP8P9MGDQNk --- internal/api/observability.go | 49 ++++++++-- internal/api/observability_test.go | 97 ++++++++++++++++++++ internal/metrics/metrics.go | 24 ++++- internal/observability/logs.go | 26 +++++- internal/observability/performance.go | 84 ++++++++++------- internal/observability/performance_test.go | 65 +++++++++++++ internal/observability/service_test.go | 9 +- internal/query/duck.go | 57 ++++++++++-- internal/query/stall_instrumentation_test.go | 71 ++++++++++++++ 9 files changed, 429 insertions(+), 53 deletions(-) create mode 100644 internal/observability/performance_test.go diff --git a/internal/api/observability.go b/internal/api/observability.go index ddd6b2447..fd8bda4b9 100644 --- a/internal/api/observability.go +++ b/internal/api/observability.go @@ -29,15 +29,42 @@ func NewObservabilityHandler(queries ObservabilityQueries) *ObservabilityHandler return &ObservabilityHandler{queries: queries, now: time.Now} } -// Register mounts the deterministic product API. These endpoints are for -// dashboards and drill-downs; the browser does not need an agent for ordinary -// telemetry navigation. +// observabilityDeadline bounds one dashboard query. +// +// The SQL API has always been bounded; these endpoints, which every widget +// calls, had no deadline of their own — a query that went pathological held a +// connection until the client gave up, and the client's own timeout is not +// something the server can rely on. +// +// Twenty seconds is well clear of the slowest query measured under load here, +// about two seconds, and stricter than the SQL API's own ceiling. It is +// deliberately shorter than the worst legitimate wait behind a stuck +// publication — the drain remainder plus the swap budget can exceed a minute — +// so a publication that goes wrong answers the dashboard instead of holding +// every widget's connection until it finishes. +const observabilityDeadline = 20 * time.Second + +// Register mounts the deterministic product API, each endpoint under a +// deadline. These endpoints are for dashboards and drill-downs; the browser +// does not need an agent for ordinary telemetry navigation. func (h *ObservabilityHandler) Register(group *echo.Group) { - group.GET("/overview", h.overview) - group.GET("/topology", h.topology) - group.GET("/performance", h.performance) - group.GET("/trace", h.trace) - group.GET("/logs", h.logs) + group.GET("/overview", h.bounded(h.overview)) + group.GET("/topology", h.bounded(h.topology)) + group.GET("/performance", h.bounded(h.performance)) + group.GET("/trace", h.bounded(h.trace)) + group.GET("/logs", h.bounded(h.logs)) +} + +// bounded gives a handler a deadline. The context reaches DuckDB through +// QueryContext, so an expired one stops the statement rather than only the +// reply — and releases the pool connection it was holding. +func (h *ObservabilityHandler) bounded(handler func(*echo.Context) error) func(*echo.Context) error { + return func(c *echo.Context) error { + ctx, cancel := context.WithTimeout(c.Request().Context(), observabilityDeadline) + defer cancel() + c.SetRequest(c.Request().WithContext(ctx)) + return handler(c) + } } func (h *ObservabilityHandler) performance(c *echo.Context) error { @@ -129,5 +156,11 @@ func mapQueryError(err error) error { if errors.Is(err, observability.ErrInvalidScope) || errors.Is(err, observability.ErrInvalidLimit) { return echo.NewHTTPError(http.StatusBadRequest, err.Error()) } + // A query that ran out of time is a load condition, not a fault: reporting + // it as a server error logs it at ERROR and counts it towards the 5xx rate + // that pages someone. + if errors.Is(err, context.DeadlineExceeded) { + return echo.NewHTTPError(http.StatusServiceUnavailable, "telemetry query took too long").Wrap(err) + } return echo.NewHTTPError(http.StatusInternalServerError, "telemetry query failed").Wrap(err) } diff --git a/internal/api/observability_test.go b/internal/api/observability_test.go index 0c2b8cf69..bebfe4514 100644 --- a/internal/api/observability_test.go +++ b/internal/api/observability_test.go @@ -3,6 +3,8 @@ package api import ( "context" "encoding/json" + "errors" + "fmt" "net/http" "net/http/httptest" "testing" @@ -91,3 +93,98 @@ func TestRouteRejectsInvalidWindow(t *testing.T) { t.Fatalf("status = %d, want 400", rec.Code) } } + +// A dashboard query that runs out of time is a load condition. Reporting it as +// a server error logs it at ERROR and counts towards the 5xx rate that pages +// someone; the deadline itself exists so a pathological query stops holding a +// connection once the answer can no longer be useful. +func TestObservabilityQueryDeadlineIsNotAServerError(t *testing.T) { + err := mapQueryError(fmt.Errorf("query endpoints: %w", context.DeadlineExceeded)) + var httpErr *echo.HTTPError + if !errors.As(err, &httpErr) { + t.Fatalf("mapQueryError returned %T, want an *echo.HTTPError", err) + } + if httpErr.Code != http.StatusServiceUnavailable { + t.Fatalf("a timed-out query maps to %d, want %d", httpErr.Code, http.StatusServiceUnavailable) + } + // A genuine failure still reads as one. + var failure *echo.HTTPError + if !errors.As(mapQueryError(errors.New("boom")), &failure) || failure.Code != http.StatusInternalServerError { + t.Fatal("a real query failure no longer maps to 500") + } +} + +// Every registered route carries the deadline, not just the wrapper in +// isolation: dropping bounded() from Register is the mistake a refactor makes, +// and it left the whole suite green. +func TestRegisteredObservabilityRoutesCarryADeadline(t *testing.T) { + deadlines := map[string]bool{} + handler := &ObservabilityHandler{queries: deadlineProbe{seen: deadlines}, now: time.Now} + e := echo.New() + handler.Register(e.Group("/api/observability")) + for _, path := range []string{"/api/observability/overview", "/api/observability/topology", "/api/observability/performance", "/api/observability/logs", "/api/observability/trace"} { + recorder := httptest.NewRecorder() + e.ServeHTTP(recorder, httptest.NewRequest(http.MethodGet, path+"?window=1h", nil)) + if !deadlines[path] { + t.Errorf("%s reached the query layer with no deadline", path) + } + } +} + +// deadlineProbe records, per route, whether the context that reached it had a +// deadline. It answers every query with an empty result. +type deadlineProbe struct{ seen map[string]bool } + +func (p deadlineProbe) note(ctx context.Context, path string) { + if _, ok := ctx.Deadline(); ok { + p.seen[path] = true + } +} + +func (p deadlineProbe) Overview(ctx context.Context, scope observability.Scope, limit int) (observability.Result[observability.Overview], error) { + p.note(ctx, "/api/observability/overview") + return observability.Result[observability.Overview]{}, nil +} + +func (p deadlineProbe) Topology(ctx context.Context, scope observability.Scope, limit int) (observability.Result[observability.Topology], error) { + p.note(ctx, "/api/observability/topology") + return observability.Result[observability.Topology]{}, nil +} + +func (p deadlineProbe) Performance(ctx context.Context, scope observability.Scope, service string, limit int) (observability.Result[observability.Performance], error) { + p.note(ctx, "/api/observability/performance") + return observability.Result[observability.Performance]{}, nil +} + +func (p deadlineProbe) Logs(ctx context.Context, scope observability.Scope, service, severity, search string, limit int) (observability.Result[observability.Logs], error) { + p.note(ctx, "/api/observability/logs") + return observability.Result[observability.Logs]{}, nil +} + +func (p deadlineProbe) Trace(ctx context.Context, scope observability.Scope, traceID, service string, limit int) (observability.Result[observability.TraceDetail], error) { + p.note(ctx, "/api/observability/trace") + return observability.Result[observability.TraceDetail]{}, nil +} + +// The handlers carry a deadline of their own, because the client's timeout is +// not something the server can rely on. +func TestObservabilityHandlersCarryADeadline(t *testing.T) { + var seen time.Duration + handler := (&ObservabilityHandler{}).bounded(func(c *echo.Context) error { + deadline, ok := c.Request().Context().Deadline() + if !ok { + t.Fatal("handler ran with no deadline") + } + seen = time.Until(deadline) + return nil + }) + e := echo.New() + request := httptest.NewRequest(http.MethodGet, "/api/observability/overview", nil) + c := e.NewContext(request, httptest.NewRecorder()) + if err := handler(c); err != nil { + t.Fatalf("bounded handler: %v", err) + } + if seen <= 0 || seen > observabilityDeadline { + t.Fatalf("deadline left %s, want a positive value no greater than %s", seen, observabilityDeadline) + } +} diff --git a/internal/metrics/metrics.go b/internal/metrics/metrics.go index bfad175c3..a855c9f28 100644 --- a/internal/metrics/metrics.go +++ b/internal/metrics/metrics.go @@ -7,6 +7,7 @@ import ( "github.com/prometheus/client_golang/prometheus" "github.com/prometheus/client_golang/prometheus/promauto" + dto "github.com/prometheus/client_model/go" ) type RollupComponent string @@ -232,8 +233,19 @@ var ( Help: "Readers that gave up waiting to enter the Parquet snapshot", }, []string{"reader"}) + // The wait to enter the snapshot and the wait for a connection are both + // measured; how long the statement itself takes was not, so a slow request + // could be attributed to queueing or to execution only by elimination. + // The connection is acquired before this timer starts, so the two do not + // overlap. + DuckDBStatement = promauto.NewHistogram(prometheus.HistogramOpts{ + Name: "fanout_duckdb_statement_seconds", + Help: "Time a read statement spent executing in DuckDB, excluding any wait to start it", + Buckets: []float64{.0005, .001, .005, .01, .025, .05, .1, .25, .5, 1, 2.5, 5, 10, 30}, + }) + // An explicit CHECKPOINT blocks new transactions for its duration, and the - // non-FORCE form fails outright while any transaction is open — so both + // non-FORCE form refuses while another write transaction is open — so both // how long it takes and how often it does not happen are worth knowing. DuckDBCheckpoint = promauto.NewHistogram(prometheus.HistogramOpts{ Name: "fanout_duckdb_checkpoint_seconds", @@ -365,6 +377,16 @@ func ParquetReadRefusalsForTest(reader string) prometheus.Counter { return parquetReadRefusals.WithLabelValues(reader) } +// DuckDBStatementSecondsForTest reports the statement histogram's running +// total, so a test can prove what the timer does and does not cover. +func DuckDBStatementSecondsForTest() float64 { + metric := &dto.Metric{} + if err := DuckDBStatement.Write(metric); err != nil { + return 0 + } + return metric.GetHistogram().GetSampleSum() +} + // DuckDBPoolStatsForTest exposes what the pool gauges are reading, so a test // can prove they point at the read pool rather than the write handle. func DuckDBPoolStatsForTest() sql.DBStats { return poolStats() } diff --git a/internal/observability/logs.go b/internal/observability/logs.go index 963788f56..9fad8200d 100644 --- a/internal/observability/logs.go +++ b/internal/observability/logs.go @@ -17,12 +17,28 @@ WHERE time >= ? AND time < ? // DuckDB answers the two questions the API asks: the newest `limit` entries // and per-bucket counts. LIMIT bounds the entry stream and GROUP BY bounds the // histogram stream regardless of the retained Parquet row count. +// +// Redaction runs outside the LIMIT, and that placement is the whole cost of +// this query. A projection in the same SELECT as ORDER BY ... LIMIT is +// evaluated below the top-N operator, so four chained regexp_replace ran over +// every row in the window to return a hundred: measured at 140ms for a page of +// 100 out of 372,000 rows, and 347ms with four DuckDB threads, against 8ms +// once the top-N runs first. It was the one statement here whose cost grew +// with retained rows rather than with the answer. +// +// The search predicate stays inside, on purpose: a search has to match what +// the reader will be shown, so it matches the redacted text. It is skipped +// entirely when no search term is given. var logEntriesQuery = ` -SELECT time, severity, coalesce(service, ''), ` + redactLogBodySQL("body") + `, - coalesce(trace_id, ''), coalesce(span_id, '') - FROM logs` + logFilters + ` -ORDER BY time DESC -LIMIT ?` +SELECT time, severity, service, ` + redactLogBodySQL("body") + `, trace_id, span_id +FROM ( + SELECT time, severity, coalesce(service, '') AS service, body, + coalesce(trace_id, '') AS trace_id, coalesce(span_id, '') AS span_id + FROM logs` + logFilters + ` + ORDER BY time DESC + LIMIT ? +) +ORDER BY time DESC` var logBucketsQueryTemplate = ` SELECT time_bucket(INTERVAL '%s', time) AS point_time, diff --git a/internal/observability/performance.go b/internal/observability/performance.go index e6d639e17..f0378304a 100644 --- a/internal/observability/performance.go +++ b/internal/observability/performance.go @@ -51,35 +51,50 @@ const endpointRollupMatureQuery = ` SELECT COUNT(*) >= 5 FROM (SELECT DISTINCT bucket FROM endpoint_rollup LIMIT 5)` +// endpointInteriorBounds returns the half-open minute range the rollup cache +// answers for, given the requested window and the cache's watermark. +// +// The interior starts at the first whole minute inside the window — a window +// that begins mid-minute has a partial minute the cache cannot answer — and +// ends at the earlier of the window's last whole minute and the watermark's, +// because the cache knows nothing past what it has rolled up. Raw spans cover +// everything outside that range. +func endpointInteriorBounds(start, end, watermark time.Time) (interiorStart, interiorEnd time.Time) { + interiorStart = start.Truncate(time.Minute) + if !interiorStart.Equal(start) { + interiorStart = interiorStart.Add(time.Minute) + } + interiorEnd = end.Truncate(time.Minute) + if rolled := watermark.Truncate(time.Minute); rolled.Before(interiorEnd) { + interiorEnd = rolled + } + return interiorStart, interiorEnd +} + // endpointRollupQuery uses the minute cache only through its current watermark. // Raw spans cover both partial boundary minutes and any newer complete minutes, -// so the cache lag cannot appear as zero traffic and raw work stays bounded. +// so the cache lag cannot appear as zero traffic. +// +// The interior bounds are computed in Go and bound as parameters rather than +// derived in a CTE. They read the same either way, but a bound derived from a +// joined row is not a constant DuckDB can push into the Parquet scan: the +// boundary exclusion below was applied after reading the window, so a query +// that wants the two partial minutes at its edges read every row between them — +// measured at 799,333 rows scanned out of 800,000 to return 7,333. Bound +// directly, the same filter prunes to 11,333 rows read. +// +// The raw half is bounded by the cache being current, not by this query. When +// the watermark sits behind the window the interior is empty, the boundary +// predicate admits everything, and the scan reads the whole window — which is +// correct, and is the case worth watching after an outage or a first run. var endpointRollupQuery = ` -WITH params AS ( - SELECT - ?::TIMESTAMP AS start_time, - ?::TIMESTAMP AS end_time, - ?::TIMESTAMP AS rollup_end, - ?::VARCHAR AS namespace, - ?::VARCHAR AS service -), -bounds AS ( - SELECT - *, - CASE - WHEN start_time = date_trunc('minute', start_time) THEN start_time - ELSE date_trunc('minute', start_time) + INTERVAL 1 MINUTE - END AS interior_start, - LEAST(date_trunc('minute', end_time), date_trunc('minute', rollup_end)) AS interior_end - FROM params -), -rollup_source AS ( +WITH rollup_source AS ( SELECT e.method, e.path, e.calls, e.error_count, e.duration_count, e.duration_buckets - FROM endpoint_rollup e, bounds b - WHERE e.bucket >= b.interior_start - AND e.bucket < b.interior_end - AND (b.namespace = '' OR e.namespace = b.namespace) - AND (b.service = '' OR e.service = b.service) + FROM endpoint_rollup e + WHERE e.bucket >= ? + AND e.bucket < ? + AND (? = '' OR e.namespace = ?) + AND (? = '' OR e.service = ?) ), boundary_source AS ( SELECT @@ -107,12 +122,12 @@ boundary_source AS ( le_30000 := COUNT(*) FILTER (WHERE s.duration_ms <= 30000), le_300000 := COUNT(*) FILTER (WHERE s.duration_ms <= 300000) ) AS duration_buckets - FROM spans s, bounds b - WHERE s.start_time >= b.start_time - AND s.start_time < b.end_time - AND (s.start_time < b.interior_start OR s.start_time >= b.interior_end) - AND (b.namespace = '' OR s.namespace = b.namespace) - AND (b.service = '' OR COALESCE(s.service, '') = b.service) + FROM spans s + WHERE s.start_time >= ? + AND s.start_time < ? + AND (s.start_time < ? OR s.start_time >= ?) + AND (? = '' OR s.namespace = ?) + AND (? = '' OR COALESCE(s.service, '') = ?) GROUP BY method, path ), sources AS ( @@ -286,7 +301,12 @@ func (s *Service) queryEndpoints(ctx context.Context, scope Scope, service strin source := "spans" if ready { query = endpointRollupQuery - args = []any{scope.Start, scope.End, watermark, scope.Namespace, service, limit} + interiorStart, interiorEnd := endpointInteriorBounds(scope.Start, scope.End, watermark) + args = []any{ + interiorStart, interiorEnd, scope.Namespace, scope.Namespace, service, service, + scope.Start, scope.End, interiorStart, interiorEnd, scope.Namespace, scope.Namespace, service, service, + limit, + } source = "endpoint_rollup + raw spans (interpolated histogram percentiles)" } rows, err := s.db.QueryContext(ctx, query, args...) diff --git a/internal/observability/performance_test.go b/internal/observability/performance_test.go new file mode 100644 index 000000000..ea3e84f0e --- /dev/null +++ b/internal/observability/performance_test.go @@ -0,0 +1,65 @@ +package observability + +import ( + "strings" + "testing" + "time" +) + +// The interior range is what the rollup cache can answer for. A window that +// starts mid-minute has a partial minute the cache does not hold, and the cache +// knows nothing past its own watermark — raw spans cover both edges. +func TestEndpointInteriorBounds(t *testing.T) { + minute := func(s string) time.Time { + at, err := time.Parse(time.RFC3339Nano, s) + if err != nil { + t.Fatal(err) + } + return at + } + for _, tc := range []struct { + name string + start, end, watermark time.Time + wantStart, wantEnd time.Time + }{ + { + name: "a window on minute boundaries with a fresh cache is entirely interior", + start: minute("2026-09-08T10:00:00Z"), end: minute("2026-09-08T11:00:00Z"), watermark: minute("2026-09-08T11:00:00Z"), + wantStart: minute("2026-09-08T10:00:00Z"), wantEnd: minute("2026-09-08T11:00:00Z"), + }, + { + name: "a mid-minute start leaves its partial minute to raw spans", + start: minute("2026-09-08T10:00:30Z"), end: minute("2026-09-08T11:00:00Z"), watermark: minute("2026-09-08T11:00:00Z"), + wantStart: minute("2026-09-08T10:01:00Z"), wantEnd: minute("2026-09-08T11:00:00Z"), + }, + { + name: "a lagging watermark ends the interior early", + start: minute("2026-09-08T10:00:00Z"), end: minute("2026-09-08T11:00:00Z"), watermark: minute("2026-09-08T10:42:30Z"), + wantStart: minute("2026-09-08T10:00:00Z"), wantEnd: minute("2026-09-08T10:42:00Z"), + }, + { + name: "a watermark behind the window leaves no interior at all", + start: minute("2026-09-08T10:00:00Z"), end: minute("2026-09-08T11:00:00Z"), watermark: minute("2026-09-08T09:00:00Z"), + wantStart: minute("2026-09-08T10:00:00Z"), wantEnd: minute("2026-09-08T09:00:00Z"), + }, + } { + t.Run(tc.name, func(t *testing.T) { + gotStart, gotEnd := endpointInteriorBounds(tc.start, tc.end, tc.watermark) + if !gotStart.Equal(tc.wantStart) || !gotEnd.Equal(tc.wantEnd) { + t.Fatalf("bounds = %s..%s, want %s..%s", gotStart, gotEnd, tc.wantStart, tc.wantEnd) + } + }) + } +} + +// The boundary scan must bind its bounds directly: a bound derived from a +// joined row is not a constant DuckDB can push into the Parquet scan, which is +// what made a two-minute question read the whole window. +func TestEndpointRollupQueryBindsBoundsDirectly(t *testing.T) { + if strings.Contains(endpointRollupQuery, "bounds b") || strings.Contains(endpointRollupQuery, "FROM params") { + t.Fatal("the query still derives its bounds from a CTE join; the scan filter cannot be pushed") + } + if !strings.Contains(endpointRollupQuery, "s.start_time < ? OR s.start_time >= ?") { + t.Fatal("the boundary exclusion is no longer expressed against bound parameters") + } +} diff --git a/internal/observability/service_test.go b/internal/observability/service_test.go index cd0477421..caeaaffa2 100644 --- a/internal/observability/service_test.go +++ b/internal/observability/service_test.go @@ -220,8 +220,15 @@ func TestPerformanceReturnsAllVisualizationDatasets(t *testing.T) { for _, bucket := range endpointDurationBuckets { endpointColumns = append(endpointColumns, bucket.Column) } + // The interior bounds are computed in Go now, so the cache range and the + // boundary exclusion arrive as parameters DuckDB can push into the scan. + interiorStart, interiorEnd := endpointInteriorBounds(start, end, end) mock.ExpectQuery(regexp.QuoteMeta(endpointRollupQuery)). - WithArgs(start, end, end, "prod", "checkout", 25). + WithArgs( + interiorStart, interiorEnd, "prod", "prod", "checkout", "checkout", + start, end, interiorStart, interiorEnd, "prod", "prod", "checkout", "checkout", + 25, + ). WillReturnRows(sqlmock.NewRows(endpointColumns).AddRow(endpointCounts...)) mock.ExpectQuery(regexp.QuoteMeta(performanceHeatmapSQL(time.Hour))). WithArgs(start, end, "prod", "prod", start, end, "prod", "prod"). diff --git a/internal/query/duck.go b/internal/query/duck.go index 1713228e9..b69e3396a 100644 --- a/internal/query/duck.go +++ b/internal/query/duck.go @@ -430,13 +430,18 @@ func (d *Duck) Close() error { if d.releasePoolGauges != nil { d.releasePoolGauges() } - var writeErr error + var errs []error if d.writeDB != nil { // The write handle holds a connector that does not close the // instance, so this releases its connection and nothing else. - writeErr = d.writeDB.Close() + errs = append(errs, d.writeDB.Close()) + } + // A Duck assembled field by field, as tests do, may hold neither + // handle; closing one is then nothing to do rather than a panic. + if d.DB != nil { + errs = append(errs, d.DB.Close()) } - d.closeErr = errors.Join(writeErr, d.DB.Close()) + d.closeErr = errors.Join(errs...) }) return d.closeErr } @@ -1708,18 +1713,48 @@ FROM messaging_edges;` // ---- Read query helpers ---- +// acquireRead takes a connection from the read pool. The wait is already +// counted by the pool's own gauges; taking it explicitly keeps it out of the +// statement histogram. +func (d *Duck) acquireRead(ctx context.Context) (*sql.Conn, error) { + return d.DB.Conn(ctx) +} + // QueryContext executes a read against immutable Parquet files and DuckDB's // local rollup cache. func (d *Duck) QueryContext(ctx context.Context, query string, args ...any) (queryrows.Rows, error) { if err := d.lockParquetRead(ctx, readerQuery); err != nil { return nil, err } - rows, err := d.DB.QueryContext(ctx, query, args...) + // The connection is taken first and timed separately: DB.QueryContext would + // fold the wait for a free connection into the statement's duration, and + // telling queueing from execution is the whole reason this is measured. + conn, err := d.acquireRead(ctx) if err != nil { d.parquetMu.RUnlock() return nil, err } - return &lockedRows{Rows: rows, unlock: d.parquetMu.RUnlock}, nil + started := time.Now() + rows, err := conn.QueryContext(ctx, query, args...) + metrics.DuckDBStatement.Observe(time.Since(started).Seconds()) + if err != nil { + _ = conn.Close() + d.parquetMu.RUnlock() + return nil, err + } + release := func() { + // The connection goes back to the pool only once its rows are done + // with it, which is what closing the rows first guarantees. + // + // A caller that neither closes nor exhausts its rows now leaks the + // pool slot as well as the snapshot read-lock — database/sql used to + // return the connection when the context died, and an explicitly held + // one is the caller's until it says otherwise. Every call site closes + // on every path; this is what it costs if one stops. + _ = conn.Close() + d.parquetMu.RUnlock() + } + return &lockedRows{Rows: rows, unlock: release}, nil } type lockedRows struct { @@ -1751,7 +1786,17 @@ func (d *Duck) QueryRowScan(ctx context.Context, dest []any, query string, args return err } defer d.parquetMu.RUnlock() - return d.DB.QueryRowContext(ctx, query, args...).Scan(dest...) + conn, err := d.acquireRead(ctx) + if err != nil { + return err + } + defer func() { _ = conn.Close() }() + // This one covers the scan as well as the statement: a single row cannot + // be fetched separately from being read. Both are DuckDB's time rather + // than the pool's, which is the distinction the histogram is drawing. + started := time.Now() + defer func() { metrics.DuckDBStatement.Observe(time.Since(started).Seconds()) }() + return conn.QueryRowContext(ctx, query, args...).Scan(dest...) } // Trace pins the immutable Parquet snapshot for the full indexed-file read. diff --git a/internal/query/stall_instrumentation_test.go b/internal/query/stall_instrumentation_test.go index 0a984182f..c4eb76682 100644 --- a/internal/query/stall_instrumentation_test.go +++ b/internal/query/stall_instrumentation_test.go @@ -224,3 +224,74 @@ func TestClosingADisplacedDuckLeavesTheGaugeSourceAlone(t *testing.T) { t.Fatalf("pool gauges read MaxOpenConnections = %d after a displaced Duck closed, want the live pool's 3", got) } } + +// The statement histogram exists to separate execution from queueing, so the +// wait for a pool connection must not be inside it. DB.QueryContext would fold +// the two together; the connection is taken first for that reason. +func TestStatementTimerExcludesTheWaitForAConnection(t *testing.T) { + ctx := context.Background() + cfg := config.Config{DataDir: t.TempDir(), RollupInterval: time.Minute, DuckDBMemory: "128MB", DuckDBMaxConns: 1} + repository, err := telemetrystore.Open(cfg.TelemetryDir()) + if err != nil { + t.Fatalf("open telemetry repository: %v", err) + } + defer repository.Close() + d, err := NewDuck(ctx, cfg, repository) + if err != nil { + t.Fatalf("NewDuck: %v", err) + } + defer d.Close() + + // Hold the pool's only connection, so a second reader must wait for it. + held, err := d.DB.Conn(ctx) + if err != nil { + t.Fatalf("hold the connection: %v", err) + } + waited := make(chan time.Duration, 1) + go func() { + started := time.Now() + rows, err := d.QueryContext(ctx, "SELECT 1") + if err != nil { + waited <- -1 + return + } + _ = rows.Close() + waited <- time.Since(started) + }() + + // Long enough that a timer covering the wait could not miss it. + time.Sleep(250 * time.Millisecond) + before := statementSeconds(t) + if err := held.Close(); err != nil { + t.Fatalf("release the connection: %v", err) + } + total := <-waited + if total < 200*time.Millisecond { + t.Fatalf("the second reader waited %s; the test did not create contention", total) + } + observed := statementSeconds(t) - before + if observed <= 0 { + t.Fatal("no statement duration was recorded") + } + if observed > 0.2 { + t.Fatalf("statement recorded %.3fs of a %s call: the wait for a connection is inside the timer", observed, total) + } +} + +// statementSeconds reports the histogram's running total. +func statementSeconds(t *testing.T) float64 { + t.Helper() + return metrics.DuckDBStatementSecondsForTest() +} + +// Close is reached from error paths and from test cleanup, where a Duck may +// hold no handles at all. It has nothing to do there, rather than a panic. +func TestClosingADuckWithNoHandlesIsHarmless(t *testing.T) { + var empty Duck + if err := empty.Close(); err != nil { + t.Fatalf("closing a Duck with no handles: %v", err) + } + if err := empty.Close(); err != nil { + t.Fatalf("closing it again: %v", err) + } +}