diff --git a/internal/api/observability.go b/internal/api/observability.go index ddd6b2447..fd8bda4b9 100644 --- a/internal/api/observability.go +++ b/internal/api/observability.go @@ -29,15 +29,42 @@ func NewObservabilityHandler(queries ObservabilityQueries) *ObservabilityHandler return &ObservabilityHandler{queries: queries, now: time.Now} } -// Register mounts the deterministic product API. These endpoints are for -// dashboards and drill-downs; the browser does not need an agent for ordinary -// telemetry navigation. +// observabilityDeadline bounds one dashboard query. +// +// The SQL API has always been bounded; these endpoints, which every widget +// calls, had no deadline of their own — a query that went pathological held a +// connection until the client gave up, and the client's own timeout is not +// something the server can rely on. +// +// Twenty seconds is well clear of the slowest query measured under load here, +// about two seconds, and stricter than the SQL API's own ceiling. It is +// deliberately shorter than the worst legitimate wait behind a stuck +// publication — the drain remainder plus the swap budget can exceed a minute — +// so a publication that goes wrong answers the dashboard instead of holding +// every widget's connection until it finishes. +const observabilityDeadline = 20 * time.Second + +// Register mounts the deterministic product API, each endpoint under a +// deadline. These endpoints are for dashboards and drill-downs; the browser +// does not need an agent for ordinary telemetry navigation. func (h *ObservabilityHandler) Register(group *echo.Group) { - group.GET("/overview", h.overview) - group.GET("/topology", h.topology) - group.GET("/performance", h.performance) - group.GET("/trace", h.trace) - group.GET("/logs", h.logs) + group.GET("/overview", h.bounded(h.overview)) + group.GET("/topology", h.bounded(h.topology)) + group.GET("/performance", h.bounded(h.performance)) + group.GET("/trace", h.bounded(h.trace)) + group.GET("/logs", h.bounded(h.logs)) +} + +// bounded gives a handler a deadline. The context reaches DuckDB through +// QueryContext, so an expired one stops the statement rather than only the +// reply — and releases the pool connection it was holding. +func (h *ObservabilityHandler) bounded(handler func(*echo.Context) error) func(*echo.Context) error { + return func(c *echo.Context) error { + ctx, cancel := context.WithTimeout(c.Request().Context(), observabilityDeadline) + defer cancel() + c.SetRequest(c.Request().WithContext(ctx)) + return handler(c) + } } func (h *ObservabilityHandler) performance(c *echo.Context) error { @@ -129,5 +156,11 @@ func mapQueryError(err error) error { if errors.Is(err, observability.ErrInvalidScope) || errors.Is(err, observability.ErrInvalidLimit) { return echo.NewHTTPError(http.StatusBadRequest, err.Error()) } + // A query that ran out of time is a load condition, not a fault: reporting + // it as a server error logs it at ERROR and counts it towards the 5xx rate + // that pages someone. + if errors.Is(err, context.DeadlineExceeded) { + return echo.NewHTTPError(http.StatusServiceUnavailable, "telemetry query took too long").Wrap(err) + } return echo.NewHTTPError(http.StatusInternalServerError, "telemetry query failed").Wrap(err) } diff --git a/internal/api/observability_test.go b/internal/api/observability_test.go index 0c2b8cf69..bebfe4514 100644 --- a/internal/api/observability_test.go +++ b/internal/api/observability_test.go @@ -3,6 +3,8 @@ package api import ( "context" "encoding/json" + "errors" + "fmt" "net/http" "net/http/httptest" "testing" @@ -91,3 +93,98 @@ func TestRouteRejectsInvalidWindow(t *testing.T) { t.Fatalf("status = %d, want 400", rec.Code) } } + +// A dashboard query that runs out of time is a load condition. Reporting it as +// a server error logs it at ERROR and counts towards the 5xx rate that pages +// someone; the deadline itself exists so a pathological query stops holding a +// connection once the answer can no longer be useful. +func TestObservabilityQueryDeadlineIsNotAServerError(t *testing.T) { + err := mapQueryError(fmt.Errorf("query endpoints: %w", context.DeadlineExceeded)) + var httpErr *echo.HTTPError + if !errors.As(err, &httpErr) { + t.Fatalf("mapQueryError returned %T, want an *echo.HTTPError", err) + } + if httpErr.Code != http.StatusServiceUnavailable { + t.Fatalf("a timed-out query maps to %d, want %d", httpErr.Code, http.StatusServiceUnavailable) + } + // A genuine failure still reads as one. + var failure *echo.HTTPError + if !errors.As(mapQueryError(errors.New("boom")), &failure) || failure.Code != http.StatusInternalServerError { + t.Fatal("a real query failure no longer maps to 500") + } +} + +// Every registered route carries the deadline, not just the wrapper in +// isolation: dropping bounded() from Register is the mistake a refactor makes, +// and it left the whole suite green. +func TestRegisteredObservabilityRoutesCarryADeadline(t *testing.T) { + deadlines := map[string]bool{} + handler := &ObservabilityHandler{queries: deadlineProbe{seen: deadlines}, now: time.Now} + e := echo.New() + handler.Register(e.Group("/api/observability")) + for _, path := range []string{"/api/observability/overview", "/api/observability/topology", "/api/observability/performance", "/api/observability/logs", "/api/observability/trace"} { + recorder := httptest.NewRecorder() + e.ServeHTTP(recorder, httptest.NewRequest(http.MethodGet, path+"?window=1h", nil)) + if !deadlines[path] { + t.Errorf("%s reached the query layer with no deadline", path) + } + } +} + +// deadlineProbe records, per route, whether the context that reached it had a +// deadline. It answers every query with an empty result. +type deadlineProbe struct{ seen map[string]bool } + +func (p deadlineProbe) note(ctx context.Context, path string) { + if _, ok := ctx.Deadline(); ok { + p.seen[path] = true + } +} + +func (p deadlineProbe) Overview(ctx context.Context, scope observability.Scope, limit int) (observability.Result[observability.Overview], error) { + p.note(ctx, "/api/observability/overview") + return observability.Result[observability.Overview]{}, nil +} + +func (p deadlineProbe) Topology(ctx context.Context, scope observability.Scope, limit int) (observability.Result[observability.Topology], error) { + p.note(ctx, "/api/observability/topology") + return observability.Result[observability.Topology]{}, nil +} + +func (p deadlineProbe) Performance(ctx context.Context, scope observability.Scope, service string, limit int) (observability.Result[observability.Performance], error) { + p.note(ctx, "/api/observability/performance") + return observability.Result[observability.Performance]{}, nil +} + +func (p deadlineProbe) Logs(ctx context.Context, scope observability.Scope, service, severity, search string, limit int) (observability.Result[observability.Logs], error) { + p.note(ctx, "/api/observability/logs") + return observability.Result[observability.Logs]{}, nil +} + +func (p deadlineProbe) Trace(ctx context.Context, scope observability.Scope, traceID, service string, limit int) (observability.Result[observability.TraceDetail], error) { + p.note(ctx, "/api/observability/trace") + return observability.Result[observability.TraceDetail]{}, nil +} + +// The handlers carry a deadline of their own, because the client's timeout is +// not something the server can rely on. +func TestObservabilityHandlersCarryADeadline(t *testing.T) { + var seen time.Duration + handler := (&ObservabilityHandler{}).bounded(func(c *echo.Context) error { + deadline, ok := c.Request().Context().Deadline() + if !ok { + t.Fatal("handler ran with no deadline") + } + seen = time.Until(deadline) + return nil + }) + e := echo.New() + request := httptest.NewRequest(http.MethodGet, "/api/observability/overview", nil) + c := e.NewContext(request, httptest.NewRecorder()) + if err := handler(c); err != nil { + t.Fatalf("bounded handler: %v", err) + } + if seen <= 0 || seen > observabilityDeadline { + t.Fatalf("deadline left %s, want a positive value no greater than %s", seen, observabilityDeadline) + } +} diff --git a/internal/metrics/metrics.go b/internal/metrics/metrics.go index 07e1c85af..a855c9f28 100644 --- a/internal/metrics/metrics.go +++ b/internal/metrics/metrics.go @@ -1,10 +1,13 @@ package metrics import ( + "database/sql" "math" + "sync/atomic" "github.com/prometheus/client_golang/prometheus" "github.com/prometheus/client_golang/prometheus/promauto" + dto "github.com/prometheus/client_model/go" ) type RollupComponent string @@ -204,6 +207,57 @@ var ( Help: "Parquet publications abandoned after reader wait timeout", }) + // The publisher's side of Parquet publication was measured and the + // reader's side was not, which left the one question a stalled query asks + // — "how long did I wait to enter the snapshot?" — unanswerable. These + // three complete the pair. + // An uncontended admission costs single-digit microseconds, so the ladder + // starts below that: a histogram whose healthy traffic all lands in the + // first bucket can tell you a stall happened but not what normal was. + snapshotBuckets = []float64{.000001, .000005, .00002, .0001, .0005, .001, .005, .025, .1, .5, 1, 2.5, 5, 10, 30, 60} + + ParquetPublishHold = promauto.NewHistogram(prometheus.HistogramOpts{ + Name: "fanout_parquet_publish_hold_seconds", + Help: "Time readers were excluded while a new Parquet file set was swapped in", + Buckets: snapshotBuckets, + }) + + parquetReadWait = promauto.NewHistogramVec(prometheus.HistogramOpts{ + Name: "fanout_parquet_read_wait_seconds", + Help: "Time a reader spent waiting to enter the Parquet snapshot", + Buckets: snapshotBuckets, + }, []string{"reader"}) + + parquetReadRefusals = promauto.NewCounterVec(prometheus.CounterOpts{ + Name: "fanout_parquet_read_refusals_total", + Help: "Readers that gave up waiting to enter the Parquet snapshot", + }, []string{"reader"}) + + // The wait to enter the snapshot and the wait for a connection are both + // measured; how long the statement itself takes was not, so a slow request + // could be attributed to queueing or to execution only by elimination. + // The connection is acquired before this timer starts, so the two do not + // overlap. + DuckDBStatement = promauto.NewHistogram(prometheus.HistogramOpts{ + Name: "fanout_duckdb_statement_seconds", + Help: "Time a read statement spent executing in DuckDB, excluding any wait to start it", + Buckets: []float64{.0005, .001, .005, .01, .025, .05, .1, .25, .5, 1, 2.5, 5, 10, 30}, + }) + + // An explicit CHECKPOINT blocks new transactions for its duration, and the + // non-FORCE form refuses while another write transaction is open — so both + // how long it takes and how often it does not happen are worth knowing. + DuckDBCheckpoint = promauto.NewHistogram(prometheus.HistogramOpts{ + Name: "fanout_duckdb_checkpoint_seconds", + Help: "Duration of an explicit DuckDB checkpoint on the rollup cache", + Buckets: []float64{.001, .005, .01, .05, .1, .25, .5, 1, 2.5, 5, 10, 30}, + }) + + DuckDBCheckpointFailures = promauto.NewCounter(prometheus.CounterOpts{ + Name: "fanout_duckdb_checkpoint_failures_total", + Help: "Explicit DuckDB checkpoints that returned an error", + }) + // HTTP metrics HTTPRequestsTotal = promauto.NewCounterVec(prometheus.CounterOpts{ Name: "fanout_http_requests_total", @@ -239,6 +293,104 @@ func RecordFlush(signal string, durationSec float64) { FlushDuration.WithLabelValues(signal).Observe(durationSec) } +// duckDBPoolStats is the live source for the connection-pool gauges below. +// +// A holder rather than a promauto.NewGaugeFunc closure over the pool: the +// gauges are declared once for the process, while a *Duck is constructed many +// times in a test binary, and registering a collector per construction panics +// on the second one. Unset, every gauge reads zero. +var duckDBPoolStats atomic.Pointer[func() sql.DBStats] + +// SetDuckDBPoolSource points the connection-pool gauges at a pool and returns +// the function that stops reading it. The last caller wins; the returned +// release only clears the source if it is still the one it installed, so a +// second Duck closing in a test binary cannot blank the gauges of one that is +// still serving. +func SetDuckDBPoolSource(stats func() sql.DBStats) (release func()) { + if stats == nil { + return func() {} + } + installed := &stats + duckDBPoolStats.Store(installed) + return func() { duckDBPoolStats.CompareAndSwap(installed, nil) } +} + +func poolStats() sql.DBStats { + if fn := duckDBPoolStats.Load(); fn != nil { + return (*fn)() + } + return sql.DBStats{} +} + +// The read pool is the one that can starve: background writers, the detector, +// the alert engine and every dashboard widget draw from it, and a request that +// waits for a connection waits invisibly — database/sql counts that wait and +// nothing exported it. +var ( + _ = promauto.NewGaugeFunc(prometheus.GaugeOpts{ + Name: "fanout_duckdb_pool_in_use", + Help: "DuckDB read connections currently in use", + }, func() float64 { return float64(poolStats().InUse) }) + + _ = promauto.NewGaugeFunc(prometheus.GaugeOpts{ + Name: "fanout_duckdb_pool_idle", + Help: "DuckDB read connections currently idle", + }, func() float64 { return float64(poolStats().Idle) }) + + _ = promauto.NewGaugeFunc(prometheus.GaugeOpts{ + Name: "fanout_duckdb_pool_max_open", + Help: "Configured DuckDB read connection limit", + }, func() float64 { return float64(poolStats().MaxOpenConnections) }) + + _ = promauto.NewCounterFunc(prometheus.CounterOpts{ + Name: "fanout_duckdb_pool_waits_total", + Help: "Times a caller waited for a free DuckDB read connection", + }, func() float64 { return float64(poolStats().WaitCount) }) + + _ = promauto.NewCounterFunc(prometheus.CounterOpts{ + Name: "fanout_duckdb_pool_wait_seconds_total", + Help: "Total time callers spent waiting for a free DuckDB read connection", + }, func() float64 { return poolStats().WaitDuration.Seconds() }) +) + +// RecordParquetRead records one attempt to enter the Parquet snapshot. +// +// Both series are behind this function for the same reason RecordWriteGate +// exists: the label is a fixed reader class, and a vector left exported is a +// vector something eventually calls with a service name in it. +func RecordParquetRead(reader string, waitSec float64, admitted bool) { + if admitted { + parquetReadWait.WithLabelValues(reader).Observe(waitSec) + return + } + parquetReadRefusals.WithLabelValues(reader).Inc() +} + +// ParquetReadWaitForTest and ParquetReadRefusalsForTest hand the two series to +// a test in another package, which asserts on them rather than on the call — +// the call is the easy half. They return collectors rather than the vectors so +// production code still cannot set an arbitrary label, and they keep +// prometheus/testutil out of the release dependency graph. +func ParquetReadWaitForTest() prometheus.Collector { return parquetReadWait } + +func ParquetReadRefusalsForTest(reader string) prometheus.Counter { + return parquetReadRefusals.WithLabelValues(reader) +} + +// DuckDBStatementSecondsForTest reports the statement histogram's running +// total, so a test can prove what the timer does and does not cover. +func DuckDBStatementSecondsForTest() float64 { + metric := &dto.Metric{} + if err := DuckDBStatement.Write(metric); err != nil { + return 0 + } + return metric.GetHistogram().GetSampleSum() +} + +// DuckDBPoolStatsForTest exposes what the pool gauges are reading, so a test +// can prove they point at the read pool rather than the write handle. +func DuckDBPoolStatsForTest() sql.DBStats { return poolStats() } + // RecordWriteGate records one complete Telemetry catalog write critical section. // Callers constrain operation to the fixed writegate.WriteOperation set so this // metric cannot grow with tenant or telemetry cardinality. diff --git a/internal/observability/logs.go b/internal/observability/logs.go index 963788f56..9fad8200d 100644 --- a/internal/observability/logs.go +++ b/internal/observability/logs.go @@ -17,12 +17,28 @@ WHERE time >= ? AND time < ? // DuckDB answers the two questions the API asks: the newest `limit` entries // and per-bucket counts. LIMIT bounds the entry stream and GROUP BY bounds the // histogram stream regardless of the retained Parquet row count. +// +// Redaction runs outside the LIMIT, and that placement is the whole cost of +// this query. A projection in the same SELECT as ORDER BY ... LIMIT is +// evaluated below the top-N operator, so four chained regexp_replace ran over +// every row in the window to return a hundred: measured at 140ms for a page of +// 100 out of 372,000 rows, and 347ms with four DuckDB threads, against 8ms +// once the top-N runs first. It was the one statement here whose cost grew +// with retained rows rather than with the answer. +// +// The search predicate stays inside, on purpose: a search has to match what +// the reader will be shown, so it matches the redacted text. It is skipped +// entirely when no search term is given. var logEntriesQuery = ` -SELECT time, severity, coalesce(service, ''), ` + redactLogBodySQL("body") + `, - coalesce(trace_id, ''), coalesce(span_id, '') - FROM logs` + logFilters + ` -ORDER BY time DESC -LIMIT ?` +SELECT time, severity, service, ` + redactLogBodySQL("body") + `, trace_id, span_id +FROM ( + SELECT time, severity, coalesce(service, '') AS service, body, + coalesce(trace_id, '') AS trace_id, coalesce(span_id, '') AS span_id + FROM logs` + logFilters + ` + ORDER BY time DESC + LIMIT ? +) +ORDER BY time DESC` var logBucketsQueryTemplate = ` SELECT time_bucket(INTERVAL '%s', time) AS point_time, diff --git a/internal/observability/performance.go b/internal/observability/performance.go index e6d639e17..f0378304a 100644 --- a/internal/observability/performance.go +++ b/internal/observability/performance.go @@ -51,35 +51,50 @@ const endpointRollupMatureQuery = ` SELECT COUNT(*) >= 5 FROM (SELECT DISTINCT bucket FROM endpoint_rollup LIMIT 5)` +// endpointInteriorBounds returns the half-open minute range the rollup cache +// answers for, given the requested window and the cache's watermark. +// +// The interior starts at the first whole minute inside the window — a window +// that begins mid-minute has a partial minute the cache cannot answer — and +// ends at the earlier of the window's last whole minute and the watermark's, +// because the cache knows nothing past what it has rolled up. Raw spans cover +// everything outside that range. +func endpointInteriorBounds(start, end, watermark time.Time) (interiorStart, interiorEnd time.Time) { + interiorStart = start.Truncate(time.Minute) + if !interiorStart.Equal(start) { + interiorStart = interiorStart.Add(time.Minute) + } + interiorEnd = end.Truncate(time.Minute) + if rolled := watermark.Truncate(time.Minute); rolled.Before(interiorEnd) { + interiorEnd = rolled + } + return interiorStart, interiorEnd +} + // endpointRollupQuery uses the minute cache only through its current watermark. // Raw spans cover both partial boundary minutes and any newer complete minutes, -// so the cache lag cannot appear as zero traffic and raw work stays bounded. +// so the cache lag cannot appear as zero traffic. +// +// The interior bounds are computed in Go and bound as parameters rather than +// derived in a CTE. They read the same either way, but a bound derived from a +// joined row is not a constant DuckDB can push into the Parquet scan: the +// boundary exclusion below was applied after reading the window, so a query +// that wants the two partial minutes at its edges read every row between them — +// measured at 799,333 rows scanned out of 800,000 to return 7,333. Bound +// directly, the same filter prunes to 11,333 rows read. +// +// The raw half is bounded by the cache being current, not by this query. When +// the watermark sits behind the window the interior is empty, the boundary +// predicate admits everything, and the scan reads the whole window — which is +// correct, and is the case worth watching after an outage or a first run. var endpointRollupQuery = ` -WITH params AS ( - SELECT - ?::TIMESTAMP AS start_time, - ?::TIMESTAMP AS end_time, - ?::TIMESTAMP AS rollup_end, - ?::VARCHAR AS namespace, - ?::VARCHAR AS service -), -bounds AS ( - SELECT - *, - CASE - WHEN start_time = date_trunc('minute', start_time) THEN start_time - ELSE date_trunc('minute', start_time) + INTERVAL 1 MINUTE - END AS interior_start, - LEAST(date_trunc('minute', end_time), date_trunc('minute', rollup_end)) AS interior_end - FROM params -), -rollup_source AS ( +WITH rollup_source AS ( SELECT e.method, e.path, e.calls, e.error_count, e.duration_count, e.duration_buckets - FROM endpoint_rollup e, bounds b - WHERE e.bucket >= b.interior_start - AND e.bucket < b.interior_end - AND (b.namespace = '' OR e.namespace = b.namespace) - AND (b.service = '' OR e.service = b.service) + FROM endpoint_rollup e + WHERE e.bucket >= ? + AND e.bucket < ? + AND (? = '' OR e.namespace = ?) + AND (? = '' OR e.service = ?) ), boundary_source AS ( SELECT @@ -107,12 +122,12 @@ boundary_source AS ( le_30000 := COUNT(*) FILTER (WHERE s.duration_ms <= 30000), le_300000 := COUNT(*) FILTER (WHERE s.duration_ms <= 300000) ) AS duration_buckets - FROM spans s, bounds b - WHERE s.start_time >= b.start_time - AND s.start_time < b.end_time - AND (s.start_time < b.interior_start OR s.start_time >= b.interior_end) - AND (b.namespace = '' OR s.namespace = b.namespace) - AND (b.service = '' OR COALESCE(s.service, '') = b.service) + FROM spans s + WHERE s.start_time >= ? + AND s.start_time < ? + AND (s.start_time < ? OR s.start_time >= ?) + AND (? = '' OR s.namespace = ?) + AND (? = '' OR COALESCE(s.service, '') = ?) GROUP BY method, path ), sources AS ( @@ -286,7 +301,12 @@ func (s *Service) queryEndpoints(ctx context.Context, scope Scope, service strin source := "spans" if ready { query = endpointRollupQuery - args = []any{scope.Start, scope.End, watermark, scope.Namespace, service, limit} + interiorStart, interiorEnd := endpointInteriorBounds(scope.Start, scope.End, watermark) + args = []any{ + interiorStart, interiorEnd, scope.Namespace, scope.Namespace, service, service, + scope.Start, scope.End, interiorStart, interiorEnd, scope.Namespace, scope.Namespace, service, service, + limit, + } source = "endpoint_rollup + raw spans (interpolated histogram percentiles)" } rows, err := s.db.QueryContext(ctx, query, args...) diff --git a/internal/observability/performance_test.go b/internal/observability/performance_test.go new file mode 100644 index 000000000..ea3e84f0e --- /dev/null +++ b/internal/observability/performance_test.go @@ -0,0 +1,65 @@ +package observability + +import ( + "strings" + "testing" + "time" +) + +// The interior range is what the rollup cache can answer for. A window that +// starts mid-minute has a partial minute the cache does not hold, and the cache +// knows nothing past its own watermark — raw spans cover both edges. +func TestEndpointInteriorBounds(t *testing.T) { + minute := func(s string) time.Time { + at, err := time.Parse(time.RFC3339Nano, s) + if err != nil { + t.Fatal(err) + } + return at + } + for _, tc := range []struct { + name string + start, end, watermark time.Time + wantStart, wantEnd time.Time + }{ + { + name: "a window on minute boundaries with a fresh cache is entirely interior", + start: minute("2026-09-08T10:00:00Z"), end: minute("2026-09-08T11:00:00Z"), watermark: minute("2026-09-08T11:00:00Z"), + wantStart: minute("2026-09-08T10:00:00Z"), wantEnd: minute("2026-09-08T11:00:00Z"), + }, + { + name: "a mid-minute start leaves its partial minute to raw spans", + start: minute("2026-09-08T10:00:30Z"), end: minute("2026-09-08T11:00:00Z"), watermark: minute("2026-09-08T11:00:00Z"), + wantStart: minute("2026-09-08T10:01:00Z"), wantEnd: minute("2026-09-08T11:00:00Z"), + }, + { + name: "a lagging watermark ends the interior early", + start: minute("2026-09-08T10:00:00Z"), end: minute("2026-09-08T11:00:00Z"), watermark: minute("2026-09-08T10:42:30Z"), + wantStart: minute("2026-09-08T10:00:00Z"), wantEnd: minute("2026-09-08T10:42:00Z"), + }, + { + name: "a watermark behind the window leaves no interior at all", + start: minute("2026-09-08T10:00:00Z"), end: minute("2026-09-08T11:00:00Z"), watermark: minute("2026-09-08T09:00:00Z"), + wantStart: minute("2026-09-08T10:00:00Z"), wantEnd: minute("2026-09-08T09:00:00Z"), + }, + } { + t.Run(tc.name, func(t *testing.T) { + gotStart, gotEnd := endpointInteriorBounds(tc.start, tc.end, tc.watermark) + if !gotStart.Equal(tc.wantStart) || !gotEnd.Equal(tc.wantEnd) { + t.Fatalf("bounds = %s..%s, want %s..%s", gotStart, gotEnd, tc.wantStart, tc.wantEnd) + } + }) + } +} + +// The boundary scan must bind its bounds directly: a bound derived from a +// joined row is not a constant DuckDB can push into the Parquet scan, which is +// what made a two-minute question read the whole window. +func TestEndpointRollupQueryBindsBoundsDirectly(t *testing.T) { + if strings.Contains(endpointRollupQuery, "bounds b") || strings.Contains(endpointRollupQuery, "FROM params") { + t.Fatal("the query still derives its bounds from a CTE join; the scan filter cannot be pushed") + } + if !strings.Contains(endpointRollupQuery, "s.start_time < ? OR s.start_time >= ?") { + t.Fatal("the boundary exclusion is no longer expressed against bound parameters") + } +} diff --git a/internal/observability/service_test.go b/internal/observability/service_test.go index cd0477421..caeaaffa2 100644 --- a/internal/observability/service_test.go +++ b/internal/observability/service_test.go @@ -220,8 +220,15 @@ func TestPerformanceReturnsAllVisualizationDatasets(t *testing.T) { for _, bucket := range endpointDurationBuckets { endpointColumns = append(endpointColumns, bucket.Column) } + // The interior bounds are computed in Go now, so the cache range and the + // boundary exclusion arrive as parameters DuckDB can push into the scan. + interiorStart, interiorEnd := endpointInteriorBounds(start, end, end) mock.ExpectQuery(regexp.QuoteMeta(endpointRollupQuery)). - WithArgs(start, end, end, "prod", "checkout", 25). + WithArgs( + interiorStart, interiorEnd, "prod", "prod", "checkout", "checkout", + start, end, interiorStart, interiorEnd, "prod", "prod", "checkout", "checkout", + 25, + ). WillReturnRows(sqlmock.NewRows(endpointColumns).AddRow(endpointCounts...)) mock.ExpectQuery(regexp.QuoteMeta(performanceHeatmapSQL(time.Hour))). WithArgs(start, end, "prod", "prod", start, end, "prod", "prod"). diff --git a/internal/query/duck.go b/internal/query/duck.go index c5405e4ce..b69e3396a 100644 --- a/internal/query/duck.go +++ b/internal/query/duck.go @@ -24,7 +24,26 @@ import ( ) type Duck struct { - DB *sql.DB + // DB is the read pool. Reads scan immutable Parquet concurrently, so it is + // sized from the machine. + DB *sql.DB + // closeOnce makes Close idempotent. Both handles come from one connector, + // and database/sql closes a connector that implements io.Closer — so a + // second Close would call duckdb_close twice on the same instance, which + // is a double free the runtime cannot recover from. + closeOnce sync.Once + closeErr error + // releasePoolGauges stops the pool gauges reading this Duck's pool, and + // only if they are still reading it. + releasePoolGauges func() + // writeDB is the single connection every rollup and maintenance write uses. + // + // Those writes are serialized by writeGate anyway, so one connection is all + // they can use — and taking it from the read pool is what let a background + // pass occupy a quarter of a four-core host's read capacity for the length + // of a rollup. A request that then waits for a connection waits invisibly: + // database/sql counts that wait, and nothing charged it to anything. + writeDB *sql.DB cfg config.Config repository *telemetrystore.Repository // rollupLagNanos holds the rollup watermark back from the max ingested @@ -101,6 +120,13 @@ const ( // so the internal lease can be exercised without a five-second test. var parquetStatsWait = 5 * time.Second +// edgeYieldMargin is how much of a queued publication's grace has to be left +// for the edge rollup to take another sub-window. It is a rough estimate of +// what one more sub-window costs: enough that the pass usually finishes before +// readers start queuing, small enough that the eight-sub-window budget is not +// given up for a publication that has only just arrived. +const edgeYieldMargin = 10 * time.Second + // rollupPublicationSafetyLag covers the maximum public SQL hold, publication // grace, bounded commit retries, and queued Parquet encoding with headroom for // a busy disk. Rows stamped at request receipt remain inside the recomputed @@ -231,12 +257,13 @@ func NewDuck(ctx context.Context, cfg config.Config, repository *telemetrystore. return nil, err } - db, err := openDuckDB(ctx, dsn, tempDir, duckDBPoolSize(cfg)) + db, writeDB, err := openDuckDB(ctx, dsn, tempDir, duckDBPoolSize(cfg)) if err != nil { return nil, fmt.Errorf("open DuckDB query cache: %w (the cache at %s is rebuildable from Parquet)", err, dbPath) } - - d := &Duck{DB: db, cfg: cfg, repository: repository, rollupLagNanos: int64(rollupPublicationSafetyLag)} + d := &Duck{DB: db, writeDB: writeDB, cfg: cfg, repository: repository, rollupLagNanos: int64(rollupPublicationSafetyLag)} + // Reads are the pool that can starve, so it is the one worth watching. + d.releasePoolGauges = metrics.SetDuckDBPoolSource(db.Stats) if cfg.DuckDBMemory == "" { // Only when the operator hasn't pinned storage.duckdb.memory: keep DuckDB's // cgroup-aware auto limit on big boxes but leave absolute RAM headroom on @@ -245,16 +272,19 @@ func NewDuck(ctx context.Context, cfg config.Config, repository *telemetrystore. slog.Warn("apply memory headroom failed; using DuckDB default memory_limit", "err", err) } } - if err := CreateCacheTables(db); err != nil { - _ = db.Close() + // DDL goes through the write handle for the same reason every other write + // does; the two handles share one DuckDB instance, so the tables and views + // are the same ones the read pool sees. + if err := CreateCacheTables(writeDB); err != nil { + _ = d.Close() return nil, err } - if err := CreateParquetViews(db, repository.Parquet.Dir()); err != nil { - _ = db.Close() + if err := CreateParquetViews(writeDB, repository.Parquet.Dir()); err != nil { + _ = d.Close() return nil, err } - if err := CreateViews(db); err != nil { - _ = db.Close() + if err := CreateViews(writeDB); err != nil { + _ = d.Close() return nil, fmt.Errorf("create views: %w", err) } if cfg.RollupSkipToLatest { @@ -278,7 +308,7 @@ func (d *Duck) skipRollupToLatest(ctx context.Context) error { unlock := d.writeGate.Lock(writegate.WriteRollupSkip) defer unlock() - tx, err := d.DB.BeginTx(ctx, nil) + tx, err := d.writer().BeginTx(ctx, nil) if err != nil { return err } @@ -323,7 +353,14 @@ func (d *Duck) skipRollupToLatest(ctx context.Context) error { return tx.Commit() } -func openDuckDB(ctx context.Context, dsn, tempDir string, maxConns int) (*sql.DB, error) { +// openDuckDB returns the read pool and the single-connection write handle. +// +// Both come from one connector so they share one DuckDB instance and therefore +// one catalog: a write committed on one handle is visible to the other, and +// DDL run at boot applies to both. (A second connector to the same file would +// also work — DuckDB keeps an instance cache — but two owners of one instance +// is exactly the hazard the wrapper below exists to avoid.) +func openDuckDB(ctx context.Context, dsn, tempDir string, maxConns int) (*sql.DB, *sql.DB, error) { // Every pooled connection must use UTC. DuckDB otherwise inherits the host // timezone and casts TIMESTAMPTZ rollup buckets into local wall-clock // TIMESTAMP values, while API windows arrive in UTC. @@ -352,7 +389,7 @@ func openDuckDB(ctx context.Context, dsn, tempDir string, maxConns int) (*sql.DB return nil }) if err != nil { - return nil, err + return nil, nil, err } db := sql.OpenDB(connector) @@ -363,15 +400,80 @@ func openDuckDB(ctx context.Context, dsn, tempDir string, maxConns int) (*sql.DB } db.SetMaxOpenConns(maxConns) db.SetMaxIdleConns(maxConns) - return db, nil + + // One connection is all the writers can use — writeGate already lets one + // through at a time — and keeping it out of the read pool is the point. + // + // The write handle gets a connector that cannot close the instance. + // database/sql closes a connector that implements io.Closer when its DB is + // closed, and duckdb's Connector.Close calls duckdb_close behind an + // unsynchronized bool — so two handles over one connector means two + // duckdb_close calls on one instance, which crashes the process rather + // than returning an error. The read pool owns the instance; the write + // handle borrows it. + writeDB := sql.OpenDB(borrowedConnector{connector}) + writeDB.SetMaxOpenConns(1) + writeDB.SetMaxIdleConns(1) + return db, writeDB, nil } +// borrowedConnector hands out connections from a connector it does not own. +// It deliberately does not implement io.Closer: that is the whole point. +type borrowedConnector struct{ driver.Connector } + func sqlLiteral(v string) string { return "'" + strings.ReplaceAll(v, "'", "''") + "'" } func (d *Duck) Close() error { - return d.DB.Close() + d.closeOnce.Do(func() { + if d.releasePoolGauges != nil { + d.releasePoolGauges() + } + var errs []error + if d.writeDB != nil { + // The write handle holds a connector that does not close the + // instance, so this releases its connection and nothing else. + errs = append(errs, d.writeDB.Close()) + } + // A Duck assembled field by field, as tests do, may hold neither + // handle; closing one is then nothing to do rather than a panic. + if d.DB != nil { + errs = append(errs, d.DB.Close()) + } + d.closeErr = errors.Join(errs...) + }) + return d.closeErr +} + +// writer returns the handle every write goes through. NewDuck always sets one; +// the fallback is for a Duck assembled field-by-field in a test, where one +// handle is the whole point. +func (d *Duck) writer() *sql.DB { + if d.writeDB != nil { + return d.writeDB + } + return d.DB +} + +// checkpoint folds the rollup cache's WAL back into its file, and says how long +// that took. A checkpoint blocks new transactions for its duration, so it is a +// candidate explanation for a stalled query and has to be measurable. +// +// The non-FORCE form refuses while another *write* transaction is open. Every +// write here is serialized behind the write gate on one connection, and this +// runs under that gate, so a refusal should not happen — the counter is there +// to say so rather than to be assumed. Only a checkpoint that ran is timed: a +// refusal returns in microseconds and would drag the distribution towards zero, +// which is the opposite of what the histogram is for. +func (d *Duck) checkpoint(ctx context.Context) error { + start := time.Now() + if _, err := d.writer().ExecContext(ctx, "CHECKPOINT"); err != nil { + metrics.DuckDBCheckpointFailures.Inc() + return err + } + metrics.DuckDBCheckpoint.Observe(time.Since(start).Seconds()) + return nil } // DefaultNamespace returns empty string so queries search all namespaces. @@ -428,7 +530,7 @@ func (d *Duck) RunRollups(ctx context.Context) { func (d *Duck) updateParquetStats(ctx context.Context) { statsCtx, cancel := context.WithTimeout(ctx, parquetStatsWait) defer cancel() - if err := d.lockParquetRead(statsCtx); err != nil { + if err := d.lockParquetRead(statsCtx, readerStats); err != nil { slog.Warn("parquet stats skipped", "err", err) return } @@ -556,13 +658,12 @@ func (d *Duck) runRepositoryMaintenance(ctx context.Context) error { var errs []error if d.cfg.RetentionDays > 0 { for _, table := range []string{"service_rollup", "endpoint_rollup", "edge_rollup"} { - if _, err := d.DB.ExecContext(ctx, fmt.Sprintf("DELETE FROM %s WHERE bucket < now() - INTERVAL %d DAY", table, d.cfg.RetentionDays)); err != nil { + if _, err := d.writer().ExecContext(ctx, fmt.Sprintf("DELETE FROM %s WHERE bucket < now() - INTERVAL %d DAY", table, d.cfg.RetentionDays)); err != nil { errs = append(errs, fmt.Errorf("prune %s: %w", table, err)) } } } - _, checkpointErr := d.DB.ExecContext(ctx, "CHECKPOINT") - return errors.Join(errors.Join(errs...), checkpointErr) + return errors.Join(errors.Join(errs...), d.checkpoint(ctx)) }() err := errors.Join(pruneErr, cacheErr) maintenanceResult := metrics.TelemetrySuccess @@ -612,7 +713,7 @@ func (d *Duck) refreshServiceRollup(ctx context.Context) (int64, error) { } defer d.parquetMu.RUnlock() - tx, err := d.DB.BeginTx(ctx, nil) + tx, err := d.writer().BeginTx(ctx, nil) if err != nil { return 0, err } @@ -733,7 +834,7 @@ func (d *Duck) refreshEndpointRollup(ctx context.Context) (int64, error) { } defer d.parquetMu.RUnlock() - tx, err := d.DB.BeginTx(ctx, nil) + tx, err := d.writer().BeginTx(ctx, nil) if err != nil { return 0, err } @@ -856,7 +957,7 @@ func (d *Duck) refreshEdgeRollup(ctx context.Context) (int64, error) { } defer d.parquetMu.RUnlock() - tx, err := d.DB.BeginTx(ctx, nil) + tx, err := d.writer().BeginTx(ctx, nil) if err != nil { return 0, err } @@ -960,7 +1061,22 @@ WHERE ingested_unix_nano > ? maxT := maxStartT.Time processed := 0 for !subLo.After(maxT) { - if processed == maxEdgeSubWindowsPerPass { + // Stop at a resumable point when the budget is spent, and also when + // a queued publication is about to start refusing readers: this + // pass holds the Parquet snapshot until it commits, so from that + // moment every arriving query waits out whatever is left of it. + // + // The trigger is the grace running out, not a publisher existing. + // A publication queues every ten seconds under compaction, and + // yielding on sight would cut a pass from eight sub-windows to one + // whenever ingest is busy — which is when the eight are needed. + // While the grace holds, readers are still being admitted and + // nothing is waiting on this pass. + // + // One sub-window is the unit, and the cursor below is what makes + // stopping free. Never before the first, so a pass always makes + // progress and a busy compaction cycle cannot starve the rollup. + if processed == maxEdgeSubWindowsPerPass || (processed > 0 && d.parquetMu.publisherGraceLeft() <= edgeYieldMargin) { completed = false nextCursor = subLo.UnixNano() break @@ -1597,18 +1713,48 @@ FROM messaging_edges;` // ---- Read query helpers ---- +// acquireRead takes a connection from the read pool. The wait is already +// counted by the pool's own gauges; taking it explicitly keeps it out of the +// statement histogram. +func (d *Duck) acquireRead(ctx context.Context) (*sql.Conn, error) { + return d.DB.Conn(ctx) +} + // QueryContext executes a read against immutable Parquet files and DuckDB's // local rollup cache. func (d *Duck) QueryContext(ctx context.Context, query string, args ...any) (queryrows.Rows, error) { - if err := d.lockParquetRead(ctx); err != nil { + if err := d.lockParquetRead(ctx, readerQuery); err != nil { return nil, err } - rows, err := d.DB.QueryContext(ctx, query, args...) + // The connection is taken first and timed separately: DB.QueryContext would + // fold the wait for a free connection into the statement's duration, and + // telling queueing from execution is the whole reason this is measured. + conn, err := d.acquireRead(ctx) + if err != nil { + d.parquetMu.RUnlock() + return nil, err + } + started := time.Now() + rows, err := conn.QueryContext(ctx, query, args...) + metrics.DuckDBStatement.Observe(time.Since(started).Seconds()) if err != nil { + _ = conn.Close() d.parquetMu.RUnlock() return nil, err } - return &lockedRows{Rows: rows, unlock: d.parquetMu.RUnlock}, nil + release := func() { + // The connection goes back to the pool only once its rows are done + // with it, which is what closing the rows first guarantees. + // + // A caller that neither closes nor exhausts its rows now leaks the + // pool slot as well as the snapshot read-lock — database/sql used to + // return the connection when the context died, and an explicitly held + // one is the caller's until it says otherwise. Every call site closes + // on every path; this is what it costs if one stops. + _ = conn.Close() + d.parquetMu.RUnlock() + } + return &lockedRows{Rows: rows, unlock: release}, nil } type lockedRows struct { @@ -1636,16 +1782,26 @@ func (r *lockedRows) release() { r.unlockOnce.Do(r.unlock) } // QueryRowScan executes a single-row query against immutable Parquet files and // DuckDB's local rollup cache. func (d *Duck) QueryRowScan(ctx context.Context, dest []any, query string, args ...any) error { - if err := d.lockParquetRead(ctx); err != nil { + if err := d.lockParquetRead(ctx, readerQuery); err != nil { return err } defer d.parquetMu.RUnlock() - return d.DB.QueryRowContext(ctx, query, args...).Scan(dest...) + conn, err := d.acquireRead(ctx) + if err != nil { + return err + } + defer func() { _ = conn.Close() }() + // This one covers the scan as well as the statement: a single row cannot + // be fetched separately from being read. Both are DuckDB's time rather + // than the pool's, which is the distinction the histogram is drawing. + started := time.Now() + defer func() { metrics.DuckDBStatement.Observe(time.Since(started).Seconds()) }() + return conn.QueryRowContext(ctx, query, args...).Scan(dest...) } // Trace pins the immutable Parquet snapshot for the full indexed-file read. func (d *Duck) Trace(ctx context.Context, query telemetry.TraceQuery) ([]telemetry.IndexedSpan, error) { - if err := d.lockParquetRead(ctx); err != nil { + if err := d.lockParquetRead(ctx, readerQuery); err != nil { return nil, err } defer d.parquetMu.RUnlock() @@ -1673,22 +1829,39 @@ func (d *Duck) PublishParquet(ctx context.Context, publish func(context.Context) return fmt.Errorf("wait for Parquet readers: %w", err) } defer d.parquetMu.Unlock() + swapStarted := time.Now() + defer func() { metrics.ParquetPublishHold.Observe(time.Since(swapStarted).Seconds()) }() swapCtx, cancelSwap := context.WithTimeout(ctx, parquetSwapBudget) defer cancelSwap() return publish(swapCtx) } -func (d *Duck) lockParquetRead(ctx context.Context) error { +// Reader classes for the snapshot-wait metric. A query that waits and a rollup +// that waits are different problems with the same symptom, and the whole point +// of measuring here is to tell them apart. +const ( + readerQuery = "query" + readerRollup = "rollup" + readerStats = "stats" +) + +// lockParquetRead admits a reader to the Parquet snapshot and records what the +// wait cost. The publisher's wait was already measured and the reader's was +// not, which is why a multi-second query could not be attributed to anything. +func (d *Duck) lockParquetRead(ctx context.Context, reader string) error { + start := time.Now() if err := d.parquetMu.RLockContext(ctx); err != nil { + metrics.RecordParquetRead(reader, time.Since(start).Seconds(), false) return errors.Join(ErrParquetReadWait, err) } + metrics.RecordParquetRead(reader, time.Since(start).Seconds(), true) return nil } func (d *Duck) lockRollupParquetRead(ctx context.Context) error { waitCtx, cancel := context.WithTimeout(ctx, rollupReaderLease) defer cancel() - return d.lockParquetRead(waitCtx) + return d.lockParquetRead(waitCtx, readerRollup) } // ---- Queries for API ---- diff --git a/internal/query/parquet_gate.go b/internal/query/parquet_gate.go index 9945de1aa..2ec1cb7c0 100644 --- a/internal/query/parquet_gate.go +++ b/internal/query/parquet_gate.go @@ -80,6 +80,34 @@ func (g *parquetReadGate) admitsReaderLocked() bool { return g.clock().Sub(g.waiting[0].queuedAt) < g.grace() } +// publisherGraceLeft reports how long the oldest queued publication may still +// be kept waiting before the gate starts refusing new readers. +// +// A long-running internal reader asks between units of work, so it can stop at +// a resumable point rather than make every query behind it wait out the rest +// of its pass. It is the remaining grace and not the mere presence of a +// publisher that matters: while the grace holds, readers are still being +// admitted and nothing is blocked, so yielding then would give up throughput +// to relieve congestion that does not exist yet. +// +// Zero means a publication is holding the gate, or has waited out its grace +// and new readers are already queuing. With nothing queued, the full grace. +func (g *parquetReadGate) publisherGraceLeft() time.Duration { + g.init() + g.mu.Lock() + defer g.mu.Unlock() + if g.writer { + return 0 + } + if len(g.waiting) == 0 { + return g.grace() + } + if left := g.grace() - g.clock().Sub(g.waiting[0].queuedAt); left > 0 { + return left + } + return 0 +} + func (g *parquetReadGate) TryRLock() bool { g.init() g.mu.Lock() diff --git a/internal/query/rollup_yield_test.go b/internal/query/rollup_yield_test.go new file mode 100644 index 000000000..f85530783 --- /dev/null +++ b/internal/query/rollup_yield_test.go @@ -0,0 +1,234 @@ +package query + +import ( + "context" + "fmt" + "sync/atomic" + "testing" + "time" + + "github.com/labstack/fanout/internal/config" +) + +// seedEdgeSpread writes SERVER/CLIENT pairs that share one ingested instant but +// spread their start_time over spreadMins, which is what forces the edge rollup +// to sub-window: a seed load or a backfill compresses a wide start_time range +// into a narrow ingested window. +func seedEdgeSpread(t *testing.T, d *Duck, rows, spreadMins int) { + t.Helper() + ctx := context.Background() + baseTime := time.Now().UTC().Truncate(time.Minute) + for _, spec := range []struct{ kind, span, parent, service string }{ + {"SPAN_KIND_SERVER", "parent-%d", "", "svc-a"}, + {"SPAN_KIND_CLIENT", "child-%d", "parent-%d", "svc-b"}, + } { + parent := "NULL" + if spec.parent != "" { + parent = fmt.Sprintf("printf('%s', i)", spec.parent) + } + _, err := d.DB.ExecContext(ctx, fmt.Sprintf(` +WITH input AS (SELECT CAST(? AS TIMESTAMP) AS base_time) +INSERT INTO telemetry.spans ( + namespace, trace_id, span_id, parent_span_id, service, operation, kind, + start_time, end_time, start_unix_nano, end_unix_nano, duration_ms, + status, ingested_at, ingested_unix_nano +) +SELECT 'default', printf('trace-%%d', i %% 2000), printf('%s', i), %s, '%s', 'op', '%s', + base_time - ((i %% %d) * INTERVAL '1' MINUTE), + base_time - ((i %% %d) * INTERVAL '1' MINUTE) + INTERVAL '10' MILLISECOND, + epoch_ns(base_time - ((i %% %d) * INTERVAL '1' MINUTE)), + epoch_ns(base_time - ((i %% %d) * INTERVAL '1' MINUTE)) + 10000000, + 10.0, 'STATUS_CODE_OK', base_time, ? +FROM range(?, ?) t(i), input`, + spec.span, parent, spec.service, spec.kind, spreadMins, spreadMins, spreadMins, spreadMins), + baseTime, baseTime.UnixNano(), 0, rows) + if err != nil { + t.Fatalf("insert %s: %v", spec.kind, err) + } + } +} + +func newSpreadDuck(t *testing.T, rows, spreadMins int) *Duck { + t.Helper() + db := openTestDuck(t) + if err := CreateTables(db); err != nil { + t.Fatalf("CreateTables: %v", err) + } + if err := CreateViews(db); err != nil { + t.Fatalf("CreateViews: %v", err) + } + d := &Duck{DB: db, cfg: config.Config{RetentionDays: 30, DuckDBMemory: "1GB"}} + seedEdgeSpread(t, d, rows, spreadMins) + return d +} + +// A rollup pass holds the Parquet snapshot until it commits, so a publication +// whose grace has run out makes every arriving query wait out the rest of that +// pass — up to eight sub-windows of work for a request that needed none of it. +// The pass now stops at the first sub-window boundary once that happens, which +// costs nothing to arrange: the resume cursor already exists for the per-pass +// budget, and the ingested watermark already stays behind an unfinished window. +func TestEdgeRollupYieldsToAQueuedPublisher(t *testing.T) { + const ( + rows = 6_000 + spreadMins = 300 + ) + ctx := context.Background() + + undisturbed := newSpreadDuck(t, rows, spreadMins) + if _, err := undisturbed.refreshEdgeRollup(ctx); err != nil { + t.Fatalf("undisturbed pass: %v", err) + } + _, freeCursor := readEdgeRollupState(t, undisturbed) + if freeCursor == 0 { + t.Fatal("the undisturbed pass consumed the whole window; this seed no longer exercises sub-windowing") + } + + yielding := newSpreadDuck(t, rows, spreadMins) + // A publisher can only queue while something is reading, so the test holds + // a reader of its own. The gate's clock is the test's: the pass is admitted + // with the grace intact, and the clock jumps past it once the pass is + // under way — which is the state the rollup yields for, and the only one + // in which a query is actually blocked behind it. + spend := spendGraceOnCue(&yielding.parquetMu) + if err := yielding.parquetMu.RLockContext(ctx); err != nil { + t.Fatalf("hold a reader: %v", err) + } + queued := queuePublisher(t, yielding) + spendOnceAdmitted(&yielding.parquetMu, spend) + + if _, err := yielding.refreshEdgeRollup(ctx); err != nil { + t.Fatalf("yielding pass: %v", err) + } + watermark, yieldCursor := readEdgeRollupState(t, yielding) + if yieldCursor == 0 { + t.Fatal("the pass ran the whole window with a publisher waiting on it") + } + if watermark != 0 { + t.Fatalf("ingested watermark advanced to %d over a window the pass did not finish", watermark) + } + if yieldCursor >= freeCursor { + t.Fatalf("yielding pass reached cursor %d, no earlier than the undisturbed %d", yieldCursor, freeCursor) + } + + // Yielding must not stall the rollup: with the publisher gone the same + // instance finishes the window. + yielding.parquetMu.RUnlock() + queued.done() + for passes := 0; ; passes++ { + if passes > spreadMins/30+4 { + t.Fatal("edge rollup did not converge after yielding") + } + if _, err := yielding.refreshEdgeRollup(ctx); err != nil { + t.Fatalf("catch-up pass: %v", err) + } + if _, cursor := readEdgeRollupState(t, yielding); cursor == 0 { + break + } + } + if watermark, _ = readEdgeRollupState(t, yielding); watermark == 0 { + t.Fatal("watermark never advanced after the window completed") + } +} + +// A publisher that is always queued must not stop the rollup dead: every pass +// has to complete at least one sub-window, or a busy compaction cycle starves +// the cache the whole product reads from. +func TestEdgeRollupAlwaysCompletesOneSubWindow(t *testing.T) { + ctx := context.Background() + d := newSpreadDuck(t, 4_000, 180) + spend := spendGraceOnCue(&d.parquetMu) + if err := d.parquetMu.RLockContext(ctx); err != nil { + t.Fatalf("hold a reader: %v", err) + } + queued := queuePublisher(t, d) + // Overdue from the pass's very first boundary: it must still complete one + // sub-window, or a busy compaction cycle starves the cache. + spendOnceAdmitted(&d.parquetMu, spend) + + rows, err := d.refreshEdgeRollup(ctx) + if err != nil { + t.Fatalf("pass with a publisher queued: %v", err) + } + if rows == 0 { + t.Fatal("a pass yielded before doing any work; a busy compaction cycle would starve the cache") + } + d.parquetMu.RUnlock() + queued.done() +} + +// queuePublisher parks a publication in the gate's wait list and returns a +// handle that releases it. It requires a reader to already hold the snapshot, +// which is the only state in which a publisher waits rather than enters. +func queuePublisher(t *testing.T, d *Duck) (handle struct{ done func() }) { + t.Helper() + entered := make(chan struct{}) + release := make(chan struct{}) + go func() { + if err := d.parquetMu.LockContext(context.Background()); err != nil { + close(entered) + return + } + close(entered) + <-release + d.parquetMu.Unlock() + }() + deadline := time.Now().Add(2 * time.Second) + for queuedPublishers(&d.parquetMu) == 0 { + if time.Now().After(deadline) { + t.Fatal("publisher never queued") + } + time.Sleep(time.Millisecond) + } + handle.done = func() { + <-entered + close(release) + } + return handle +} + +// spendGraceOnCue replaces the gate's clock with one the test moves. Until the +// returned function is called the clock stands still, so a queued publication +// keeps its whole grace and readers are admitted; afterwards the clock is an +// hour later, so the grace is spent and every new reader would queue. +func spendGraceOnCue(gate *parquetReadGate) (spend func()) { + base := time.Now() + var spent atomic.Bool + gate.now = func() time.Time { + if spent.Load() { + return base.Add(time.Hour) + } + return base + } + return func() { spent.Store(true) } +} + +// queuedPublishers reads the gate's wait list. The grace-based predicate the +// rollup uses cannot serve here: a publication that has only just queued still +// has its whole grace, which is exactly the state this waits to observe. +func queuedPublishers(gate *parquetReadGate) int { + gate.init() + gate.mu.Lock() + defer gate.mu.Unlock() + return len(gate.waiting) +} + +// spendOnceAdmitted spends the queued publication's grace the moment the rollup +// has entered the snapshot, so the pass is admitted with the grace intact and +// finds it gone at its first sub-window boundary. Waiting on the reader count +// rather than on a duration keeps that ordering exact. +func spendOnceAdmitted(gate *parquetReadGate, spend func()) { + go func() { + for readers(gate) < 2 { + time.Sleep(time.Millisecond) + } + spend() + }() +} + +func readers(gate *parquetReadGate) int { + gate.init() + gate.mu.Lock() + defer gate.mu.Unlock() + return gate.readers +} diff --git a/internal/query/stall_instrumentation_test.go b/internal/query/stall_instrumentation_test.go new file mode 100644 index 000000000..c4eb76682 --- /dev/null +++ b/internal/query/stall_instrumentation_test.go @@ -0,0 +1,297 @@ +package query + +import ( + "context" + "fmt" + "sync" + "testing" + "time" + + "github.com/labstack/fanout/internal/config" + "github.com/labstack/fanout/internal/metrics" + telemetrystore "github.com/labstack/fanout/internal/telemetry/store" + "github.com/prometheus/client_golang/prometheus/testutil" +) + +// A rollup or a maintenance pass used to take its connection from the same +// pool every read draws from. On a four-core host that pool holds four +// connections, so one background pass occupied a quarter of the machine's read +// capacity for its whole duration — and a request that waits for a connection +// waits invisibly. +func TestWritesDoNotDrawFromTheReadPool(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + cfg := config.Config{DataDir: t.TempDir(), RollupInterval: time.Minute, DuckDBMemory: "128MB", DuckDBMaxConns: 4} + repository, err := telemetrystore.Open(cfg.TelemetryDir()) + if err != nil { + t.Fatalf("open telemetry repository: %v", err) + } + defer repository.Close() + d, err := NewDuck(ctx, cfg, repository) + if err != nil { + t.Fatalf("NewDuck: %v", err) + } + defer d.Close() + + if got := d.DB.Stats().MaxOpenConnections; got != 4 { + t.Fatalf("read pool MaxOpenConnections = %d, want 4", got) + } + if got := d.writeDB.Stats().MaxOpenConnections; got != 1 { + t.Fatalf("write handle MaxOpenConnections = %d, want 1 — writes are serialized by the gate anyway", got) + } + if d.writer() != d.writeDB { + t.Fatal("writes are not going through the write handle") + } + + // Use the read pool first, so its idle count is non-zero and the assertion + // below is about where the write went rather than about a pool nobody has + // touched. + if err := d.DB.QueryRowContext(ctx, "SELECT 1").Scan(new(int)); err != nil { + t.Fatalf("read: %v", err) + } + if idle := d.DB.Stats().Idle; idle == 0 { + t.Fatal("the read pool holds no connection after a read; the assertion below would prove nothing") + } + + // Hold a write transaction open and prove the read pool is untouched. + tx, err := d.writer().BeginTx(ctx, nil) + if err != nil { + t.Fatalf("begin write: %v", err) + } + defer func() { _ = tx.Rollback() }() + if _, err := tx.ExecContext(ctx, "CREATE OR REPLACE TABLE stall_probe (n INTEGER)"); err != nil { + t.Fatalf("write inside transaction: %v", err) + } + if inUse := d.DB.Stats().InUse; inUse != 0 { + t.Fatalf("read pool has %d connections in use while a write is open", inUse) + } + if inUse := d.writeDB.Stats().InUse; inUse != 1 { + t.Fatalf("write handle has %d connections in use, want 1", inUse) + } + + // The read pool is the one worth watching, so it is the one the gauges read. + if got := metrics.DuckDBPoolStatsForTest().MaxOpenConnections; got != 4 { + t.Fatalf("pool gauges read a pool with MaxOpenConnections = %d, want the read pool's 4", got) + } +} + +// A stalled query asks one question — how long did I wait to enter the +// snapshot? — and until this metric existed the answer was unavailable, which +// is why the first diagnosis of the stalls named a lock that does not exist. +func TestSnapshotWaitIsMeasuredForReaders(t *testing.T) { + // Label values of this test's own, so the counts are this test's alone + // however the package's tests are ordered. + admitted := fmt.Sprintf("test-admitted-%d", time.Now().UnixNano()) + refused := fmt.Sprintf("test-refused-%d", time.Now().UnixNano()) + d := &Duck{} + before := testutil.CollectAndCount(metrics.ParquetReadWaitForTest()) + if err := d.lockParquetRead(context.Background(), admitted); err != nil { + t.Fatalf("lockParquetRead: %v", err) + } + d.parquetMu.RUnlock() + admittedCount := testutil.CollectAndCount(metrics.ParquetReadWaitForTest()) + if admittedCount != before+1 { + t.Fatalf("an admitted reader added %d wait series, want 1", admittedCount-before) + } + + // A reader that gives up is counted separately: a refusal and a long wait + // are different failures. + if err := d.parquetMu.LockContext(context.Background()); err != nil { + t.Fatalf("take the publisher side: %v", err) + } + defer d.parquetMu.Unlock() + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Millisecond) + defer cancel() + if err := d.lockParquetRead(ctx, refused); err == nil { + t.Fatal("reader entered a snapshot held by a publisher") + } + if refusals := testutil.ToFloat64(metrics.ParquetReadRefusalsForTest(refused)); refusals != 1 { + t.Fatalf("refusals = %v, want 1", refusals) + } + // A refusal is not also counted as a wait: they are different failures. + if series := testutil.CollectAndCount(metrics.ParquetReadWaitForTest()); series != admittedCount { + t.Fatalf("a refused reader added %d wait series, want 0", series-admittedCount) + } +} + +func TestPublisherGraceLeftTracksAWaitingSwap(t *testing.T) { + gate := parquetReadGate{writerGrace: time.Second} + if left := gate.publisherGraceLeft(); left != time.Second { + t.Fatalf("an idle gate reports %v of grace left, want the whole %v", left, time.Second) + } + if err := gate.RLockContext(context.Background()); err != nil { + t.Fatalf("RLockContext: %v", err) + } + queued := make(chan struct{}) + go func() { + close(queued) + if err := gate.LockContext(context.Background()); err == nil { + gate.Unlock() + } + }() + <-queued + deadline := time.Now().Add(time.Second) + for gate.publisherGraceLeft() == time.Second { + if time.Now().After(deadline) { + t.Fatal("a publisher waiting for readers did not start consuming its grace") + } + time.Sleep(time.Millisecond) + } + // The grace runs down rather than vanishing: a reader admitted now is not + // blocking anything yet, which is what stops the rollup yielding on sight. + if left := gate.publisherGraceLeft(); left <= 0 || left > time.Second { + t.Fatalf("grace left = %v, want a positive remainder under %v", left, time.Second) + } + deadline = time.Now().Add(3 * time.Second) + for gate.publisherGraceLeft() > 0 { + if time.Now().After(deadline) { + t.Fatal("grace never ran out") + } + time.Sleep(5 * time.Millisecond) + } + gate.RUnlock() +} + +// database/sql closes a connector that implements io.Closer when its DB is +// closed, and duckdb's connector closes the instance behind an unsynchronized +// bool. Two handles over one connector therefore meant two duckdb_close calls +// on one instance — a double free that kills the process rather than returning +// an error. The write handle borrows the instance; the read pool owns it. +func TestClosingTheWriteHandleLeavesTheInstanceAlive(t *testing.T) { + ctx := context.Background() + cfg := config.Config{DataDir: t.TempDir(), RollupInterval: time.Minute, DuckDBMemory: "128MB"} + repository, err := telemetrystore.Open(cfg.TelemetryDir()) + if err != nil { + t.Fatalf("open telemetry repository: %v", err) + } + defer repository.Close() + d, err := NewDuck(ctx, cfg, repository) + if err != nil { + t.Fatalf("NewDuck: %v", err) + } + + if err := d.writeDB.Close(); err != nil { + t.Fatalf("close write handle: %v", err) + } + var one int + if err := d.DB.QueryRowContext(ctx, "SELECT 1").Scan(&one); err != nil { + t.Fatalf("read pool died with the write handle: %v", err) + } + + // Close is idempotent, including under a second shutdown path arriving at + // the same time. + var wg sync.WaitGroup + for range 8 { + wg.Add(1) + go func() { + defer wg.Done() + _ = d.Close() + }() + } + wg.Wait() + if err := d.Close(); err != nil { + t.Fatalf("closing an already-closed Duck: %v", err) + } +} + +// Installing a gauge source is last-caller-wins; removing one is not. A Duck +// that closes after another has taken over must clear nothing, or the gauges +// go blank while a pool is still serving. +func TestClosingADisplacedDuckLeavesTheGaugeSourceAlone(t *testing.T) { + ctx := context.Background() + open := func(conns int) *Duck { + t.Helper() + cfg := config.Config{DataDir: t.TempDir(), RollupInterval: time.Minute, DuckDBMemory: "128MB", DuckDBMaxConns: conns} + repository, err := telemetrystore.Open(cfg.TelemetryDir()) + if err != nil { + t.Fatalf("open telemetry repository: %v", err) + } + t.Cleanup(func() { _ = repository.Close() }) + d, err := NewDuck(ctx, cfg, repository) + if err != nil { + t.Fatalf("NewDuck: %v", err) + } + return d + } + first := open(2) + second := open(3) + defer second.Close() + // second installed itself over first. first closing must leave it alone. + if err := first.Close(); err != nil { + t.Fatalf("close first: %v", err) + } + if got := metrics.DuckDBPoolStatsForTest().MaxOpenConnections; got != 3 { + t.Fatalf("pool gauges read MaxOpenConnections = %d after a displaced Duck closed, want the live pool's 3", got) + } +} + +// The statement histogram exists to separate execution from queueing, so the +// wait for a pool connection must not be inside it. DB.QueryContext would fold +// the two together; the connection is taken first for that reason. +func TestStatementTimerExcludesTheWaitForAConnection(t *testing.T) { + ctx := context.Background() + cfg := config.Config{DataDir: t.TempDir(), RollupInterval: time.Minute, DuckDBMemory: "128MB", DuckDBMaxConns: 1} + repository, err := telemetrystore.Open(cfg.TelemetryDir()) + if err != nil { + t.Fatalf("open telemetry repository: %v", err) + } + defer repository.Close() + d, err := NewDuck(ctx, cfg, repository) + if err != nil { + t.Fatalf("NewDuck: %v", err) + } + defer d.Close() + + // Hold the pool's only connection, so a second reader must wait for it. + held, err := d.DB.Conn(ctx) + if err != nil { + t.Fatalf("hold the connection: %v", err) + } + waited := make(chan time.Duration, 1) + go func() { + started := time.Now() + rows, err := d.QueryContext(ctx, "SELECT 1") + if err != nil { + waited <- -1 + return + } + _ = rows.Close() + waited <- time.Since(started) + }() + + // Long enough that a timer covering the wait could not miss it. + time.Sleep(250 * time.Millisecond) + before := statementSeconds(t) + if err := held.Close(); err != nil { + t.Fatalf("release the connection: %v", err) + } + total := <-waited + if total < 200*time.Millisecond { + t.Fatalf("the second reader waited %s; the test did not create contention", total) + } + observed := statementSeconds(t) - before + if observed <= 0 { + t.Fatal("no statement duration was recorded") + } + if observed > 0.2 { + t.Fatalf("statement recorded %.3fs of a %s call: the wait for a connection is inside the timer", observed, total) + } +} + +// statementSeconds reports the histogram's running total. +func statementSeconds(t *testing.T) float64 { + t.Helper() + return metrics.DuckDBStatementSecondsForTest() +} + +// Close is reached from error paths and from test cleanup, where a Duck may +// hold no handles at all. It has nothing to do there, rather than a panic. +func TestClosingADuckWithNoHandlesIsHarmless(t *testing.T) { + var empty Duck + if err := empty.Close(); err != nil { + t.Fatalf("closing a Duck with no handles: %v", err) + } + if err := empty.Close(); err != nil { + t.Fatalf("closing it again: %v", err) + } +}