From 2fe8ec69e1c057fa71f9202e16ff9a73f10ed3db Mon Sep 17 00:00:00 2001 From: Vishal Rana Date: Fri, 16 Oct 2026 08:58:23 -0700 Subject: [PATCH 1/3] fix(metrics): never block a /metrics scrape on DuckDB An 8-vCPU benchmark aborted when a /metrics scrape exceeded the harness's 15 s timeout. Most scrapes took about 4 ms, but 21 took over a second and the slowest took 15.9 s. Each scrape queried duckdb_memory() on the read pool. Its 5 s context did not bound the wait, because the query waited inside the engine behind long writes: one 15.5 s scrape returned at the instant a version-rollup INSERT was interrupted. - A background sampler reads DuckDB memory every 5 s with its own bounded context, and scrapes serve the last complete sample. fanout_duckdb_memory_last_success_timestamp_seconds shows when that sample is stale, and repeated failures are logged once per streak. - Pool statistics and the compaction backlog are read from cached snapshots, so no scrape waits on a pool, a lock or the engine. Scrapes now return in 0.2-0.6 ms while DuckDB is blocked. Claude-Session: https://claude.ai/code/session_01XA52nWyQ3R5f5kK8kjpAQW --- internal/metrics/duckdb_memory_test.go | 85 +++++++----- internal/metrics/duckdb_pool_test.go | 80 +++++++++++ internal/metrics/metrics.go | 133 ++++++++++++------ internal/query/duck.go | 41 ++---- internal/query/duck_memory.go | 78 +++++++++++ internal/query/duck_memory_test.go | 154 +++++++++++++++++++++ internal/telemetry/compaction_debt.go | 8 +- internal/telemetry/compaction_debt_test.go | 31 +++++ internal/telemetry/parquet.go | 11 +- 9 files changed, 511 insertions(+), 110 deletions(-) create mode 100644 internal/metrics/duckdb_pool_test.go create mode 100644 internal/query/duck_memory.go create mode 100644 internal/query/duck_memory_test.go diff --git a/internal/metrics/duckdb_memory_test.go b/internal/metrics/duckdb_memory_test.go index 4ac7ea5b5..034c3b159 100644 --- a/internal/metrics/duckdb_memory_test.go +++ b/internal/metrics/duckdb_memory_test.go @@ -1,12 +1,11 @@ package metrics import ( - "strings" "sync" "testing" + "time" "github.com/prometheus/client_golang/prometheus" - dto "github.com/prometheus/client_model/go" ) // gatherDuckDBMemory returns the exported per-tag memory series, by tag. @@ -32,42 +31,57 @@ func gatherDuckDBMemory(t *testing.T) map[string]float64 { return out } -// The whole purpose of this series is to measure what DuckDB holds right now, -// so that the untracked gap against process RSS can be derived from it. A -// failed read must therefore export nothing. Reporting the last successful -// figures as if they were current is the one outcome that actively misleads: -// the gap is computed against a stale sum, and the number that was supposed to -// reveal an impending out-of-memory kill reads normal. -func TestDuckDBMemoryDropsSeriesWhenTheReadFails(t *testing.T) { - release := SetDuckDBMemorySource(func() map[string]int64 { - return map[string]int64{"BASE_TABLE": 4096} - }) +func TestDuckDBMemoryPublishesCompleteSamplesAndTimestamp(t *testing.T) { + record, release := SetDuckDBMemorySource() + defer release() + if got := gatherDuckDBMemory(t); len(got) != 0 { + t.Fatalf("before the first sample exported %v", got) + } + if got := gatherMemoryTimestamp(t); got != 0 { + t.Fatalf("before the first sample timestamp=%g", got) + } + tags := map[string]int64{"BASE_TABLE": 4096} + record(tags, time.Unix(1234, 500_000_000)) + tags["BASE_TABLE"] = 0 // Publication owns an immutable copy. if got := gatherDuckDBMemory(t); got["BASE_TABLE"] != 4096 { - release() t.Fatalf("healthy read exported %v, want BASE_TABLE=4096", got) } - release() + if got := gatherMemoryTimestamp(t); got != 1234.5 { + t.Fatalf("timestamp=%g, want 1234.5", got) + } + record(map[string]int64{"HASH_TABLE": 2048}, time.Unix(1235, 0)) + if got := gatherDuckDBMemory(t); len(got) != 1 || got["HASH_TABLE"] != 2048 { + t.Fatalf("replacement sample exported %v", got) + } +} - // The source is installed but cannot read: the closure in internal/query - // returns nil for a query error, a scan error, or a partial read. - release = SetDuckDBMemorySource(func() map[string]int64 { return nil }) +func TestDuckDBMemoryReleaseOnlyClearsItsOwner(t *testing.T) { + recordOld, releaseOld := SetDuckDBMemorySource() + record, release := SetDuckDBMemorySource() defer release() + record(map[string]int64{"BASE_TABLE": 2}, time.Now()) + releaseOld() + recordOld(map[string]int64{"BASE_TABLE": 1}, time.Now()) + if got := gatherDuckDBMemory(t); got["BASE_TABLE"] != 2 { + t.Fatalf("old owner changed the current sample: %v", got) + } + release() if got := gatherDuckDBMemory(t); len(got) != 0 { - t.Errorf("after a failed read the gauge still exports %v; stale values are reported as current", got) + t.Fatalf("released source still exports %v", got) } } -// Prometheus scrapes concurrently, and a GaugeVec that is Reset and refilled on -// every scrape is shared mutable state: two scrapes interleave, and one of them -// observes the other's half-finished refill -- missing tags, and a sum that -// understates what DuckDB holds. func TestDuckDBMemoryIsSafeUnderConcurrentScrapes(t *testing.T) { - release := SetDuckDBMemorySource(func() map[string]int64 { - return map[string]int64{"BASE_TABLE": 1, "HASH_TABLE": 2, "ORDER_BY": 3} - }) + record, release := SetDuckDBMemorySource() defer release() + record(map[string]int64{"BASE_TABLE": 1, "HASH_TABLE": 2, "ORDER_BY": 3}, time.Now()) var wg sync.WaitGroup + wg.Go(func() { + for range 200 { + record(map[string]int64{"BASE_TABLE": 1, "HASH_TABLE": 2, "ORDER_BY": 3}, time.Now()) + } + }) for range 8 { wg.Add(1) go func() { @@ -86,7 +100,6 @@ func TestDuckDBMemoryIsSafeUnderConcurrentScrapes(t *testing.T) { t.Errorf("scrape saw %d tags, want 3: a concurrent refill was observed half-done", n) return } - _ = dto.MetricType_GAUGE } } }() @@ -100,11 +113,21 @@ func TestDuckDBMemoryExportsNothingWithoutASource(t *testing.T) { if got := gatherDuckDBMemory(t); len(got) != 0 { t.Errorf("with no source installed the gauge exports %v", got) } - if families, err := prometheus.DefaultGatherer.Gather(); err == nil { - for _, family := range families { - if strings.Contains(family.GetName(), "duckdb_memory_scrape") { - t.Errorf("%s exists only to pump a push gauge; a collector does not need it", family.GetName()) - } + if got := gatherMemoryTimestamp(t); got != -1 { + t.Errorf("without a source timestamp=%g, want absent", got) + } +} + +func gatherMemoryTimestamp(t *testing.T) float64 { + t.Helper() + families, err := prometheus.DefaultGatherer.Gather() + if err != nil { + t.Fatal(err) + } + for _, family := range families { + if family.GetName() == "fanout_duckdb_memory_last_success_timestamp_seconds" { + return family.Metric[0].GetGauge().GetValue() } } + return -1 } diff --git a/internal/metrics/duckdb_pool_test.go b/internal/metrics/duckdb_pool_test.go new file mode 100644 index 000000000..6ee958272 --- /dev/null +++ b/internal/metrics/duckdb_pool_test.go @@ -0,0 +1,80 @@ +package metrics + +import ( + "database/sql" + "net/http" + "net/http/httptest" + "strings" + "testing" + "time" + + "github.com/prometheus/client_golang/prometheus/promhttp" +) + +func TestMetricsScrapeKeepsCachedPoolStatsWhileSamplerBlocks(t *testing.T) { + blocked, unblock := make(chan struct{}), make(chan struct{}) + calls := 0 + release := setDuckDBPoolSource(func() sql.DBStats { + calls++ + if calls == 2 { + close(blocked) + <-unblock + } + return sql.DBStats{InUse: 1, Idle: 2, MaxOpenConnections: 3, WaitCount: 4, WaitDuration: 5 * time.Second} + }, time.Millisecond) + t.Cleanup(func() { + close(unblock) + release() + release() // Release is idempotent. + }) + select { + case <-blocked: + case <-time.After(5 * time.Second): + t.Fatal("pool sampler did not reach the blocking call") + } + recorder := httptest.NewRecorder() + done := make(chan struct{}) + handler := promhttp.Handler() + go func() { + handler.ServeHTTP(recorder, httptest.NewRequest(http.MethodGet, "/metrics", nil)) + close(done) + }() + select { + case <-done: + case <-time.After(100 * time.Millisecond): + t.Fatal("/metrics waited for pool bookkeeping") + } + if recorder.Code != http.StatusOK { + t.Fatalf("/metrics status=%d", recorder.Code) + } + for _, series := range []string{ + "fanout_duckdb_pool_in_use 1\n", + "fanout_duckdb_pool_idle 2\n", + "fanout_duckdb_pool_max_open 3\n", + "fanout_duckdb_pool_waits_total 4\n", + "fanout_duckdb_pool_wait_seconds_total 5\n", + } { + if !strings.Contains(recorder.Body.String(), series) { + t.Fatalf("missing cached series %q", series) + } + } +} + +func TestDuckDBPoolReleaseOnlyClearsItsOwner(t *testing.T) { + releaseOld := setDuckDBPoolSource(func() sql.DBStats { return sql.DBStats{InUse: 1} }, time.Hour) + release := setDuckDBPoolSource(func() sql.DBStats { return sql.DBStats{InUse: 2} }, time.Hour) + defer release() + releaseOld() + deadline := time.After(5 * time.Second) + for poolStats().InUse != 2 { + select { + case <-deadline: + t.Fatal("new owner's pool stats were cleared") + case <-time.After(time.Millisecond): + } + } + release() + if got := poolStats(); got != (sql.DBStats{}) { + t.Fatalf("released pool retained stats: %+v", got) + } +} diff --git a/internal/metrics/metrics.go b/internal/metrics/metrics.go index f411b98f9..70add7368 100644 --- a/internal/metrics/metrics.go +++ b/internal/metrics/metrics.go @@ -2,8 +2,11 @@ package metrics import ( "database/sql" + "maps" "math" + "sync" "sync/atomic" + "time" "github.com/prometheus/client_golang/prometheus" "github.com/prometheus/client_golang/prometheus/promauto" @@ -312,15 +315,26 @@ func RecordFlush(signal string, durationSec float64) { FlushDuration.WithLabelValues(signal).Observe(durationSec) } -// duckDBPoolStats is the live source for the connection-pool gauges below. +type duckDBPoolSource struct{ sample atomic.Pointer[sql.DBStats] } + +// duckDBPoolStats is the cached source for the connection-pool gauges below. // // A holder rather than a promauto.NewGaugeFunc closure over the pool: the // gauges are declared once for the process, while a *Duck is constructed many // times in a test binary, and registering a collector per construction panics // on the second one. Unset, every gauge reads zero. -var duckDBPoolStats atomic.Pointer[func() sql.DBStats] +var duckDBPoolStats atomic.Pointer[duckDBPoolSource] + +type duckDBMemorySample struct { + tags map[string]int64 + at time.Time +} + +type duckDBMemorySource struct { + sample atomic.Pointer[duckDBMemorySample] +} -// duckDBMemoryStats is the live source for the per-tag DuckDB memory gauge. +// duckDBMemoryStats holds the latest background sample for the memory gauge. // // memory_limit bounds DuckDB's buffer pool and nothing else, so the figure that // actually predicts an out-of-memory kill is the gap between what the process @@ -329,72 +343,109 @@ var duckDBPoolStats atomic.Pointer[func() sql.DBStats] // - sum(fanout_duckdb_memory_bytes) - go_memstats_heap_sys_bytes. Without it // every diagnosis of this process is an inference, which is how a whole day // got spent optimising a Go heap that was never the problem. -var duckDBMemoryStats atomic.Pointer[func() map[string]int64] +var duckDBMemoryStats atomic.Pointer[duckDBMemorySource] -// SetDuckDBMemorySource installs the reader for the per-tag memory gauge. -func SetDuckDBMemorySource(tags func() map[string]int64) (release func()) { - if tags == nil { - return func() {} - } - installed := &tags +// SetDuckDBMemorySource installs a cache, never a scrape-time query. Record +// publishes a complete successful sample; failures leave the last one intact. +// Releasing an older Duck cannot clear a newer Duck's source. +func SetDuckDBMemorySource() (record func(map[string]int64, time.Time), release func()) { + installed := &duckDBMemorySource{} duckDBMemoryStats.Store(installed) - return func() { duckDBMemoryStats.CompareAndSwap(installed, nil) } + return func(tags map[string]int64, at time.Time) { + installed.sample.Store(&duckDBMemorySample{tags: maps.Clone(tags), at: at}) + }, func() { duckDBMemoryStats.CompareAndSwap(installed, nil) } } -// duckDBMemoryDesc is emitted by a Collector, not held in a GaugeVec. -// -// The obvious implementation -- a GaugeVec that a scrape hook Resets and -// refills -- is shared mutable state across concurrent scrapes, and a scrape -// that lands mid-refill exports a subset of the tags. That understates what -// DuckDB holds and correspondingly overstates the untracked gap, which is the -// single number this series exists to compute. It also leaves the last good -// values in place when a read fails, reporting them as current: the reading -// that was meant to reveal an impending out-of-memory kill then looks normal. -// -// A Collector has neither problem. Each scrape builds its own metrics from its -// own read, and a read that returns nothing exports nothing -- the series goes -// absent, which is visibly different from "DuckDB is holding nothing". +// Both series come from one immutable sample, so concurrent scrapes never see +// a partial tag set or a timestamp belonging to a different sample. var duckDBMemoryDesc = prometheus.NewDesc( "fanout_duckdb_memory_bytes", - "DuckDB memory usage by tag, as DuckDB itself accounts for it", + "Last sampled DuckDB memory usage by tag, as DuckDB itself accounts for it", []string{"tag"}, nil, ) +var duckDBMemorySampleDesc = prometheus.NewDesc( + "fanout_duckdb_memory_last_success_timestamp_seconds", + "Unix timestamp of the last successful DuckDB memory sample, or zero before the first success", + nil, nil, +) + type duckDBMemoryCollector struct{} -func (duckDBMemoryCollector) Describe(ch chan<- *prometheus.Desc) { ch <- duckDBMemoryDesc } +func (duckDBMemoryCollector) Describe(ch chan<- *prometheus.Desc) { + ch <- duckDBMemoryDesc + ch <- duckDBMemorySampleDesc +} func (duckDBMemoryCollector) Collect(ch chan<- prometheus.Metric) { - fn := duckDBMemoryStats.Load() - if fn == nil { + source := duckDBMemoryStats.Load() + if source == nil { return } - for tag, bytes := range (*fn)() { - ch <- prometheus.MustNewConstMetric(duckDBMemoryDesc, prometheus.GaugeValue, float64(bytes), tag) + at := 0.0 + if sample := source.sample.Load(); sample != nil { + at = float64(sample.at.UnixNano()) / 1e9 + for tag, bytes := range sample.tags { + ch <- prometheus.MustNewConstMetric(duckDBMemoryDesc, prometheus.GaugeValue, float64(bytes), tag) + } } + ch <- prometheus.MustNewConstMetric(duckDBMemorySampleDesc, prometheus.GaugeValue, at) } func init() { prometheus.DefaultRegisterer.MustRegister(duckDBMemoryCollector{}) } -// SetDuckDBPoolSource points the connection-pool gauges at a pool and returns -// the function that stops reading it. The last caller wins; the returned -// release only clears the source if it is still the one it installed, so a -// second Duck closing in a test binary cannot blank the gauges of one that is -// still serving. +// SetDuckDBPoolSource samples pool bookkeeping independently of memory queries, +// so a stuck engine cannot hide pool starvation. Stats takes database/sql's +// mutex, which some connection cleanup paths hold during driver calls; scrapes +// therefore read only the cached result. Release stops and joins this sampler +// before the owning Duck closes its handles, without clearing a newer owner. func SetDuckDBPoolSource(stats func() sql.DBStats) (release func()) { + return setDuckDBPoolSource(stats, 5*time.Second) +} + +func setDuckDBPoolSource(stats func() sql.DBStats, interval time.Duration) func() { if stats == nil { return func() {} } - installed := &stats + installed := &duckDBPoolSource{} duckDBPoolStats.Store(installed) - return func() { duckDBPoolStats.CompareAndSwap(installed, nil) } + stop, done := make(chan struct{}), make(chan struct{}) + go func() { + defer close(done) + ticker := time.NewTicker(interval) + defer ticker.Stop() + for { + select { + case <-stop: + return + default: + } + sample := stats() + installed.sample.Store(&sample) + select { + case <-stop: + return + case <-ticker.C: + } + } + }() + var once sync.Once + return func() { + once.Do(func() { + duckDBPoolStats.CompareAndSwap(installed, nil) + close(stop) + <-done + }) + } } func poolStats() sql.DBStats { - if fn := duckDBPoolStats.Load(); fn != nil { - return (*fn)() + if source := duckDBPoolStats.Load(); source != nil { + if sample := source.sample.Load(); sample != nil { + return *sample + } } return sql.DBStats{} } @@ -571,8 +622,8 @@ type compactionBacklogSource struct{ count func() int } var compactionBacklog atomic.Pointer[compactionBacklogSource] -// SetCompactionBacklogSource reads the exact inventory rather than a periodic -// file-stat snapshot. Closing an older repository cannot clear a newer source. +// SetCompactionBacklogSource reads the cached exact inventory. The source must +// not take the catalog lock. Closing an older repository cannot clear a newer source. func SetCompactionBacklogSource(count func() int) func() { source := &compactionBacklogSource{count: count} compactionBacklog.Store(source) diff --git a/internal/query/duck.go b/internal/query/duck.go index 2465ce9c9..9daaac8af 100644 --- a/internal/query/duck.go +++ b/internal/query/duck.go @@ -42,6 +42,8 @@ type Duck struct { // only if they are still reading it. releasePoolGauges func() releaseMemoryGauges func() + stopMemorySampler context.CancelFunc + memorySamplerDone chan struct{} // writeDB has two connections: analytical rollups share writeGate, while // completed-batch aggregates use cacheGate and disjoint tables. Both stay // outside the read pool. Maintenance acquires both gates for checkpointing. @@ -251,32 +253,6 @@ func NewDuck(ctx context.Context, cfg config.Config, repository *telemetrystore. d := &Duck{DB: db, writeDB: writeDB, cfg: cfg, repository: repository, publicationBounds: repository.PublicationBounds} // Reads are the pool that can starve, so it is the one worth watching. d.releasePoolGauges = metrics.SetDuckDBPoolSource(db.Stats) - // And what DuckDB admits to holding, so the gap against process RSS is - // observable rather than inferred. - d.releaseMemoryGauges = metrics.SetDuckDBMemorySource(func() map[string]int64 { - ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) - defer cancel() - rows, err := db.QueryContext(ctx, "SELECT tag, sum(memory_usage_bytes) FROM duckdb_memory() GROUP BY tag") - if err != nil { - return nil - } - defer rows.Close() - tags := make(map[string]int64, 16) - for rows.Next() { - var tag string - var bytes int64 - if err := rows.Scan(&tag, &bytes); err != nil { - return nil - } - tags[tag] = bytes - } - // A partial read is worse than no read: it publishes a low sum, and - // the untracked figure derived from it reads correspondingly high. - if err := rows.Err(); err != nil { - return nil - } - return tags - }) if cfg.DuckDBMemory == "" { // Only when the operator hasn't pinned FANOUT_DUCKDB_MEMORY: keep DuckDB's // cgroup-aware auto limit on big boxes but leave absolute RAM headroom on @@ -316,6 +292,7 @@ func NewDuck(ctx context.Context, cfg config.Config, repository *telemetrystore. slog.Info("rollup skip-to-latest: existing data marked already-rolled-up") } } + d.startMemorySampler(duckDBMemorySampleInterval, d.sampleMemory) return d, nil } @@ -431,16 +408,20 @@ func sqlLiteral(v string) string { func (d *Duck) Close() error { d.closeOnce.Do(func() { + if d.stopMemorySampler != nil { + d.stopMemorySampler() + } if d.releasePoolGauges != nil { d.releasePoolGauges() } - // Same reason as the pool source: the closure captures this Duck's - // read pool, so leaving it installed past Close means the next scrape - // queries a closed pool and the gauges freeze at stale values that - // still look live. if d.releaseMemoryGauges != nil { d.releaseMemoryGauges() } + if d.stopMemorySampler != nil { + // Join before closing either handle: the read pool owns the shared + // connector, which must outlive every sampling query. + <-d.memorySamplerDone + } var errs []error if d.writeDB != nil { // The write handle holds a connector that does not close the diff --git a/internal/query/duck_memory.go b/internal/query/duck_memory.go new file mode 100644 index 000000000..a438d0523 --- /dev/null +++ b/internal/query/duck_memory.go @@ -0,0 +1,78 @@ +package query + +import ( + "context" + "log/slog" + "time" + + "github.com/labstack/fanout/internal/metrics" +) + +const duckDBMemorySampleInterval = 5 * time.Second + +// startMemorySampler keeps engine calls off the scrape path. There is only one +// sampling goroutine, even if DuckDB ignores cancellation inside a statement. +// Its lifetime belongs to Duck, rather than the context used to open it. +func (d *Duck) startMemorySampler(interval time.Duration, sample func(context.Context) (map[string]int64, error)) { + ctx, cancel := context.WithCancel(context.Background()) + d.stopMemorySampler = cancel + d.memorySamplerDone = make(chan struct{}) + record, release := metrics.SetDuckDBMemorySource() + d.releaseMemoryGauges = release + go func() { + defer close(d.memorySamplerDone) + ticker := time.NewTicker(interval) + defer ticker.Stop() + failures := 0 + for { + if ctx.Err() != nil { + return + } + readCtx, readCancel := context.WithTimeout(ctx, duckDBMemorySampleInterval) + tags, err := sample(readCtx) + if err == nil { + err = readCtx.Err() + } + readCancel() + if ctx.Err() != nil { + return + } + if err != nil { + failures++ + if failures == 3 { + slog.Warn("DuckDB memory sampling failed repeatedly; retaining last successful sample", "err", err) + } + } else { + record(tags, time.Now()) + failures = 0 + } + select { + case <-ctx.Done(): + return + case <-ticker.C: + } + } + }() +} + +func (d *Duck) sampleMemory(ctx context.Context) (map[string]int64, error) { + rows, err := d.DB.QueryContext(ctx, "SELECT tag, sum(memory_usage_bytes) FROM duckdb_memory() GROUP BY tag") + if err != nil { + return nil, err + } + defer rows.Close() + tags := make(map[string]int64, 16) + for rows.Next() { + var tag string + var bytes int64 + if err := rows.Scan(&tag, &bytes); err != nil { + return nil, err + } + tags[tag] = bytes + } + // Publish only complete samples: a partial sum misstates untracked memory. + if err := rows.Err(); err != nil { + return nil, err + } + return tags, nil +} diff --git a/internal/query/duck_memory_test.go b/internal/query/duck_memory_test.go new file mode 100644 index 000000000..edde0a5eb --- /dev/null +++ b/internal/query/duck_memory_test.go @@ -0,0 +1,154 @@ +package query + +import ( + "bytes" + "context" + "errors" + "log/slog" + "net/http" + "net/http/httptest" + "strconv" + "strings" + "testing" + "time" + + "github.com/DATA-DOG/go-sqlmock" + "github.com/prometheus/client_golang/prometheus/promhttp" +) + +func TestMetricsScrapeKeepsLastMemorySampleWhileSamplerBlocks(t *testing.T) { + var log bytes.Buffer + logger := slog.Default() + slog.SetDefault(slog.New(slog.NewTextHandler(&log, nil))) + defer slog.SetDefault(logger) + + d := &Duck{} + blocked := make(chan struct{}) + unblock := make(chan struct{}) + t.Cleanup(func() { + d.stopMemorySampler() + close(unblock) + if err := d.Close(); err != nil { + t.Error(err) + } + }) + calls := 0 + d.startMemorySampler(time.Millisecond, func(ctx context.Context) (map[string]int64, error) { + if _, ok := ctx.Deadline(); !ok { + return nil, errors.New("missing sampling deadline") + } + calls++ + switch { + case calls == 1: + return map[string]int64{"BASE_TABLE": 4096, "HASH_TABLE": 2048}, nil + case calls <= 5: + return nil, errors.New("memory query failed") + default: + close(blocked) + // Model an engine call that ignores cancellation, as observed in + // the benchmark. Scrapes must not wait for it or spawn more calls. + <-unblock + return nil, ctx.Err() + } + }) + select { + case <-blocked: + case <-time.After(5 * time.Second): + t.Fatal("sampler did not reach the blocking query") + } + if n := strings.Count(log.String(), "memory sampling failed repeatedly"); n != 1 { + t.Fatalf("repeated failures logged %d warnings, want 1: %s", n, log.String()) + } + + handler := promhttp.Handler() + var timestamp string + for range 3 { + recorder := httptest.NewRecorder() + done := make(chan struct{}) + start := time.Now() + go func() { + handler.ServeHTTP(recorder, httptest.NewRequest(http.MethodGet, "/metrics", nil)) + close(done) + }() + select { + case <-done: + case <-time.After(100 * time.Millisecond): + t.Fatal("/metrics waited for the blocked memory query") + } + t.Logf("/metrics returned in %s with the sampler blocked", time.Since(start)) + if recorder.Code != http.StatusOK { + t.Fatalf("/metrics status=%d: %s", recorder.Code, recorder.Body.String()) + } + body := recorder.Body.String() + for _, series := range []string{ + `fanout_duckdb_memory_bytes{tag="BASE_TABLE"} 4096`, + `fanout_duckdb_memory_bytes{tag="HASH_TABLE"} 2048`, + } { + if !strings.Contains(body, series) { + t.Fatalf("missing cached series %s", series) + } + } + const prefix = "fanout_duckdb_memory_last_success_timestamp_seconds " + for _, line := range strings.Split(body, "\n") { + if strings.HasPrefix(line, prefix) { + value := strings.TrimPrefix(line, prefix) + at, err := strconv.ParseFloat(value, 64) + if err != nil || at <= 0 { + t.Fatalf("invalid last-success timestamp: %s", line) + } + if timestamp != "" && value != timestamp { + t.Fatal("blocked sampler advanced its last-success timestamp") + } + timestamp = value + } + } + if timestamp == "" { + t.Fatal("/metrics did not expose sample freshness") + } + } +} + +func TestDuckCloseCancelsAndJoinsMemorySampler(t *testing.T) { + d := &Duck{} + started := make(chan struct{}) + stopped := make(chan struct{}) + d.startMemorySampler(time.Hour, func(ctx context.Context) (map[string]int64, error) { + close(started) + <-ctx.Done() + close(stopped) + return nil, ctx.Err() + }) + <-started + if err := d.Close(); err != nil { + t.Fatal(err) + } + select { + case <-stopped: + default: + t.Fatal("Close returned before the sampler stopped") + } + if err := d.Close(); err != nil { + t.Fatal(err) + } +} + +func TestMemorySamplerRejectsPartialQueryResults(t *testing.T) { + db, mock, err := sqlmock.New() + if err != nil { + t.Fatal(err) + } + defer db.Close() + mock.ExpectQuery("SELECT tag, sum").WillReturnRows( + sqlmock.NewRows([]string{"tag", "bytes"}). + AddRow("BASE_TABLE", 4096). + AddRow("HASH_TABLE", 2048). + RowError(1, errors.New("read failed")), + ) + d := &Duck{DB: db} + if tags, err := d.sampleMemory(t.Context()); err == nil || tags != nil { + t.Fatalf("partial query returned tags=%v, err=%v", tags, err) + } + if err := mock.ExpectationsWereMet(); err != nil { + t.Fatal(err) + } +} diff --git a/internal/telemetry/compaction_debt.go b/internal/telemetry/compaction_debt.go index 7f86943f7..a38ed5a83 100644 --- a/internal/telemetry/compaction_debt.go +++ b/internal/telemetry/compaction_debt.go @@ -16,11 +16,10 @@ const ( ) // Generation0BatchCount is the exact live ingest-batch inventory, including -// terminal oversized batches and underfilled day groups. No metadata scan. +// terminal oversized batches and underfilled day groups. The cached count +// lets metrics read it without waiting for the catalog lock or a metadata scan. func (p *ParquetStore) Generation0BatchCount() int { - p.mu.RLock() - defer p.mu.RUnlock() - return p.generation0Batches + return int(p.generation0Snapshot.Load()) } // CompactionDebt returns the reclaimable generation-0 backlog, excluding @@ -74,6 +73,7 @@ func (p *ParquetStore) setBatchLocked(id string, batch *storedBatch) { p.batches[id] = batch p.changeGeneration0Locked(batch.metadata, 1) } + p.generation0Snapshot.Store(int64(p.generation0Batches)) if p.compactionDebt < debtBefore && p.debtChanged != nil { close(p.debtChanged) p.debtChanged = nil diff --git a/internal/telemetry/compaction_debt_test.go b/internal/telemetry/compaction_debt_test.go index 05ffa280a..892ad86eb 100644 --- a/internal/telemetry/compaction_debt_test.go +++ b/internal/telemetry/compaction_debt_test.go @@ -4,10 +4,41 @@ import ( "context" "errors" "fmt" + "net/http" + "net/http/httptest" + "strings" "testing" "time" + + "github.com/labstack/fanout/internal/metrics" + "github.com/prometheus/client_golang/prometheus/promhttp" ) +func TestMetricsScrapeDoesNotWaitForCatalogLock(t *testing.T) { + p := &ParquetStore{batches: make(map[string]*storedBatch)} + p.installBatch(&storedBatch{metadata: BatchMetadata{ID: "a", Generation: 0, Spans: 1}}) + release := metrics.SetCompactionBacklogSource(p.Generation0BatchCount) + defer release() + + p.mu.Lock() + defer p.mu.Unlock() + recorder := httptest.NewRecorder() + done := make(chan struct{}) + handler := promhttp.Handler() + go func() { + handler.ServeHTTP(recorder, httptest.NewRequest(http.MethodGet, "/metrics", nil)) + close(done) + }() + select { + case <-done: + case <-time.After(100 * time.Millisecond): + t.Fatal("/metrics waited for the catalog lock") + } + if recorder.Code != http.StatusOK || !strings.Contains(recorder.Body.String(), "fanout_compaction_backlog_batches 1\n") { + t.Fatalf("scrape did not return the cached backlog: %d %s", recorder.Code, recorder.Body.String()) + } +} + func TestCompactionLimitNotificationDoesNotMissCrossings(t *testing.T) { p := &ParquetStore{batches: make(map[string]*storedBatch)} for i := range CompactionDebtLimit + CompactionBatchLimit - 2 { diff --git a/internal/telemetry/parquet.go b/internal/telemetry/parquet.go index 332b0f4b0..1f33d0c4f 100644 --- a/internal/telemetry/parquet.go +++ b/internal/telemetry/parquet.go @@ -14,6 +14,7 @@ import ( "sort" "strings" "sync" + "sync/atomic" "time" "github.com/parquet-go/parquet-go" @@ -91,10 +92,12 @@ type ParquetStore struct { publishGate chan struct{} batches map[string]*storedBatch generation0Batches int - generation0ByDay map[int64]int - compactionDebt int - debtChanged chan struct{} - debtLimitReached chan struct{} + // Published after each catalog mutation; scrapes never take mu. + generation0Snapshot atomic.Int64 + generation0ByDay map[int64]int + compactionDebt int + debtChanged chan struct{} + debtLimitReached chan struct{} } type ParquetStats struct { From d4dc0a356f7bc80f74a6ffee551765d0cc1f30b3 Mon Sep 17 00:00:00 2001 From: Vishal Rana Date: Fri, 16 Oct 2026 09:09:43 -0700 Subject: [PATCH 2/3] fix(query): sample DuckDB memory on its own connection The background memory sampler borrowed a read-pool connection every 5 s. A memory query blocked inside the engine could therefore hold a request slot for as long as it waited, and the stall test that asserts no read connection is in use during a write raced with it. The sampler now uses one dedicated connection from the shared connector, so it never takes read or write capacity. Claude-Session: https://claude.ai/code/session_01XA52nWyQ3R5f5kK8kjpAQW --- internal/query/duck.go | 34 ++++++++---- internal/query/duck_memory.go | 2 +- internal/query/duck_memory_test.go | 57 +++++++++++++++++++- internal/query/stall_instrumentation_test.go | 20 +++++-- 4 files changed, 97 insertions(+), 16 deletions(-) diff --git a/internal/query/duck.go b/internal/query/duck.go index 9daaac8af..f731148e3 100644 --- a/internal/query/duck.go +++ b/internal/query/duck.go @@ -32,7 +32,7 @@ type Duck struct { // DB is the read pool. Reads scan immutable Parquet concurrently, so it is // sized from the machine. DB *sql.DB - // closeOnce makes Close idempotent. Both handles come from one connector, + // closeOnce makes Close idempotent. All handles come from one connector, // and database/sql closes a connector that implements io.Closer — so a // second Close would call duckdb_close twice on the same instance, which // is a double free the runtime cannot recover from. @@ -44,6 +44,9 @@ type Duck struct { releaseMemoryGauges func() stopMemorySampler context.CancelFunc memorySamplerDone chan struct{} + // memoryDB has one borrowed connection so a blocked memory query cannot + // occupy a request or write slot. The read pool owns the shared instance. + memoryDB *sql.DB // writeDB has two connections: analytical rollups share writeGate, while // completed-batch aggregates use cacheGate and disjoint tables. Both stay // outside the read pool. Maintenance acquires both gates for checkpointing. @@ -246,11 +249,11 @@ func NewDuck(ctx context.Context, cfg config.Config, repository *telemetrystore. return nil, err } - db, writeDB, err := openDuckDB(ctx, dsn, tempDir, duckDBPoolSize(cfg)) + db, writeDB, memoryDB, err := openDuckDB(ctx, dsn, tempDir, duckDBPoolSize(cfg)) if err != nil { return nil, fmt.Errorf("open DuckDB query cache: %w (the cache at %s is rebuildable from Parquet)", err, dbPath) } - d := &Duck{DB: db, writeDB: writeDB, cfg: cfg, repository: repository, publicationBounds: repository.PublicationBounds} + d := &Duck{DB: db, writeDB: writeDB, memoryDB: memoryDB, cfg: cfg, repository: repository, publicationBounds: repository.PublicationBounds} // Reads are the pool that can starve, so it is the one worth watching. d.releasePoolGauges = metrics.SetDuckDBPoolSource(db.Stats) if cfg.DuckDBMemory == "" { @@ -334,14 +337,15 @@ func (d *Duck) skipRollupToLatest(ctx context.Context) error { return nil } -// openDuckDB returns the read pool and the single-connection write handle. +// openDuckDB returns the read pool, the two-connection write handle, and the +// single-connection memory sampling handle. // -// Both come from one connector so they share one DuckDB instance and therefore +// All come from one connector so they share one DuckDB instance and therefore // one catalog: a write committed on one handle is visible to the other, and // DDL run at boot applies to both. (A second connector to the same file would // also work — DuckDB keeps an instance cache — but two owners of one instance // is exactly the hazard the wrapper below exists to avoid.) -func openDuckDB(ctx context.Context, dsn, tempDir string, maxConns int) (*sql.DB, *sql.DB, error) { +func openDuckDB(ctx context.Context, dsn, tempDir string, maxConns int) (*sql.DB, *sql.DB, *sql.DB, error) { // Every pooled connection must use UTC. DuckDB otherwise inherits the host // timezone and casts TIMESTAMPTZ rollup buckets into local wall-clock // TIMESTAMP values, while API windows arrive in UTC. @@ -370,7 +374,7 @@ func openDuckDB(ctx context.Context, dsn, tempDir string, maxConns int) (*sql.DB return nil }) if err != nil { - return nil, nil, err + return nil, nil, nil, err } db := sql.OpenDB(connector) @@ -395,7 +399,12 @@ func openDuckDB(ctx context.Context, dsn, tempDir string, maxConns int) (*sql.DB writeDB := sql.OpenDB(borrowedConnector{connector}) writeDB.SetMaxOpenConns(2) writeDB.SetMaxIdleConns(2) - return db, writeDB, nil + // Memory sampling can block inside the engine despite its deadline. Keep + // its one connection separate from both request and write capacity. + memoryDB := sql.OpenDB(borrowedConnector{connector}) + memoryDB.SetMaxOpenConns(1) + memoryDB.SetMaxIdleConns(1) + return db, writeDB, memoryDB, nil } // borrowedConnector hands out connections from a connector it does not own. @@ -418,18 +427,21 @@ func (d *Duck) Close() error { d.releaseMemoryGauges() } if d.stopMemorySampler != nil { - // Join before closing either handle: the read pool owns the shared + // Join before closing any handle: the read pool owns the shared // connector, which must outlive every sampling query. <-d.memorySamplerDone } var errs []error + if d.memoryDB != nil { + errs = append(errs, d.memoryDB.Close()) + } if d.writeDB != nil { // The write handle holds a connector that does not close the // instance, so this releases its connection and nothing else. errs = append(errs, d.writeDB.Close()) } - // A Duck assembled field by field, as tests do, may hold neither - // handle; closing one is then nothing to do rather than a panic. + // A Duck assembled field by field, as tests do, may hold no handles; + // closing one is then nothing to do rather than a panic. if d.DB != nil { errs = append(errs, d.DB.Close()) } diff --git a/internal/query/duck_memory.go b/internal/query/duck_memory.go index a438d0523..6596ea1eb 100644 --- a/internal/query/duck_memory.go +++ b/internal/query/duck_memory.go @@ -56,7 +56,7 @@ func (d *Duck) startMemorySampler(interval time.Duration, sample func(context.Co } func (d *Duck) sampleMemory(ctx context.Context) (map[string]int64, error) { - rows, err := d.DB.QueryContext(ctx, "SELECT tag, sum(memory_usage_bytes) FROM duckdb_memory() GROUP BY tag") + rows, err := d.memoryDB.QueryContext(ctx, "SELECT tag, sum(memory_usage_bytes) FROM duckdb_memory() GROUP BY tag") if err != nil { return nil, err } diff --git a/internal/query/duck_memory_test.go b/internal/query/duck_memory_test.go index edde0a5eb..a1350e2ca 100644 --- a/internal/query/duck_memory_test.go +++ b/internal/query/duck_memory_test.go @@ -13,6 +13,8 @@ import ( "time" "github.com/DATA-DOG/go-sqlmock" + "github.com/labstack/fanout/internal/config" + telemetrystore "github.com/labstack/fanout/internal/telemetry/store" "github.com/prometheus/client_golang/prometheus/promhttp" ) @@ -132,6 +134,59 @@ func TestDuckCloseCancelsAndJoinsMemorySampler(t *testing.T) { } } +func TestMemorySamplingDoesNotNeedReadOrWritePoolCapacity(t *testing.T) { + ctx, cancel := context.WithTimeout(t.Context(), 5*time.Second) + defer cancel() + cfg := config.Config{DataDir: t.TempDir(), DuckDBMemory: "128MB", DuckDBMaxConns: 1} + repository, err := telemetrystore.Open(cfg.TelemetryDir()) + if err != nil { + t.Fatal(err) + } + defer repository.Close() + d, err := NewDuck(ctx, cfg, repository) + if err != nil { + t.Fatal(err) + } + defer func() { + if err := d.Close(); err != nil { + t.Error(err) + } + if open := d.memoryDB.Stats().OpenConnections; open != 0 { + t.Errorf("Close left %d memory connections open", open) + } + }() + // Sample synchronously below so no timer determines which query runs. + d.stopMemorySampler() + <-d.memorySamplerDone + if got := d.memoryDB.Stats().MaxOpenConnections; got != 1 { + t.Fatalf("memory handle MaxOpenConnections = %d, want 1", got) + } + + // Exhaust both application pools. Sampling must still reach the engine. + read, err := d.DB.Conn(ctx) + if err != nil { + t.Fatal(err) + } + defer read.Close() + for range 2 { + write, err := d.writeDB.Conn(ctx) + if err != nil { + t.Fatal(err) + } + defer write.Close() + } + tags, err := d.sampleMemory(ctx) + if err != nil { + t.Fatalf("sample with read and write pools exhausted: %v", err) + } + if len(tags) == 0 { + t.Fatal("memory sample contains no tags") + } + if inUse := d.memoryDB.Stats().InUse; inUse != 0 { + t.Fatalf("memory query retained %d connections after reading rows", inUse) + } +} + func TestMemorySamplerRejectsPartialQueryResults(t *testing.T) { db, mock, err := sqlmock.New() if err != nil { @@ -144,7 +199,7 @@ func TestMemorySamplerRejectsPartialQueryResults(t *testing.T) { AddRow("HASH_TABLE", 2048). RowError(1, errors.New("read failed")), ) - d := &Duck{DB: db} + d := &Duck{memoryDB: db} if tags, err := d.sampleMemory(t.Context()); err == nil || tags != nil { t.Fatalf("partial query returned tags=%v, err=%v", tags, err) } diff --git a/internal/query/stall_instrumentation_test.go b/internal/query/stall_instrumentation_test.go index 2192c6693..ff0e1b735 100644 --- a/internal/query/stall_instrumentation_test.go +++ b/internal/query/stall_instrumentation_test.go @@ -42,6 +42,13 @@ func TestWritesDoNotDrawFromTheReadPool(t *testing.T) { if d.writer() != d.writeDB { t.Fatal("writes are not going through the write handle") } + // Occupy the sampler's dedicated slot throughout the assertions. Even a + // blocked memory query must leave request and write capacity untouched. + memory, err := d.memoryDB.Conn(ctx) + if err != nil { + t.Fatalf("hold memory connection: %v", err) + } + defer memory.Close() // Use the read pool first, so its idle count is non-zero and the assertion // below is about where the write went rather than about a pool nobody has @@ -153,8 +160,9 @@ func TestPublisherWaitingTracksFreshAndCanceledPublication(t *testing.T) { // closed, and duckdb's connector closes the instance behind an unsynchronized // bool. Two handles over one connector therefore meant two duckdb_close calls // on one instance — a double free that kills the process rather than returning -// an error. The write handle borrows the instance; the read pool owns it. -func TestClosingTheWriteHandleLeavesTheInstanceAlive(t *testing.T) { +// an error. The write and memory handles borrow the instance; the read pool +// owns it. +func TestClosingBorrowedHandlesLeavesTheInstanceAlive(t *testing.T) { ctx := context.Background() cfg := config.Config{DataDir: t.TempDir(), RollupInterval: time.Minute, DuckDBMemory: "128MB"} repository, err := telemetrystore.Open(cfg.TelemetryDir()) @@ -166,13 +174,19 @@ func TestClosingTheWriteHandleLeavesTheInstanceAlive(t *testing.T) { if err != nil { t.Fatalf("NewDuck: %v", err) } + defer d.Close() + d.stopMemorySampler() + <-d.memorySamplerDone if err := d.writeDB.Close(); err != nil { t.Fatalf("close write handle: %v", err) } + if err := d.memoryDB.Close(); err != nil { + t.Fatalf("close memory handle: %v", err) + } var one int if err := d.DB.QueryRowContext(ctx, "SELECT 1").Scan(&one); err != nil { - t.Fatalf("read pool died with the write handle: %v", err) + t.Fatalf("read pool died with the borrowed handles: %v", err) } // Close is idempotent, including under a second shutdown path arriving at From 8d6276c7e60950f59b0e8340481cee168d119e50 Mon Sep 17 00:00:00 2001 From: Vishal Rana Date: Fri, 16 Oct 2026 09:39:13 -0700 Subject: [PATCH 3/3] fix(metrics): bound sampler shutdown and expire stale memory readings Review fixes for the background DuckDB memory sampler: - Close waits at most one second for the sampler. If the engine ignores cancellation, handles stay alive until the abandoned query returns, so native code never runs against a closed instance. - fanout_duckdb_memory_bytes is exported only while the sample is at most two intervals old. A failed or hung sampler therefore drops the series instead of reporting stale values as current; the last-success timestamp is always exported. - Pool gauges read live db.Stats() again. The pool cache, its sampler and its join are removed. - Each read uses the sampling interval as its timeout, the next read starts one interval after the previous one finishes, and a successful sample is kept even if its deadline passes after the query returns. - Connection initialization runs SET TimeZone='UTC' with the context the driver passes to Connect, so connections opened after startup still initialize. Claude-Session: https://claude.ai/code/session_01XA52nWyQ3R5f5kK8kjpAQW --- internal/metrics/duckdb_memory_test.go | 30 +++-- internal/metrics/duckdb_pool_test.go | 80 ----------- internal/metrics/metrics.go | 62 ++------- internal/query/duck.go | 85 ++++++++---- internal/query/duck_memory.go | 15 +-- internal/query/duck_memory_test.go | 175 +++++++++++++++++++++++-- internal/query/duck_test.go | 33 +++++ 7 files changed, 301 insertions(+), 179 deletions(-) delete mode 100644 internal/metrics/duckdb_pool_test.go diff --git a/internal/metrics/duckdb_memory_test.go b/internal/metrics/duckdb_memory_test.go index 034c3b159..e01b0b4bb 100644 --- a/internal/metrics/duckdb_memory_test.go +++ b/internal/metrics/duckdb_memory_test.go @@ -32,7 +32,7 @@ func gatherDuckDBMemory(t *testing.T) map[string]float64 { } func TestDuckDBMemoryPublishesCompleteSamplesAndTimestamp(t *testing.T) { - record, release := SetDuckDBMemorySource() + record, release := SetDuckDBMemorySource(time.Minute) defer release() if got := gatherDuckDBMemory(t); len(got) != 0 { t.Fatalf("before the first sample exported %v", got) @@ -41,23 +41,24 @@ func TestDuckDBMemoryPublishesCompleteSamplesAndTimestamp(t *testing.T) { t.Fatalf("before the first sample timestamp=%g", got) } tags := map[string]int64{"BASE_TABLE": 4096} - record(tags, time.Unix(1234, 500_000_000)) + at := time.Now() + record(tags, at) tags["BASE_TABLE"] = 0 // Publication owns an immutable copy. if got := gatherDuckDBMemory(t); got["BASE_TABLE"] != 4096 { t.Fatalf("healthy read exported %v, want BASE_TABLE=4096", got) } - if got := gatherMemoryTimestamp(t); got != 1234.5 { - t.Fatalf("timestamp=%g, want 1234.5", got) + if got := gatherMemoryTimestamp(t); got != float64(at.UnixNano())/1e9 { + t.Fatalf("timestamp=%g, want %v", got, at) } - record(map[string]int64{"HASH_TABLE": 2048}, time.Unix(1235, 0)) + record(map[string]int64{"HASH_TABLE": 2048}, time.Now()) if got := gatherDuckDBMemory(t); len(got) != 1 || got["HASH_TABLE"] != 2048 { t.Fatalf("replacement sample exported %v", got) } } func TestDuckDBMemoryReleaseOnlyClearsItsOwner(t *testing.T) { - recordOld, releaseOld := SetDuckDBMemorySource() - record, release := SetDuckDBMemorySource() + recordOld, releaseOld := SetDuckDBMemorySource(time.Minute) + record, release := SetDuckDBMemorySource(time.Minute) defer release() record(map[string]int64{"BASE_TABLE": 2}, time.Now()) releaseOld() @@ -72,7 +73,7 @@ func TestDuckDBMemoryReleaseOnlyClearsItsOwner(t *testing.T) { } func TestDuckDBMemoryIsSafeUnderConcurrentScrapes(t *testing.T) { - record, release := SetDuckDBMemorySource() + record, release := SetDuckDBMemorySource(time.Minute) defer release() record(map[string]int64{"BASE_TABLE": 1, "HASH_TABLE": 2, "ORDER_BY": 3}, time.Now()) @@ -131,3 +132,16 @@ func gatherMemoryTimestamp(t *testing.T) float64 { } return -1 } + +func TestDuckDBMemoryDropsStaleSeriesAndKeepsTimestamp(t *testing.T) { + record, release := SetDuckDBMemorySource(time.Second) + defer release() + at := time.Now().Add(-3 * time.Second) + record(map[string]int64{"BASE_TABLE": 4096}, at) + if got := gatherDuckDBMemory(t); len(got) != 0 { + t.Fatalf("stale sample exported %v", got) + } + if got := gatherMemoryTimestamp(t); got != float64(at.UnixNano())/1e9 { + t.Fatalf("stale sample timestamp=%g, want %v", got, at) + } +} diff --git a/internal/metrics/duckdb_pool_test.go b/internal/metrics/duckdb_pool_test.go deleted file mode 100644 index 6ee958272..000000000 --- a/internal/metrics/duckdb_pool_test.go +++ /dev/null @@ -1,80 +0,0 @@ -package metrics - -import ( - "database/sql" - "net/http" - "net/http/httptest" - "strings" - "testing" - "time" - - "github.com/prometheus/client_golang/prometheus/promhttp" -) - -func TestMetricsScrapeKeepsCachedPoolStatsWhileSamplerBlocks(t *testing.T) { - blocked, unblock := make(chan struct{}), make(chan struct{}) - calls := 0 - release := setDuckDBPoolSource(func() sql.DBStats { - calls++ - if calls == 2 { - close(blocked) - <-unblock - } - return sql.DBStats{InUse: 1, Idle: 2, MaxOpenConnections: 3, WaitCount: 4, WaitDuration: 5 * time.Second} - }, time.Millisecond) - t.Cleanup(func() { - close(unblock) - release() - release() // Release is idempotent. - }) - select { - case <-blocked: - case <-time.After(5 * time.Second): - t.Fatal("pool sampler did not reach the blocking call") - } - recorder := httptest.NewRecorder() - done := make(chan struct{}) - handler := promhttp.Handler() - go func() { - handler.ServeHTTP(recorder, httptest.NewRequest(http.MethodGet, "/metrics", nil)) - close(done) - }() - select { - case <-done: - case <-time.After(100 * time.Millisecond): - t.Fatal("/metrics waited for pool bookkeeping") - } - if recorder.Code != http.StatusOK { - t.Fatalf("/metrics status=%d", recorder.Code) - } - for _, series := range []string{ - "fanout_duckdb_pool_in_use 1\n", - "fanout_duckdb_pool_idle 2\n", - "fanout_duckdb_pool_max_open 3\n", - "fanout_duckdb_pool_waits_total 4\n", - "fanout_duckdb_pool_wait_seconds_total 5\n", - } { - if !strings.Contains(recorder.Body.String(), series) { - t.Fatalf("missing cached series %q", series) - } - } -} - -func TestDuckDBPoolReleaseOnlyClearsItsOwner(t *testing.T) { - releaseOld := setDuckDBPoolSource(func() sql.DBStats { return sql.DBStats{InUse: 1} }, time.Hour) - release := setDuckDBPoolSource(func() sql.DBStats { return sql.DBStats{InUse: 2} }, time.Hour) - defer release() - releaseOld() - deadline := time.After(5 * time.Second) - for poolStats().InUse != 2 { - select { - case <-deadline: - t.Fatal("new owner's pool stats were cleared") - case <-time.After(time.Millisecond): - } - } - release() - if got := poolStats(); got != (sql.DBStats{}) { - t.Fatalf("released pool retained stats: %+v", got) - } -} diff --git a/internal/metrics/metrics.go b/internal/metrics/metrics.go index 70add7368..a862cfb5c 100644 --- a/internal/metrics/metrics.go +++ b/internal/metrics/metrics.go @@ -4,7 +4,6 @@ import ( "database/sql" "maps" "math" - "sync" "sync/atomic" "time" @@ -315,9 +314,9 @@ func RecordFlush(signal string, durationSec float64) { FlushDuration.WithLabelValues(signal).Observe(durationSec) } -type duckDBPoolSource struct{ sample atomic.Pointer[sql.DBStats] } +type duckDBPoolSource struct{ stats func() sql.DBStats } -// duckDBPoolStats is the cached source for the connection-pool gauges below. +// duckDBPoolStats is the live source for the connection-pool gauges below. // // A holder rather than a promauto.NewGaugeFunc closure over the pool: the // gauges are declared once for the process, while a *Duck is constructed many @@ -332,6 +331,7 @@ type duckDBMemorySample struct { type duckDBMemorySource struct { sample atomic.Pointer[duckDBMemorySample] + maxAge time.Duration } // duckDBMemoryStats holds the latest background sample for the memory gauge. @@ -348,8 +348,8 @@ var duckDBMemoryStats atomic.Pointer[duckDBMemorySource] // SetDuckDBMemorySource installs a cache, never a scrape-time query. Record // publishes a complete successful sample; failures leave the last one intact. // Releasing an older Duck cannot clear a newer Duck's source. -func SetDuckDBMemorySource() (record func(map[string]int64, time.Time), release func()) { - installed := &duckDBMemorySource{} +func SetDuckDBMemorySource(interval time.Duration) (record func(map[string]int64, time.Time), release func()) { + installed := &duckDBMemorySource{maxAge: 2 * interval} duckDBMemoryStats.Store(installed) return func(tags map[string]int64, at time.Time) { installed.sample.Store(&duckDBMemorySample{tags: maps.Clone(tags), at: at}) @@ -385,8 +385,10 @@ func (duckDBMemoryCollector) Collect(ch chan<- prometheus.Metric) { at := 0.0 if sample := source.sample.Load(); sample != nil { at = float64(sample.at.UnixNano()) / 1e9 - for tag, bytes := range sample.tags { - ch <- prometheus.MustNewConstMetric(duckDBMemoryDesc, prometheus.GaugeValue, float64(bytes), tag) + if time.Since(sample.at) <= source.maxAge { + for tag, bytes := range sample.tags { + ch <- prometheus.MustNewConstMetric(duckDBMemoryDesc, prometheus.GaugeValue, float64(bytes), tag) + } } } ch <- prometheus.MustNewConstMetric(duckDBMemorySampleDesc, prometheus.GaugeValue, at) @@ -396,56 +398,20 @@ func init() { prometheus.DefaultRegisterer.MustRegister(duckDBMemoryCollector{}) } -// SetDuckDBPoolSource samples pool bookkeeping independently of memory queries, -// so a stuck engine cannot hide pool starvation. Stats takes database/sql's -// mutex, which some connection cleanup paths hold during driver calls; scrapes -// therefore read only the cached result. Release stops and joins this sampler -// before the owning Duck closes its handles, without clearing a newer owner. +// SetDuckDBPoolSource reads database/sql bookkeeping on each scrape without +// querying the engine. Releasing an older Duck cannot clear a newer source. func SetDuckDBPoolSource(stats func() sql.DBStats) (release func()) { - return setDuckDBPoolSource(stats, 5*time.Second) -} - -func setDuckDBPoolSource(stats func() sql.DBStats, interval time.Duration) func() { if stats == nil { return func() {} } - installed := &duckDBPoolSource{} + installed := &duckDBPoolSource{stats: stats} duckDBPoolStats.Store(installed) - stop, done := make(chan struct{}), make(chan struct{}) - go func() { - defer close(done) - ticker := time.NewTicker(interval) - defer ticker.Stop() - for { - select { - case <-stop: - return - default: - } - sample := stats() - installed.sample.Store(&sample) - select { - case <-stop: - return - case <-ticker.C: - } - } - }() - var once sync.Once - return func() { - once.Do(func() { - duckDBPoolStats.CompareAndSwap(installed, nil) - close(stop) - <-done - }) - } + return func() { duckDBPoolStats.CompareAndSwap(installed, nil) } } func poolStats() sql.DBStats { if source := duckDBPoolStats.Load(); source != nil { - if sample := source.sample.Load(); sample != nil { - return *sample - } + return source.stats() } return sql.DBStats{} } diff --git a/internal/query/duck.go b/internal/query/duck.go index f731148e3..dca88af76 100644 --- a/internal/query/duck.go +++ b/internal/query/duck.go @@ -249,7 +249,7 @@ func NewDuck(ctx context.Context, cfg config.Config, repository *telemetrystore. return nil, err } - db, writeDB, memoryDB, err := openDuckDB(ctx, dsn, tempDir, duckDBPoolSize(cfg)) + db, writeDB, memoryDB, err := openDuckDB(dsn, tempDir, duckDBPoolSize(cfg)) if err != nil { return nil, fmt.Errorf("open DuckDB query cache: %w (the cache at %s is rebuildable from Parquet)", err, dbPath) } @@ -345,7 +345,7 @@ func (d *Duck) skipRollupToLatest(ctx context.Context) error { // DDL run at boot applies to both. (A second connector to the same file would // also work — DuckDB keeps an instance cache — but two owners of one instance // is exactly the hazard the wrapper below exists to avoid.) -func openDuckDB(ctx context.Context, dsn, tempDir string, maxConns int) (*sql.DB, *sql.DB, *sql.DB, error) { +func openDuckDB(dsn, tempDir string, maxConns int) (*sql.DB, *sql.DB, *sql.DB, error) { // Every pooled connection must use UTC. DuckDB otherwise inherits the host // timezone and casts TIMESTAMPTZ rollup buckets into local wall-clock // TIMESTAMP values, while API windows arrive in UTC. @@ -354,11 +354,15 @@ func openDuckDB(ctx context.Context, dsn, tempDir string, maxConns int) (*sql.DB // dir has already been used fails with "Cannot switch temporary directory // after the current one has been used". The boot hook runs once per pooled // connection, so only the first connection sets it; the rest inherit the - // instance-wide value. LOAD/ATTACH stay per-connection — they're idempotent. + // instance-wide value. var tempDirOnce sync.Once var tempDirErr error - connector, err := duckdb.NewConnector(dsn, func(execer driver.ExecerContext) error { + native, err := duckdb.NewConnector(dsn, nil) + if err != nil { + return nil, nil, nil, err + } + connector := &initializedConnector{Connector: native, init: func(ctx context.Context, execer driver.ExecerContext) error { if _, err := execer.ExecContext(ctx, "SET TimeZone='UTC'", nil); err != nil { return fmt.Errorf("set timezone: %w", err) } @@ -372,10 +376,7 @@ func openDuckDB(ctx context.Context, dsn, tempDir string, maxConns int) (*sql.DB return fmt.Errorf("set temp_directory: %w", tempDirErr) } return nil - }) - if err != nil { - return nil, nil, nil, err - } + }} db := sql.OpenDB(connector) // The on-disk DuckDB file contains only rebuildable rollups and views; Parquet @@ -411,6 +412,24 @@ func openDuckDB(ctx context.Context, dsn, tempDir string, maxConns int) (*sql.DB // It deliberately does not implement io.Closer: that is the whole point. type borrowedConnector struct{ driver.Connector } +// Initialize with each Connect call's context, never the startup context: +// database/sql may open replacement connections long after startup finishes. +type initializedConnector struct { + *duckdb.Connector + init func(context.Context, driver.ExecerContext) error +} + +func (c *initializedConnector) Connect(ctx context.Context) (driver.Conn, error) { + conn, err := c.Connector.Connect(ctx) + if err != nil { + return nil, err + } + if err := c.init(ctx, conn.(driver.ExecerContext)); err != nil { + return nil, errors.Join(err, conn.Close()) + } + return conn, nil +} + func sqlLiteral(v string) string { return "'" + strings.ReplaceAll(v, "'", "''") + "'" } @@ -427,29 +446,43 @@ func (d *Duck) Close() error { d.releaseMemoryGauges() } if d.stopMemorySampler != nil { - // Join before closing any handle: the read pool owns the shared - // connector, which must outlive every sampling query. - <-d.memorySamplerDone - } - var errs []error - if d.memoryDB != nil { - errs = append(errs, d.memoryDB.Close()) - } - if d.writeDB != nil { - // The write handle holds a connector that does not close the - // instance, so this releases its connection and nothing else. - errs = append(errs, d.writeDB.Close()) - } - // A Duck assembled field by field, as tests do, may hold no handles; - // closing one is then nothing to do rather than a panic. - if d.DB != nil { - errs = append(errs, d.DB.Close()) + timer := time.NewTimer(time.Second) + defer timer.Stop() + select { + case <-d.memorySamplerDone: + case <-timer.C: + // The engine may ignore cancellation. Keep all handles and the + // owning connector alive until the abandoned query returns; + // closing the instance underneath it can crash native code. + go func() { + <-d.memorySamplerDone + if err := d.closeHandles(); err != nil { + slog.Warn("close DuckDB after memory sampler stopped", "err", err) + } + }() + return + } } - d.closeErr = errors.Join(errs...) + d.closeErr = d.closeHandles() }) return d.closeErr } +func (d *Duck) closeHandles() error { + var errs []error + if d.memoryDB != nil { + errs = append(errs, d.memoryDB.Close()) + } + if d.writeDB != nil { + errs = append(errs, d.writeDB.Close()) + } + // The read pool owns the instance and must close after its borrowers. + if d.DB != nil { + errs = append(errs, d.DB.Close()) + } + return errors.Join(errs...) +} + // writer returns the dedicated write handle initialized by NewDuck. func (d *Duck) writer() *sql.DB { return d.writeDB } diff --git a/internal/query/duck_memory.go b/internal/query/duck_memory.go index 6596ea1eb..8405924d8 100644 --- a/internal/query/duck_memory.go +++ b/internal/query/duck_memory.go @@ -17,22 +17,19 @@ func (d *Duck) startMemorySampler(interval time.Duration, sample func(context.Co ctx, cancel := context.WithCancel(context.Background()) d.stopMemorySampler = cancel d.memorySamplerDone = make(chan struct{}) - record, release := metrics.SetDuckDBMemorySource() + record, release := metrics.SetDuckDBMemorySource(interval) d.releaseMemoryGauges = release go func() { defer close(d.memorySamplerDone) - ticker := time.NewTicker(interval) - defer ticker.Stop() + timer := time.NewTimer(interval) + defer timer.Stop() failures := 0 for { if ctx.Err() != nil { return } - readCtx, readCancel := context.WithTimeout(ctx, duckDBMemorySampleInterval) + readCtx, readCancel := context.WithTimeout(ctx, interval) tags, err := sample(readCtx) - if err == nil { - err = readCtx.Err() - } readCancel() if ctx.Err() != nil { return @@ -46,10 +43,12 @@ func (d *Duck) startMemorySampler(interval time.Duration, sample func(context.Co record(tags, time.Now()) failures = 0 } + // Pace from completion, including slow calls that ignore the deadline. + timer.Reset(interval) select { case <-ctx.Done(): return - case <-ticker.C: + case <-timer.C: } } }() diff --git a/internal/query/duck_memory_test.go b/internal/query/duck_memory_test.go index a1350e2ca..c928bad3a 100644 --- a/internal/query/duck_memory_test.go +++ b/internal/query/duck_memory_test.go @@ -9,16 +9,178 @@ import ( "net/http/httptest" "strconv" "strings" + "sync/atomic" "testing" + "testing/synctest" "time" "github.com/DATA-DOG/go-sqlmock" "github.com/labstack/fanout/internal/config" telemetrystore "github.com/labstack/fanout/internal/telemetry/store" + "github.com/prometheus/client_golang/prometheus" "github.com/prometheus/client_golang/prometheus/promhttp" ) -func TestMetricsScrapeKeepsLastMemorySampleWhileSamplerBlocks(t *testing.T) { +func TestDuckDBMemoryDropsSeriesWhenTheReadFails(t *testing.T) { + synctest.Test(t, func(t *testing.T) { + d := &Duck{} + defer d.Close() + calls := 0 + d.startMemorySampler(time.Second, func(context.Context) (map[string]int64, error) { + calls++ + if calls == 1 { + return map[string]int64{"BASE_TABLE": 4096}, nil + } + return nil, errors.New("memory query failed") + }) + synctest.Wait() + if memory, timestamp := memoryMetrics(t); memory != 4096 || timestamp <= 0 { + t.Fatalf("first sample: memory=%g, timestamp=%g", memory, timestamp) + } + time.Sleep(2500 * time.Millisecond) + synctest.Wait() + if memory, timestamp := memoryMetrics(t); memory != -1 || timestamp <= 0 { + t.Fatalf("failed reads: memory=%g, timestamp=%g", memory, timestamp) + } + }) +} + +func TestMemorySamplerBlockedSampleAgesOut(t *testing.T) { + synctest.Test(t, func(t *testing.T) { + d := &Duck{} + unblock := make(chan struct{}) + defer func() { + close(unblock) + d.Close() + }() + calls := 0 + d.startMemorySampler(time.Second, func(ctx context.Context) (map[string]int64, error) { + calls++ + if calls == 1 { + return map[string]int64{"BASE_TABLE": 4096}, nil + } + <-unblock + return nil, ctx.Err() + }) + time.Sleep(time.Second) + synctest.Wait() + memory, timestamp := memoryMetrics(t) + if memory != 4096 || timestamp <= 0 { + t.Fatalf("fresh sample: memory=%g, timestamp=%g", memory, timestamp) + } + time.Sleep(time.Second) + synctest.Wait() + if memory, _ := memoryMetrics(t); memory != 4096 { + t.Fatalf("sample exactly two intervals old: memory=%g", memory) + } + time.Sleep(500 * time.Millisecond) + synctest.Wait() + if memory, lastSuccess := memoryMetrics(t); memory != -1 || lastSuccess != timestamp { + t.Fatalf("hung sampler: memory=%g, timestamp=%g, want absent memory and timestamp=%g", memory, lastSuccess, timestamp) + } + if calls != 2 { + t.Fatalf("blocked sampler made %d calls, want 2", calls) + } + }) +} + +func TestMemorySamplerUsesIntervalDeadlineAndPacesFromCompletion(t *testing.T) { + synctest.Test(t, func(t *testing.T) { + d := &Duck{} + defer d.Close() + const interval = time.Second + var calls atomic.Int32 + var finished time.Time + d.startMemorySampler(interval, func(ctx context.Context) (map[string]int64, error) { + if calls.Add(1) > 1 { + if gap := time.Since(finished); gap != interval { + t.Errorf("next sample started after %s, want %s", gap, interval) + } + <-ctx.Done() + return nil, ctx.Err() + } + deadline, ok := ctx.Deadline() + if !ok || time.Until(deadline) != interval { + t.Errorf("read deadline=%v, want one interval from now", deadline) + } + // A successful query may return after its deadline; keep its result. + <-ctx.Done() + time.Sleep(interval / 2) + finished = time.Now() + return map[string]int64{"BASE_TABLE": 4096}, nil + }) + time.Sleep(2 * interval) + synctest.Wait() + if memory, _ := memoryMetrics(t); memory != 4096 { + t.Fatalf("successful overdue sample discarded: memory=%g", memory) + } + if calls.Load() != 1 { + t.Fatalf("sampler made %d calls before the next interval", calls.Load()) + } + time.Sleep(interval / 2) + synctest.Wait() + if calls.Load() != 2 { + t.Fatalf("sampler made %d calls, want 2", calls.Load()) + } + }) +} + +func TestDuckCloseBoundsWaitAndDefersHandleCleanup(t *testing.T) { + synctest.Test(t, func(t *testing.T) { + db, mock, err := sqlmock.New(sqlmock.MonitorPingsOption(true)) + if err != nil { + t.Fatal(err) + } + d := &Duck{DB: db} + unblock := make(chan struct{}) + d.startMemorySampler(time.Hour, func(ctx context.Context) (map[string]int64, error) { + <-unblock // Ignore cancellation, then use the handle before returning. + if err := db.PingContext(context.Background()); err != nil { + t.Errorf("sampler's handle closed while still in use: %v", err) + } + return nil, ctx.Err() + }) + synctest.Wait() + start := time.Now() + if err := d.Close(); err != nil { + t.Fatal(err) + } + if elapsed := time.Since(start); elapsed != time.Second { + t.Fatalf("Close waited %s, want 1s", elapsed) + } + if err := d.Close(); err != nil { + t.Fatal(err) + } + mock.ExpectPing() + mock.ExpectClose() + close(unblock) + synctest.Wait() + if err := mock.ExpectationsWereMet(); err != nil { + t.Fatal(err) + } + }) +} + +// A missing memory series is distinct from a real zero-byte sample. +func memoryMetrics(t *testing.T) (memory, timestamp float64) { + t.Helper() + memory, timestamp = -1, -1 + families, err := prometheus.DefaultGatherer.Gather() + if err != nil { + t.Fatal(err) + } + for _, family := range families { + switch family.GetName() { + case "fanout_duckdb_memory_bytes": + memory = family.Metric[0].GetGauge().GetValue() + case "fanout_duckdb_memory_last_success_timestamp_seconds": + timestamp = family.Metric[0].GetGauge().GetValue() + } + } + return +} + +func TestMetricsScrapeDropsStaleMemoryWhileSamplerBlocks(t *testing.T) { var log bytes.Buffer logger := slog.Default() slog.SetDefault(slog.New(slog.NewTextHandler(&log, nil))) @@ -35,7 +197,7 @@ func TestMetricsScrapeKeepsLastMemorySampleWhileSamplerBlocks(t *testing.T) { } }) calls := 0 - d.startMemorySampler(time.Millisecond, func(ctx context.Context) (map[string]int64, error) { + d.startMemorySampler(10*time.Millisecond, func(ctx context.Context) (map[string]int64, error) { if _, ok := ctx.Deadline(); !ok { return nil, errors.New("missing sampling deadline") } @@ -82,13 +244,8 @@ func TestMetricsScrapeKeepsLastMemorySampleWhileSamplerBlocks(t *testing.T) { t.Fatalf("/metrics status=%d: %s", recorder.Code, recorder.Body.String()) } body := recorder.Body.String() - for _, series := range []string{ - `fanout_duckdb_memory_bytes{tag="BASE_TABLE"} 4096`, - `fanout_duckdb_memory_bytes{tag="HASH_TABLE"} 2048`, - } { - if !strings.Contains(body, series) { - t.Fatalf("missing cached series %s", series) - } + if strings.Contains(body, "fanout_duckdb_memory_bytes") { + t.Fatal("blocked sampler still exports stale memory series") } const prefix = "fanout_duckdb_memory_last_success_timestamp_seconds " for _, line := range strings.Split(body, "\n") { diff --git a/internal/query/duck_test.go b/internal/query/duck_test.go index 67b53a6f9..3a0319d5a 100644 --- a/internal/query/duck_test.go +++ b/internal/query/duck_test.go @@ -221,6 +221,39 @@ WHERE bucket >= ? AND bucket < ?`, eventTime.Add(-time.Minute), eventTime.Add(ti } } +func TestNewDuckReconnectsInUTCAfterStartupContextCancelled(t *testing.T) { + t.Setenv("TZ", "America/Los_Angeles") + ctx, cancel := context.WithCancel(t.Context()) + defer cancel() + cfg := config.Config{DataDir: t.TempDir(), DuckDBMemory: "128MB"} + repository, err := telemetrystore.Open(cfg.TelemetryDir()) + if err != nil { + t.Fatal(err) + } + defer repository.Close() + d, err := NewDuck(ctx, cfg, repository) + if err != nil { + t.Fatal(err) + } + defer d.Close() + d.stopMemorySampler() + <-d.memorySamplerDone + cancel() + for name, db := range map[string]*sql.DB{"read": d.DB, "write": d.writeDB, "memory": d.memoryDB} { + t.Run(name, func(t *testing.T) { + // Retire every existing connection so this query must reconnect. + db.SetMaxIdleConns(0) + var timezone string + if err := db.QueryRowContext(t.Context(), "SELECT current_setting('TimeZone')").Scan(&timezone); err != nil { + t.Fatal(err) + } + if timezone != "UTC" { + t.Fatalf("new connection timezone=%q, want UTC", timezone) + } + }) + } +} + // A cancelled statement's interrupt can reach the ROLLBACK that ends its read // transaction, leaving the transaction open. The read must roll back itself // after cancellation, and a connection whose rollback failed must leave the