diff --git a/internal/metrics/duckdb_memory_test.go b/internal/metrics/duckdb_memory_test.go index 4ac7ea5b5..e01b0b4bb 100644 --- a/internal/metrics/duckdb_memory_test.go +++ b/internal/metrics/duckdb_memory_test.go @@ -1,12 +1,11 @@ package metrics import ( - "strings" "sync" "testing" + "time" "github.com/prometheus/client_golang/prometheus" - dto "github.com/prometheus/client_model/go" ) // gatherDuckDBMemory returns the exported per-tag memory series, by tag. @@ -32,42 +31,58 @@ func gatherDuckDBMemory(t *testing.T) map[string]float64 { return out } -// The whole purpose of this series is to measure what DuckDB holds right now, -// so that the untracked gap against process RSS can be derived from it. A -// failed read must therefore export nothing. Reporting the last successful -// figures as if they were current is the one outcome that actively misleads: -// the gap is computed against a stale sum, and the number that was supposed to -// reveal an impending out-of-memory kill reads normal. -func TestDuckDBMemoryDropsSeriesWhenTheReadFails(t *testing.T) { - release := SetDuckDBMemorySource(func() map[string]int64 { - return map[string]int64{"BASE_TABLE": 4096} - }) +func TestDuckDBMemoryPublishesCompleteSamplesAndTimestamp(t *testing.T) { + record, release := SetDuckDBMemorySource(time.Minute) + defer release() + if got := gatherDuckDBMemory(t); len(got) != 0 { + t.Fatalf("before the first sample exported %v", got) + } + if got := gatherMemoryTimestamp(t); got != 0 { + t.Fatalf("before the first sample timestamp=%g", got) + } + tags := map[string]int64{"BASE_TABLE": 4096} + at := time.Now() + record(tags, at) + tags["BASE_TABLE"] = 0 // Publication owns an immutable copy. if got := gatherDuckDBMemory(t); got["BASE_TABLE"] != 4096 { - release() t.Fatalf("healthy read exported %v, want BASE_TABLE=4096", got) } - release() + if got := gatherMemoryTimestamp(t); got != float64(at.UnixNano())/1e9 { + t.Fatalf("timestamp=%g, want %v", got, at) + } + record(map[string]int64{"HASH_TABLE": 2048}, time.Now()) + if got := gatherDuckDBMemory(t); len(got) != 1 || got["HASH_TABLE"] != 2048 { + t.Fatalf("replacement sample exported %v", got) + } +} - // The source is installed but cannot read: the closure in internal/query - // returns nil for a query error, a scan error, or a partial read. - release = SetDuckDBMemorySource(func() map[string]int64 { return nil }) +func TestDuckDBMemoryReleaseOnlyClearsItsOwner(t *testing.T) { + recordOld, releaseOld := SetDuckDBMemorySource(time.Minute) + record, release := SetDuckDBMemorySource(time.Minute) defer release() + record(map[string]int64{"BASE_TABLE": 2}, time.Now()) + releaseOld() + recordOld(map[string]int64{"BASE_TABLE": 1}, time.Now()) + if got := gatherDuckDBMemory(t); got["BASE_TABLE"] != 2 { + t.Fatalf("old owner changed the current sample: %v", got) + } + release() if got := gatherDuckDBMemory(t); len(got) != 0 { - t.Errorf("after a failed read the gauge still exports %v; stale values are reported as current", got) + t.Fatalf("released source still exports %v", got) } } -// Prometheus scrapes concurrently, and a GaugeVec that is Reset and refilled on -// every scrape is shared mutable state: two scrapes interleave, and one of them -// observes the other's half-finished refill -- missing tags, and a sum that -// understates what DuckDB holds. func TestDuckDBMemoryIsSafeUnderConcurrentScrapes(t *testing.T) { - release := SetDuckDBMemorySource(func() map[string]int64 { - return map[string]int64{"BASE_TABLE": 1, "HASH_TABLE": 2, "ORDER_BY": 3} - }) + record, release := SetDuckDBMemorySource(time.Minute) defer release() + record(map[string]int64{"BASE_TABLE": 1, "HASH_TABLE": 2, "ORDER_BY": 3}, time.Now()) var wg sync.WaitGroup + wg.Go(func() { + for range 200 { + record(map[string]int64{"BASE_TABLE": 1, "HASH_TABLE": 2, "ORDER_BY": 3}, time.Now()) + } + }) for range 8 { wg.Add(1) go func() { @@ -86,7 +101,6 @@ func TestDuckDBMemoryIsSafeUnderConcurrentScrapes(t *testing.T) { t.Errorf("scrape saw %d tags, want 3: a concurrent refill was observed half-done", n) return } - _ = dto.MetricType_GAUGE } } }() @@ -100,11 +114,34 @@ func TestDuckDBMemoryExportsNothingWithoutASource(t *testing.T) { if got := gatherDuckDBMemory(t); len(got) != 0 { t.Errorf("with no source installed the gauge exports %v", got) } - if families, err := prometheus.DefaultGatherer.Gather(); err == nil { - for _, family := range families { - if strings.Contains(family.GetName(), "duckdb_memory_scrape") { - t.Errorf("%s exists only to pump a push gauge; a collector does not need it", family.GetName()) - } + if got := gatherMemoryTimestamp(t); got != -1 { + t.Errorf("without a source timestamp=%g, want absent", got) + } +} + +func gatherMemoryTimestamp(t *testing.T) float64 { + t.Helper() + families, err := prometheus.DefaultGatherer.Gather() + if err != nil { + t.Fatal(err) + } + for _, family := range families { + if family.GetName() == "fanout_duckdb_memory_last_success_timestamp_seconds" { + return family.Metric[0].GetGauge().GetValue() } } + return -1 +} + +func TestDuckDBMemoryDropsStaleSeriesAndKeepsTimestamp(t *testing.T) { + record, release := SetDuckDBMemorySource(time.Second) + defer release() + at := time.Now().Add(-3 * time.Second) + record(map[string]int64{"BASE_TABLE": 4096}, at) + if got := gatherDuckDBMemory(t); len(got) != 0 { + t.Fatalf("stale sample exported %v", got) + } + if got := gatherMemoryTimestamp(t); got != float64(at.UnixNano())/1e9 { + t.Fatalf("stale sample timestamp=%g, want %v", got, at) + } } diff --git a/internal/metrics/metrics.go b/internal/metrics/metrics.go index f411b98f9..a862cfb5c 100644 --- a/internal/metrics/metrics.go +++ b/internal/metrics/metrics.go @@ -2,8 +2,10 @@ package metrics import ( "database/sql" + "maps" "math" "sync/atomic" + "time" "github.com/prometheus/client_golang/prometheus" "github.com/prometheus/client_golang/prometheus/promauto" @@ -312,15 +314,27 @@ func RecordFlush(signal string, durationSec float64) { FlushDuration.WithLabelValues(signal).Observe(durationSec) } +type duckDBPoolSource struct{ stats func() sql.DBStats } + // duckDBPoolStats is the live source for the connection-pool gauges below. // // A holder rather than a promauto.NewGaugeFunc closure over the pool: the // gauges are declared once for the process, while a *Duck is constructed many // times in a test binary, and registering a collector per construction panics // on the second one. Unset, every gauge reads zero. -var duckDBPoolStats atomic.Pointer[func() sql.DBStats] +var duckDBPoolStats atomic.Pointer[duckDBPoolSource] + +type duckDBMemorySample struct { + tags map[string]int64 + at time.Time +} -// duckDBMemoryStats is the live source for the per-tag DuckDB memory gauge. +type duckDBMemorySource struct { + sample atomic.Pointer[duckDBMemorySample] + maxAge time.Duration +} + +// duckDBMemoryStats holds the latest background sample for the memory gauge. // // memory_limit bounds DuckDB's buffer pool and nothing else, so the figure that // actually predicts an out-of-memory kill is the gap between what the process @@ -329,72 +343,75 @@ var duckDBPoolStats atomic.Pointer[func() sql.DBStats] // - sum(fanout_duckdb_memory_bytes) - go_memstats_heap_sys_bytes. Without it // every diagnosis of this process is an inference, which is how a whole day // got spent optimising a Go heap that was never the problem. -var duckDBMemoryStats atomic.Pointer[func() map[string]int64] +var duckDBMemoryStats atomic.Pointer[duckDBMemorySource] -// SetDuckDBMemorySource installs the reader for the per-tag memory gauge. -func SetDuckDBMemorySource(tags func() map[string]int64) (release func()) { - if tags == nil { - return func() {} - } - installed := &tags +// SetDuckDBMemorySource installs a cache, never a scrape-time query. Record +// publishes a complete successful sample; failures leave the last one intact. +// Releasing an older Duck cannot clear a newer Duck's source. +func SetDuckDBMemorySource(interval time.Duration) (record func(map[string]int64, time.Time), release func()) { + installed := &duckDBMemorySource{maxAge: 2 * interval} duckDBMemoryStats.Store(installed) - return func() { duckDBMemoryStats.CompareAndSwap(installed, nil) } + return func(tags map[string]int64, at time.Time) { + installed.sample.Store(&duckDBMemorySample{tags: maps.Clone(tags), at: at}) + }, func() { duckDBMemoryStats.CompareAndSwap(installed, nil) } } -// duckDBMemoryDesc is emitted by a Collector, not held in a GaugeVec. -// -// The obvious implementation -- a GaugeVec that a scrape hook Resets and -// refills -- is shared mutable state across concurrent scrapes, and a scrape -// that lands mid-refill exports a subset of the tags. That understates what -// DuckDB holds and correspondingly overstates the untracked gap, which is the -// single number this series exists to compute. It also leaves the last good -// values in place when a read fails, reporting them as current: the reading -// that was meant to reveal an impending out-of-memory kill then looks normal. -// -// A Collector has neither problem. Each scrape builds its own metrics from its -// own read, and a read that returns nothing exports nothing -- the series goes -// absent, which is visibly different from "DuckDB is holding nothing". +// Both series come from one immutable sample, so concurrent scrapes never see +// a partial tag set or a timestamp belonging to a different sample. var duckDBMemoryDesc = prometheus.NewDesc( "fanout_duckdb_memory_bytes", - "DuckDB memory usage by tag, as DuckDB itself accounts for it", + "Last sampled DuckDB memory usage by tag, as DuckDB itself accounts for it", []string{"tag"}, nil, ) +var duckDBMemorySampleDesc = prometheus.NewDesc( + "fanout_duckdb_memory_last_success_timestamp_seconds", + "Unix timestamp of the last successful DuckDB memory sample, or zero before the first success", + nil, nil, +) + type duckDBMemoryCollector struct{} -func (duckDBMemoryCollector) Describe(ch chan<- *prometheus.Desc) { ch <- duckDBMemoryDesc } +func (duckDBMemoryCollector) Describe(ch chan<- *prometheus.Desc) { + ch <- duckDBMemoryDesc + ch <- duckDBMemorySampleDesc +} func (duckDBMemoryCollector) Collect(ch chan<- prometheus.Metric) { - fn := duckDBMemoryStats.Load() - if fn == nil { + source := duckDBMemoryStats.Load() + if source == nil { return } - for tag, bytes := range (*fn)() { - ch <- prometheus.MustNewConstMetric(duckDBMemoryDesc, prometheus.GaugeValue, float64(bytes), tag) + at := 0.0 + if sample := source.sample.Load(); sample != nil { + at = float64(sample.at.UnixNano()) / 1e9 + if time.Since(sample.at) <= source.maxAge { + for tag, bytes := range sample.tags { + ch <- prometheus.MustNewConstMetric(duckDBMemoryDesc, prometheus.GaugeValue, float64(bytes), tag) + } + } } + ch <- prometheus.MustNewConstMetric(duckDBMemorySampleDesc, prometheus.GaugeValue, at) } func init() { prometheus.DefaultRegisterer.MustRegister(duckDBMemoryCollector{}) } -// SetDuckDBPoolSource points the connection-pool gauges at a pool and returns -// the function that stops reading it. The last caller wins; the returned -// release only clears the source if it is still the one it installed, so a -// second Duck closing in a test binary cannot blank the gauges of one that is -// still serving. +// SetDuckDBPoolSource reads database/sql bookkeeping on each scrape without +// querying the engine. Releasing an older Duck cannot clear a newer source. func SetDuckDBPoolSource(stats func() sql.DBStats) (release func()) { if stats == nil { return func() {} } - installed := &stats + installed := &duckDBPoolSource{stats: stats} duckDBPoolStats.Store(installed) return func() { duckDBPoolStats.CompareAndSwap(installed, nil) } } func poolStats() sql.DBStats { - if fn := duckDBPoolStats.Load(); fn != nil { - return (*fn)() + if source := duckDBPoolStats.Load(); source != nil { + return source.stats() } return sql.DBStats{} } @@ -571,8 +588,8 @@ type compactionBacklogSource struct{ count func() int } var compactionBacklog atomic.Pointer[compactionBacklogSource] -// SetCompactionBacklogSource reads the exact inventory rather than a periodic -// file-stat snapshot. Closing an older repository cannot clear a newer source. +// SetCompactionBacklogSource reads the cached exact inventory. The source must +// not take the catalog lock. Closing an older repository cannot clear a newer source. func SetCompactionBacklogSource(count func() int) func() { source := &compactionBacklogSource{count: count} compactionBacklog.Store(source) diff --git a/internal/query/duck.go b/internal/query/duck.go index 2465ce9c9..dca88af76 100644 --- a/internal/query/duck.go +++ b/internal/query/duck.go @@ -32,7 +32,7 @@ type Duck struct { // DB is the read pool. Reads scan immutable Parquet concurrently, so it is // sized from the machine. DB *sql.DB - // closeOnce makes Close idempotent. Both handles come from one connector, + // closeOnce makes Close idempotent. All handles come from one connector, // and database/sql closes a connector that implements io.Closer — so a // second Close would call duckdb_close twice on the same instance, which // is a double free the runtime cannot recover from. @@ -42,6 +42,11 @@ type Duck struct { // only if they are still reading it. releasePoolGauges func() releaseMemoryGauges func() + stopMemorySampler context.CancelFunc + memorySamplerDone chan struct{} + // memoryDB has one borrowed connection so a blocked memory query cannot + // occupy a request or write slot. The read pool owns the shared instance. + memoryDB *sql.DB // writeDB has two connections: analytical rollups share writeGate, while // completed-batch aggregates use cacheGate and disjoint tables. Both stay // outside the read pool. Maintenance acquires both gates for checkpointing. @@ -244,39 +249,13 @@ func NewDuck(ctx context.Context, cfg config.Config, repository *telemetrystore. return nil, err } - db, writeDB, err := openDuckDB(ctx, dsn, tempDir, duckDBPoolSize(cfg)) + db, writeDB, memoryDB, err := openDuckDB(dsn, tempDir, duckDBPoolSize(cfg)) if err != nil { return nil, fmt.Errorf("open DuckDB query cache: %w (the cache at %s is rebuildable from Parquet)", err, dbPath) } - d := &Duck{DB: db, writeDB: writeDB, cfg: cfg, repository: repository, publicationBounds: repository.PublicationBounds} + d := &Duck{DB: db, writeDB: writeDB, memoryDB: memoryDB, cfg: cfg, repository: repository, publicationBounds: repository.PublicationBounds} // Reads are the pool that can starve, so it is the one worth watching. d.releasePoolGauges = metrics.SetDuckDBPoolSource(db.Stats) - // And what DuckDB admits to holding, so the gap against process RSS is - // observable rather than inferred. - d.releaseMemoryGauges = metrics.SetDuckDBMemorySource(func() map[string]int64 { - ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) - defer cancel() - rows, err := db.QueryContext(ctx, "SELECT tag, sum(memory_usage_bytes) FROM duckdb_memory() GROUP BY tag") - if err != nil { - return nil - } - defer rows.Close() - tags := make(map[string]int64, 16) - for rows.Next() { - var tag string - var bytes int64 - if err := rows.Scan(&tag, &bytes); err != nil { - return nil - } - tags[tag] = bytes - } - // A partial read is worse than no read: it publishes a low sum, and - // the untracked figure derived from it reads correspondingly high. - if err := rows.Err(); err != nil { - return nil - } - return tags - }) if cfg.DuckDBMemory == "" { // Only when the operator hasn't pinned FANOUT_DUCKDB_MEMORY: keep DuckDB's // cgroup-aware auto limit on big boxes but leave absolute RAM headroom on @@ -316,6 +295,7 @@ func NewDuck(ctx context.Context, cfg config.Config, repository *telemetrystore. slog.Info("rollup skip-to-latest: existing data marked already-rolled-up") } } + d.startMemorySampler(duckDBMemorySampleInterval, d.sampleMemory) return d, nil } @@ -357,14 +337,15 @@ func (d *Duck) skipRollupToLatest(ctx context.Context) error { return nil } -// openDuckDB returns the read pool and the single-connection write handle. +// openDuckDB returns the read pool, the two-connection write handle, and the +// single-connection memory sampling handle. // -// Both come from one connector so they share one DuckDB instance and therefore +// All come from one connector so they share one DuckDB instance and therefore // one catalog: a write committed on one handle is visible to the other, and // DDL run at boot applies to both. (A second connector to the same file would // also work — DuckDB keeps an instance cache — but two owners of one instance // is exactly the hazard the wrapper below exists to avoid.) -func openDuckDB(ctx context.Context, dsn, tempDir string, maxConns int) (*sql.DB, *sql.DB, error) { +func openDuckDB(dsn, tempDir string, maxConns int) (*sql.DB, *sql.DB, *sql.DB, error) { // Every pooled connection must use UTC. DuckDB otherwise inherits the host // timezone and casts TIMESTAMPTZ rollup buckets into local wall-clock // TIMESTAMP values, while API windows arrive in UTC. @@ -373,11 +354,15 @@ func openDuckDB(ctx context.Context, dsn, tempDir string, maxConns int) (*sql.DB // dir has already been used fails with "Cannot switch temporary directory // after the current one has been used". The boot hook runs once per pooled // connection, so only the first connection sets it; the rest inherit the - // instance-wide value. LOAD/ATTACH stay per-connection — they're idempotent. + // instance-wide value. var tempDirOnce sync.Once var tempDirErr error - connector, err := duckdb.NewConnector(dsn, func(execer driver.ExecerContext) error { + native, err := duckdb.NewConnector(dsn, nil) + if err != nil { + return nil, nil, nil, err + } + connector := &initializedConnector{Connector: native, init: func(ctx context.Context, execer driver.ExecerContext) error { if _, err := execer.ExecContext(ctx, "SET TimeZone='UTC'", nil); err != nil { return fmt.Errorf("set timezone: %w", err) } @@ -391,10 +376,7 @@ func openDuckDB(ctx context.Context, dsn, tempDir string, maxConns int) (*sql.DB return fmt.Errorf("set temp_directory: %w", tempDirErr) } return nil - }) - if err != nil { - return nil, nil, err - } + }} db := sql.OpenDB(connector) // The on-disk DuckDB file contains only rebuildable rollups and views; Parquet @@ -418,45 +400,89 @@ func openDuckDB(ctx context.Context, dsn, tempDir string, maxConns int) (*sql.DB writeDB := sql.OpenDB(borrowedConnector{connector}) writeDB.SetMaxOpenConns(2) writeDB.SetMaxIdleConns(2) - return db, writeDB, nil + // Memory sampling can block inside the engine despite its deadline. Keep + // its one connection separate from both request and write capacity. + memoryDB := sql.OpenDB(borrowedConnector{connector}) + memoryDB.SetMaxOpenConns(1) + memoryDB.SetMaxIdleConns(1) + return db, writeDB, memoryDB, nil } // borrowedConnector hands out connections from a connector it does not own. // It deliberately does not implement io.Closer: that is the whole point. type borrowedConnector struct{ driver.Connector } +// Initialize with each Connect call's context, never the startup context: +// database/sql may open replacement connections long after startup finishes. +type initializedConnector struct { + *duckdb.Connector + init func(context.Context, driver.ExecerContext) error +} + +func (c *initializedConnector) Connect(ctx context.Context) (driver.Conn, error) { + conn, err := c.Connector.Connect(ctx) + if err != nil { + return nil, err + } + if err := c.init(ctx, conn.(driver.ExecerContext)); err != nil { + return nil, errors.Join(err, conn.Close()) + } + return conn, nil +} + func sqlLiteral(v string) string { return "'" + strings.ReplaceAll(v, "'", "''") + "'" } func (d *Duck) Close() error { d.closeOnce.Do(func() { + if d.stopMemorySampler != nil { + d.stopMemorySampler() + } if d.releasePoolGauges != nil { d.releasePoolGauges() } - // Same reason as the pool source: the closure captures this Duck's - // read pool, so leaving it installed past Close means the next scrape - // queries a closed pool and the gauges freeze at stale values that - // still look live. if d.releaseMemoryGauges != nil { d.releaseMemoryGauges() } - var errs []error - if d.writeDB != nil { - // The write handle holds a connector that does not close the - // instance, so this releases its connection and nothing else. - errs = append(errs, d.writeDB.Close()) - } - // A Duck assembled field by field, as tests do, may hold neither - // handle; closing one is then nothing to do rather than a panic. - if d.DB != nil { - errs = append(errs, d.DB.Close()) + if d.stopMemorySampler != nil { + timer := time.NewTimer(time.Second) + defer timer.Stop() + select { + case <-d.memorySamplerDone: + case <-timer.C: + // The engine may ignore cancellation. Keep all handles and the + // owning connector alive until the abandoned query returns; + // closing the instance underneath it can crash native code. + go func() { + <-d.memorySamplerDone + if err := d.closeHandles(); err != nil { + slog.Warn("close DuckDB after memory sampler stopped", "err", err) + } + }() + return + } } - d.closeErr = errors.Join(errs...) + d.closeErr = d.closeHandles() }) return d.closeErr } +func (d *Duck) closeHandles() error { + var errs []error + if d.memoryDB != nil { + errs = append(errs, d.memoryDB.Close()) + } + if d.writeDB != nil { + errs = append(errs, d.writeDB.Close()) + } + // The read pool owns the instance and must close after its borrowers. + if d.DB != nil { + errs = append(errs, d.DB.Close()) + } + return errors.Join(errs...) +} + // writer returns the dedicated write handle initialized by NewDuck. func (d *Duck) writer() *sql.DB { return d.writeDB } diff --git a/internal/query/duck_memory.go b/internal/query/duck_memory.go new file mode 100644 index 000000000..8405924d8 --- /dev/null +++ b/internal/query/duck_memory.go @@ -0,0 +1,77 @@ +package query + +import ( + "context" + "log/slog" + "time" + + "github.com/labstack/fanout/internal/metrics" +) + +const duckDBMemorySampleInterval = 5 * time.Second + +// startMemorySampler keeps engine calls off the scrape path. There is only one +// sampling goroutine, even if DuckDB ignores cancellation inside a statement. +// Its lifetime belongs to Duck, rather than the context used to open it. +func (d *Duck) startMemorySampler(interval time.Duration, sample func(context.Context) (map[string]int64, error)) { + ctx, cancel := context.WithCancel(context.Background()) + d.stopMemorySampler = cancel + d.memorySamplerDone = make(chan struct{}) + record, release := metrics.SetDuckDBMemorySource(interval) + d.releaseMemoryGauges = release + go func() { + defer close(d.memorySamplerDone) + timer := time.NewTimer(interval) + defer timer.Stop() + failures := 0 + for { + if ctx.Err() != nil { + return + } + readCtx, readCancel := context.WithTimeout(ctx, interval) + tags, err := sample(readCtx) + readCancel() + if ctx.Err() != nil { + return + } + if err != nil { + failures++ + if failures == 3 { + slog.Warn("DuckDB memory sampling failed repeatedly; retaining last successful sample", "err", err) + } + } else { + record(tags, time.Now()) + failures = 0 + } + // Pace from completion, including slow calls that ignore the deadline. + timer.Reset(interval) + select { + case <-ctx.Done(): + return + case <-timer.C: + } + } + }() +} + +func (d *Duck) sampleMemory(ctx context.Context) (map[string]int64, error) { + rows, err := d.memoryDB.QueryContext(ctx, "SELECT tag, sum(memory_usage_bytes) FROM duckdb_memory() GROUP BY tag") + if err != nil { + return nil, err + } + defer rows.Close() + tags := make(map[string]int64, 16) + for rows.Next() { + var tag string + var bytes int64 + if err := rows.Scan(&tag, &bytes); err != nil { + return nil, err + } + tags[tag] = bytes + } + // Publish only complete samples: a partial sum misstates untracked memory. + if err := rows.Err(); err != nil { + return nil, err + } + return tags, nil +} diff --git a/internal/query/duck_memory_test.go b/internal/query/duck_memory_test.go new file mode 100644 index 000000000..c928bad3a --- /dev/null +++ b/internal/query/duck_memory_test.go @@ -0,0 +1,366 @@ +package query + +import ( + "bytes" + "context" + "errors" + "log/slog" + "net/http" + "net/http/httptest" + "strconv" + "strings" + "sync/atomic" + "testing" + "testing/synctest" + "time" + + "github.com/DATA-DOG/go-sqlmock" + "github.com/labstack/fanout/internal/config" + telemetrystore "github.com/labstack/fanout/internal/telemetry/store" + "github.com/prometheus/client_golang/prometheus" + "github.com/prometheus/client_golang/prometheus/promhttp" +) + +func TestDuckDBMemoryDropsSeriesWhenTheReadFails(t *testing.T) { + synctest.Test(t, func(t *testing.T) { + d := &Duck{} + defer d.Close() + calls := 0 + d.startMemorySampler(time.Second, func(context.Context) (map[string]int64, error) { + calls++ + if calls == 1 { + return map[string]int64{"BASE_TABLE": 4096}, nil + } + return nil, errors.New("memory query failed") + }) + synctest.Wait() + if memory, timestamp := memoryMetrics(t); memory != 4096 || timestamp <= 0 { + t.Fatalf("first sample: memory=%g, timestamp=%g", memory, timestamp) + } + time.Sleep(2500 * time.Millisecond) + synctest.Wait() + if memory, timestamp := memoryMetrics(t); memory != -1 || timestamp <= 0 { + t.Fatalf("failed reads: memory=%g, timestamp=%g", memory, timestamp) + } + }) +} + +func TestMemorySamplerBlockedSampleAgesOut(t *testing.T) { + synctest.Test(t, func(t *testing.T) { + d := &Duck{} + unblock := make(chan struct{}) + defer func() { + close(unblock) + d.Close() + }() + calls := 0 + d.startMemorySampler(time.Second, func(ctx context.Context) (map[string]int64, error) { + calls++ + if calls == 1 { + return map[string]int64{"BASE_TABLE": 4096}, nil + } + <-unblock + return nil, ctx.Err() + }) + time.Sleep(time.Second) + synctest.Wait() + memory, timestamp := memoryMetrics(t) + if memory != 4096 || timestamp <= 0 { + t.Fatalf("fresh sample: memory=%g, timestamp=%g", memory, timestamp) + } + time.Sleep(time.Second) + synctest.Wait() + if memory, _ := memoryMetrics(t); memory != 4096 { + t.Fatalf("sample exactly two intervals old: memory=%g", memory) + } + time.Sleep(500 * time.Millisecond) + synctest.Wait() + if memory, lastSuccess := memoryMetrics(t); memory != -1 || lastSuccess != timestamp { + t.Fatalf("hung sampler: memory=%g, timestamp=%g, want absent memory and timestamp=%g", memory, lastSuccess, timestamp) + } + if calls != 2 { + t.Fatalf("blocked sampler made %d calls, want 2", calls) + } + }) +} + +func TestMemorySamplerUsesIntervalDeadlineAndPacesFromCompletion(t *testing.T) { + synctest.Test(t, func(t *testing.T) { + d := &Duck{} + defer d.Close() + const interval = time.Second + var calls atomic.Int32 + var finished time.Time + d.startMemorySampler(interval, func(ctx context.Context) (map[string]int64, error) { + if calls.Add(1) > 1 { + if gap := time.Since(finished); gap != interval { + t.Errorf("next sample started after %s, want %s", gap, interval) + } + <-ctx.Done() + return nil, ctx.Err() + } + deadline, ok := ctx.Deadline() + if !ok || time.Until(deadline) != interval { + t.Errorf("read deadline=%v, want one interval from now", deadline) + } + // A successful query may return after its deadline; keep its result. + <-ctx.Done() + time.Sleep(interval / 2) + finished = time.Now() + return map[string]int64{"BASE_TABLE": 4096}, nil + }) + time.Sleep(2 * interval) + synctest.Wait() + if memory, _ := memoryMetrics(t); memory != 4096 { + t.Fatalf("successful overdue sample discarded: memory=%g", memory) + } + if calls.Load() != 1 { + t.Fatalf("sampler made %d calls before the next interval", calls.Load()) + } + time.Sleep(interval / 2) + synctest.Wait() + if calls.Load() != 2 { + t.Fatalf("sampler made %d calls, want 2", calls.Load()) + } + }) +} + +func TestDuckCloseBoundsWaitAndDefersHandleCleanup(t *testing.T) { + synctest.Test(t, func(t *testing.T) { + db, mock, err := sqlmock.New(sqlmock.MonitorPingsOption(true)) + if err != nil { + t.Fatal(err) + } + d := &Duck{DB: db} + unblock := make(chan struct{}) + d.startMemorySampler(time.Hour, func(ctx context.Context) (map[string]int64, error) { + <-unblock // Ignore cancellation, then use the handle before returning. + if err := db.PingContext(context.Background()); err != nil { + t.Errorf("sampler's handle closed while still in use: %v", err) + } + return nil, ctx.Err() + }) + synctest.Wait() + start := time.Now() + if err := d.Close(); err != nil { + t.Fatal(err) + } + if elapsed := time.Since(start); elapsed != time.Second { + t.Fatalf("Close waited %s, want 1s", elapsed) + } + if err := d.Close(); err != nil { + t.Fatal(err) + } + mock.ExpectPing() + mock.ExpectClose() + close(unblock) + synctest.Wait() + if err := mock.ExpectationsWereMet(); err != nil { + t.Fatal(err) + } + }) +} + +// A missing memory series is distinct from a real zero-byte sample. +func memoryMetrics(t *testing.T) (memory, timestamp float64) { + t.Helper() + memory, timestamp = -1, -1 + families, err := prometheus.DefaultGatherer.Gather() + if err != nil { + t.Fatal(err) + } + for _, family := range families { + switch family.GetName() { + case "fanout_duckdb_memory_bytes": + memory = family.Metric[0].GetGauge().GetValue() + case "fanout_duckdb_memory_last_success_timestamp_seconds": + timestamp = family.Metric[0].GetGauge().GetValue() + } + } + return +} + +func TestMetricsScrapeDropsStaleMemoryWhileSamplerBlocks(t *testing.T) { + var log bytes.Buffer + logger := slog.Default() + slog.SetDefault(slog.New(slog.NewTextHandler(&log, nil))) + defer slog.SetDefault(logger) + + d := &Duck{} + blocked := make(chan struct{}) + unblock := make(chan struct{}) + t.Cleanup(func() { + d.stopMemorySampler() + close(unblock) + if err := d.Close(); err != nil { + t.Error(err) + } + }) + calls := 0 + d.startMemorySampler(10*time.Millisecond, func(ctx context.Context) (map[string]int64, error) { + if _, ok := ctx.Deadline(); !ok { + return nil, errors.New("missing sampling deadline") + } + calls++ + switch { + case calls == 1: + return map[string]int64{"BASE_TABLE": 4096, "HASH_TABLE": 2048}, nil + case calls <= 5: + return nil, errors.New("memory query failed") + default: + close(blocked) + // Model an engine call that ignores cancellation, as observed in + // the benchmark. Scrapes must not wait for it or spawn more calls. + <-unblock + return nil, ctx.Err() + } + }) + select { + case <-blocked: + case <-time.After(5 * time.Second): + t.Fatal("sampler did not reach the blocking query") + } + if n := strings.Count(log.String(), "memory sampling failed repeatedly"); n != 1 { + t.Fatalf("repeated failures logged %d warnings, want 1: %s", n, log.String()) + } + + handler := promhttp.Handler() + var timestamp string + for range 3 { + recorder := httptest.NewRecorder() + done := make(chan struct{}) + start := time.Now() + go func() { + handler.ServeHTTP(recorder, httptest.NewRequest(http.MethodGet, "/metrics", nil)) + close(done) + }() + select { + case <-done: + case <-time.After(100 * time.Millisecond): + t.Fatal("/metrics waited for the blocked memory query") + } + t.Logf("/metrics returned in %s with the sampler blocked", time.Since(start)) + if recorder.Code != http.StatusOK { + t.Fatalf("/metrics status=%d: %s", recorder.Code, recorder.Body.String()) + } + body := recorder.Body.String() + if strings.Contains(body, "fanout_duckdb_memory_bytes") { + t.Fatal("blocked sampler still exports stale memory series") + } + const prefix = "fanout_duckdb_memory_last_success_timestamp_seconds " + for _, line := range strings.Split(body, "\n") { + if strings.HasPrefix(line, prefix) { + value := strings.TrimPrefix(line, prefix) + at, err := strconv.ParseFloat(value, 64) + if err != nil || at <= 0 { + t.Fatalf("invalid last-success timestamp: %s", line) + } + if timestamp != "" && value != timestamp { + t.Fatal("blocked sampler advanced its last-success timestamp") + } + timestamp = value + } + } + if timestamp == "" { + t.Fatal("/metrics did not expose sample freshness") + } + } +} + +func TestDuckCloseCancelsAndJoinsMemorySampler(t *testing.T) { + d := &Duck{} + started := make(chan struct{}) + stopped := make(chan struct{}) + d.startMemorySampler(time.Hour, func(ctx context.Context) (map[string]int64, error) { + close(started) + <-ctx.Done() + close(stopped) + return nil, ctx.Err() + }) + <-started + if err := d.Close(); err != nil { + t.Fatal(err) + } + select { + case <-stopped: + default: + t.Fatal("Close returned before the sampler stopped") + } + if err := d.Close(); err != nil { + t.Fatal(err) + } +} + +func TestMemorySamplingDoesNotNeedReadOrWritePoolCapacity(t *testing.T) { + ctx, cancel := context.WithTimeout(t.Context(), 5*time.Second) + defer cancel() + cfg := config.Config{DataDir: t.TempDir(), DuckDBMemory: "128MB", DuckDBMaxConns: 1} + repository, err := telemetrystore.Open(cfg.TelemetryDir()) + if err != nil { + t.Fatal(err) + } + defer repository.Close() + d, err := NewDuck(ctx, cfg, repository) + if err != nil { + t.Fatal(err) + } + defer func() { + if err := d.Close(); err != nil { + t.Error(err) + } + if open := d.memoryDB.Stats().OpenConnections; open != 0 { + t.Errorf("Close left %d memory connections open", open) + } + }() + // Sample synchronously below so no timer determines which query runs. + d.stopMemorySampler() + <-d.memorySamplerDone + if got := d.memoryDB.Stats().MaxOpenConnections; got != 1 { + t.Fatalf("memory handle MaxOpenConnections = %d, want 1", got) + } + + // Exhaust both application pools. Sampling must still reach the engine. + read, err := d.DB.Conn(ctx) + if err != nil { + t.Fatal(err) + } + defer read.Close() + for range 2 { + write, err := d.writeDB.Conn(ctx) + if err != nil { + t.Fatal(err) + } + defer write.Close() + } + tags, err := d.sampleMemory(ctx) + if err != nil { + t.Fatalf("sample with read and write pools exhausted: %v", err) + } + if len(tags) == 0 { + t.Fatal("memory sample contains no tags") + } + if inUse := d.memoryDB.Stats().InUse; inUse != 0 { + t.Fatalf("memory query retained %d connections after reading rows", inUse) + } +} + +func TestMemorySamplerRejectsPartialQueryResults(t *testing.T) { + db, mock, err := sqlmock.New() + if err != nil { + t.Fatal(err) + } + defer db.Close() + mock.ExpectQuery("SELECT tag, sum").WillReturnRows( + sqlmock.NewRows([]string{"tag", "bytes"}). + AddRow("BASE_TABLE", 4096). + AddRow("HASH_TABLE", 2048). + RowError(1, errors.New("read failed")), + ) + d := &Duck{memoryDB: db} + if tags, err := d.sampleMemory(t.Context()); err == nil || tags != nil { + t.Fatalf("partial query returned tags=%v, err=%v", tags, err) + } + if err := mock.ExpectationsWereMet(); err != nil { + t.Fatal(err) + } +} diff --git a/internal/query/duck_test.go b/internal/query/duck_test.go index 67b53a6f9..3a0319d5a 100644 --- a/internal/query/duck_test.go +++ b/internal/query/duck_test.go @@ -221,6 +221,39 @@ WHERE bucket >= ? AND bucket < ?`, eventTime.Add(-time.Minute), eventTime.Add(ti } } +func TestNewDuckReconnectsInUTCAfterStartupContextCancelled(t *testing.T) { + t.Setenv("TZ", "America/Los_Angeles") + ctx, cancel := context.WithCancel(t.Context()) + defer cancel() + cfg := config.Config{DataDir: t.TempDir(), DuckDBMemory: "128MB"} + repository, err := telemetrystore.Open(cfg.TelemetryDir()) + if err != nil { + t.Fatal(err) + } + defer repository.Close() + d, err := NewDuck(ctx, cfg, repository) + if err != nil { + t.Fatal(err) + } + defer d.Close() + d.stopMemorySampler() + <-d.memorySamplerDone + cancel() + for name, db := range map[string]*sql.DB{"read": d.DB, "write": d.writeDB, "memory": d.memoryDB} { + t.Run(name, func(t *testing.T) { + // Retire every existing connection so this query must reconnect. + db.SetMaxIdleConns(0) + var timezone string + if err := db.QueryRowContext(t.Context(), "SELECT current_setting('TimeZone')").Scan(&timezone); err != nil { + t.Fatal(err) + } + if timezone != "UTC" { + t.Fatalf("new connection timezone=%q, want UTC", timezone) + } + }) + } +} + // A cancelled statement's interrupt can reach the ROLLBACK that ends its read // transaction, leaving the transaction open. The read must roll back itself // after cancellation, and a connection whose rollback failed must leave the diff --git a/internal/query/stall_instrumentation_test.go b/internal/query/stall_instrumentation_test.go index 2192c6693..ff0e1b735 100644 --- a/internal/query/stall_instrumentation_test.go +++ b/internal/query/stall_instrumentation_test.go @@ -42,6 +42,13 @@ func TestWritesDoNotDrawFromTheReadPool(t *testing.T) { if d.writer() != d.writeDB { t.Fatal("writes are not going through the write handle") } + // Occupy the sampler's dedicated slot throughout the assertions. Even a + // blocked memory query must leave request and write capacity untouched. + memory, err := d.memoryDB.Conn(ctx) + if err != nil { + t.Fatalf("hold memory connection: %v", err) + } + defer memory.Close() // Use the read pool first, so its idle count is non-zero and the assertion // below is about where the write went rather than about a pool nobody has @@ -153,8 +160,9 @@ func TestPublisherWaitingTracksFreshAndCanceledPublication(t *testing.T) { // closed, and duckdb's connector closes the instance behind an unsynchronized // bool. Two handles over one connector therefore meant two duckdb_close calls // on one instance — a double free that kills the process rather than returning -// an error. The write handle borrows the instance; the read pool owns it. -func TestClosingTheWriteHandleLeavesTheInstanceAlive(t *testing.T) { +// an error. The write and memory handles borrow the instance; the read pool +// owns it. +func TestClosingBorrowedHandlesLeavesTheInstanceAlive(t *testing.T) { ctx := context.Background() cfg := config.Config{DataDir: t.TempDir(), RollupInterval: time.Minute, DuckDBMemory: "128MB"} repository, err := telemetrystore.Open(cfg.TelemetryDir()) @@ -166,13 +174,19 @@ func TestClosingTheWriteHandleLeavesTheInstanceAlive(t *testing.T) { if err != nil { t.Fatalf("NewDuck: %v", err) } + defer d.Close() + d.stopMemorySampler() + <-d.memorySamplerDone if err := d.writeDB.Close(); err != nil { t.Fatalf("close write handle: %v", err) } + if err := d.memoryDB.Close(); err != nil { + t.Fatalf("close memory handle: %v", err) + } var one int if err := d.DB.QueryRowContext(ctx, "SELECT 1").Scan(&one); err != nil { - t.Fatalf("read pool died with the write handle: %v", err) + t.Fatalf("read pool died with the borrowed handles: %v", err) } // Close is idempotent, including under a second shutdown path arriving at diff --git a/internal/telemetry/compaction_debt.go b/internal/telemetry/compaction_debt.go index 7f86943f7..a38ed5a83 100644 --- a/internal/telemetry/compaction_debt.go +++ b/internal/telemetry/compaction_debt.go @@ -16,11 +16,10 @@ const ( ) // Generation0BatchCount is the exact live ingest-batch inventory, including -// terminal oversized batches and underfilled day groups. No metadata scan. +// terminal oversized batches and underfilled day groups. The cached count +// lets metrics read it without waiting for the catalog lock or a metadata scan. func (p *ParquetStore) Generation0BatchCount() int { - p.mu.RLock() - defer p.mu.RUnlock() - return p.generation0Batches + return int(p.generation0Snapshot.Load()) } // CompactionDebt returns the reclaimable generation-0 backlog, excluding @@ -74,6 +73,7 @@ func (p *ParquetStore) setBatchLocked(id string, batch *storedBatch) { p.batches[id] = batch p.changeGeneration0Locked(batch.metadata, 1) } + p.generation0Snapshot.Store(int64(p.generation0Batches)) if p.compactionDebt < debtBefore && p.debtChanged != nil { close(p.debtChanged) p.debtChanged = nil diff --git a/internal/telemetry/compaction_debt_test.go b/internal/telemetry/compaction_debt_test.go index 05ffa280a..892ad86eb 100644 --- a/internal/telemetry/compaction_debt_test.go +++ b/internal/telemetry/compaction_debt_test.go @@ -4,10 +4,41 @@ import ( "context" "errors" "fmt" + "net/http" + "net/http/httptest" + "strings" "testing" "time" + + "github.com/labstack/fanout/internal/metrics" + "github.com/prometheus/client_golang/prometheus/promhttp" ) +func TestMetricsScrapeDoesNotWaitForCatalogLock(t *testing.T) { + p := &ParquetStore{batches: make(map[string]*storedBatch)} + p.installBatch(&storedBatch{metadata: BatchMetadata{ID: "a", Generation: 0, Spans: 1}}) + release := metrics.SetCompactionBacklogSource(p.Generation0BatchCount) + defer release() + + p.mu.Lock() + defer p.mu.Unlock() + recorder := httptest.NewRecorder() + done := make(chan struct{}) + handler := promhttp.Handler() + go func() { + handler.ServeHTTP(recorder, httptest.NewRequest(http.MethodGet, "/metrics", nil)) + close(done) + }() + select { + case <-done: + case <-time.After(100 * time.Millisecond): + t.Fatal("/metrics waited for the catalog lock") + } + if recorder.Code != http.StatusOK || !strings.Contains(recorder.Body.String(), "fanout_compaction_backlog_batches 1\n") { + t.Fatalf("scrape did not return the cached backlog: %d %s", recorder.Code, recorder.Body.String()) + } +} + func TestCompactionLimitNotificationDoesNotMissCrossings(t *testing.T) { p := &ParquetStore{batches: make(map[string]*storedBatch)} for i := range CompactionDebtLimit + CompactionBatchLimit - 2 { diff --git a/internal/telemetry/parquet.go b/internal/telemetry/parquet.go index 332b0f4b0..1f33d0c4f 100644 --- a/internal/telemetry/parquet.go +++ b/internal/telemetry/parquet.go @@ -14,6 +14,7 @@ import ( "sort" "strings" "sync" + "sync/atomic" "time" "github.com/parquet-go/parquet-go" @@ -91,10 +92,12 @@ type ParquetStore struct { publishGate chan struct{} batches map[string]*storedBatch generation0Batches int - generation0ByDay map[int64]int - compactionDebt int - debtChanged chan struct{} - debtLimitReached chan struct{} + // Published after each catalog mutation; scrapes never take mu. + generation0Snapshot atomic.Int64 + generation0ByDay map[int64]int + compactionDebt int + debtChanged chan struct{} + debtLimitReached chan struct{} } type ParquetStats struct {