Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
99 changes: 68 additions & 31 deletions internal/metrics/duckdb_memory_test.go
Original file line number Diff line number Diff line change
@@ -1,12 +1,11 @@
package metrics

import (
"strings"
"sync"
"testing"
"time"

"github.com/prometheus/client_golang/prometheus"
dto "github.com/prometheus/client_model/go"
)

// gatherDuckDBMemory returns the exported per-tag memory series, by tag.
Expand All @@ -32,42 +31,58 @@ func gatherDuckDBMemory(t *testing.T) map[string]float64 {
return out
}

// The whole purpose of this series is to measure what DuckDB holds right now,
// so that the untracked gap against process RSS can be derived from it. A
// failed read must therefore export nothing. Reporting the last successful
// figures as if they were current is the one outcome that actively misleads:
// the gap is computed against a stale sum, and the number that was supposed to
// reveal an impending out-of-memory kill reads normal.
func TestDuckDBMemoryDropsSeriesWhenTheReadFails(t *testing.T) {
release := SetDuckDBMemorySource(func() map[string]int64 {
return map[string]int64{"BASE_TABLE": 4096}
})
func TestDuckDBMemoryPublishesCompleteSamplesAndTimestamp(t *testing.T) {
record, release := SetDuckDBMemorySource(time.Minute)
defer release()
if got := gatherDuckDBMemory(t); len(got) != 0 {
t.Fatalf("before the first sample exported %v", got)
}
if got := gatherMemoryTimestamp(t); got != 0 {
t.Fatalf("before the first sample timestamp=%g", got)
}
tags := map[string]int64{"BASE_TABLE": 4096}
at := time.Now()
record(tags, at)
tags["BASE_TABLE"] = 0 // Publication owns an immutable copy.
if got := gatherDuckDBMemory(t); got["BASE_TABLE"] != 4096 {
release()
t.Fatalf("healthy read exported %v, want BASE_TABLE=4096", got)
}
release()
if got := gatherMemoryTimestamp(t); got != float64(at.UnixNano())/1e9 {
t.Fatalf("timestamp=%g, want %v", got, at)
}
record(map[string]int64{"HASH_TABLE": 2048}, time.Now())
if got := gatherDuckDBMemory(t); len(got) != 1 || got["HASH_TABLE"] != 2048 {
t.Fatalf("replacement sample exported %v", got)
}
}

// The source is installed but cannot read: the closure in internal/query
// returns nil for a query error, a scan error, or a partial read.
release = SetDuckDBMemorySource(func() map[string]int64 { return nil })
func TestDuckDBMemoryReleaseOnlyClearsItsOwner(t *testing.T) {
recordOld, releaseOld := SetDuckDBMemorySource(time.Minute)
record, release := SetDuckDBMemorySource(time.Minute)
defer release()
record(map[string]int64{"BASE_TABLE": 2}, time.Now())
releaseOld()
recordOld(map[string]int64{"BASE_TABLE": 1}, time.Now())
if got := gatherDuckDBMemory(t); got["BASE_TABLE"] != 2 {
t.Fatalf("old owner changed the current sample: %v", got)
}
release()
if got := gatherDuckDBMemory(t); len(got) != 0 {
t.Errorf("after a failed read the gauge still exports %v; stale values are reported as current", got)
t.Fatalf("released source still exports %v", got)
}
}

// Prometheus scrapes concurrently, and a GaugeVec that is Reset and refilled on
// every scrape is shared mutable state: two scrapes interleave, and one of them
// observes the other's half-finished refill -- missing tags, and a sum that
// understates what DuckDB holds.
func TestDuckDBMemoryIsSafeUnderConcurrentScrapes(t *testing.T) {
release := SetDuckDBMemorySource(func() map[string]int64 {
return map[string]int64{"BASE_TABLE": 1, "HASH_TABLE": 2, "ORDER_BY": 3}
})
record, release := SetDuckDBMemorySource(time.Minute)
defer release()
record(map[string]int64{"BASE_TABLE": 1, "HASH_TABLE": 2, "ORDER_BY": 3}, time.Now())

var wg sync.WaitGroup
wg.Go(func() {
for range 200 {
record(map[string]int64{"BASE_TABLE": 1, "HASH_TABLE": 2, "ORDER_BY": 3}, time.Now())
}
})
for range 8 {
wg.Add(1)
go func() {
Expand All @@ -86,7 +101,6 @@ func TestDuckDBMemoryIsSafeUnderConcurrentScrapes(t *testing.T) {
t.Errorf("scrape saw %d tags, want 3: a concurrent refill was observed half-done", n)
return
}
_ = dto.MetricType_GAUGE
}
}
}()
Expand All @@ -100,11 +114,34 @@ func TestDuckDBMemoryExportsNothingWithoutASource(t *testing.T) {
if got := gatherDuckDBMemory(t); len(got) != 0 {
t.Errorf("with no source installed the gauge exports %v", got)
}
if families, err := prometheus.DefaultGatherer.Gather(); err == nil {
for _, family := range families {
if strings.Contains(family.GetName(), "duckdb_memory_scrape") {
t.Errorf("%s exists only to pump a push gauge; a collector does not need it", family.GetName())
}
if got := gatherMemoryTimestamp(t); got != -1 {
t.Errorf("without a source timestamp=%g, want absent", got)
}
}

func gatherMemoryTimestamp(t *testing.T) float64 {
t.Helper()
families, err := prometheus.DefaultGatherer.Gather()
if err != nil {
t.Fatal(err)
}
for _, family := range families {
if family.GetName() == "fanout_duckdb_memory_last_success_timestamp_seconds" {
return family.Metric[0].GetGauge().GetValue()
}
}
return -1
}

func TestDuckDBMemoryDropsStaleSeriesAndKeepsTimestamp(t *testing.T) {
record, release := SetDuckDBMemorySource(time.Second)
defer release()
at := time.Now().Add(-3 * time.Second)
record(map[string]int64{"BASE_TABLE": 4096}, at)
if got := gatherDuckDBMemory(t); len(got) != 0 {
t.Fatalf("stale sample exported %v", got)
}
if got := gatherMemoryTimestamp(t); got != float64(at.UnixNano())/1e9 {
t.Fatalf("stale sample timestamp=%g, want %v", got, at)
}
}
95 changes: 56 additions & 39 deletions internal/metrics/metrics.go
Original file line number Diff line number Diff line change
Expand Up @@ -2,8 +2,10 @@ package metrics

import (
"database/sql"
"maps"
"math"
"sync/atomic"
"time"

"github.com/prometheus/client_golang/prometheus"
"github.com/prometheus/client_golang/prometheus/promauto"
Expand Down Expand Up @@ -312,15 +314,27 @@ func RecordFlush(signal string, durationSec float64) {
FlushDuration.WithLabelValues(signal).Observe(durationSec)
}

type duckDBPoolSource struct{ stats func() sql.DBStats }

// duckDBPoolStats is the live source for the connection-pool gauges below.
//
// A holder rather than a promauto.NewGaugeFunc closure over the pool: the
// gauges are declared once for the process, while a *Duck is constructed many
// times in a test binary, and registering a collector per construction panics
// on the second one. Unset, every gauge reads zero.
var duckDBPoolStats atomic.Pointer[func() sql.DBStats]
var duckDBPoolStats atomic.Pointer[duckDBPoolSource]

type duckDBMemorySample struct {
tags map[string]int64
at time.Time
}

// duckDBMemoryStats is the live source for the per-tag DuckDB memory gauge.
type duckDBMemorySource struct {
sample atomic.Pointer[duckDBMemorySample]
maxAge time.Duration
}

// duckDBMemoryStats holds the latest background sample for the memory gauge.
//
// memory_limit bounds DuckDB's buffer pool and nothing else, so the figure that
// actually predicts an out-of-memory kill is the gap between what the process
Expand All @@ -329,72 +343,75 @@ var duckDBPoolStats atomic.Pointer[func() sql.DBStats]
// - sum(fanout_duckdb_memory_bytes) - go_memstats_heap_sys_bytes. Without it
// every diagnosis of this process is an inference, which is how a whole day
// got spent optimising a Go heap that was never the problem.
var duckDBMemoryStats atomic.Pointer[func() map[string]int64]
var duckDBMemoryStats atomic.Pointer[duckDBMemorySource]

// SetDuckDBMemorySource installs the reader for the per-tag memory gauge.
func SetDuckDBMemorySource(tags func() map[string]int64) (release func()) {
if tags == nil {
return func() {}
}
installed := &tags
// SetDuckDBMemorySource installs a cache, never a scrape-time query. Record
// publishes a complete successful sample; failures leave the last one intact.
// Releasing an older Duck cannot clear a newer Duck's source.
func SetDuckDBMemorySource(interval time.Duration) (record func(map[string]int64, time.Time), release func()) {
installed := &duckDBMemorySource{maxAge: 2 * interval}
duckDBMemoryStats.Store(installed)
return func() { duckDBMemoryStats.CompareAndSwap(installed, nil) }
return func(tags map[string]int64, at time.Time) {
installed.sample.Store(&duckDBMemorySample{tags: maps.Clone(tags), at: at})
}, func() { duckDBMemoryStats.CompareAndSwap(installed, nil) }
}

// duckDBMemoryDesc is emitted by a Collector, not held in a GaugeVec.
//
// The obvious implementation -- a GaugeVec that a scrape hook Resets and
// refills -- is shared mutable state across concurrent scrapes, and a scrape
// that lands mid-refill exports a subset of the tags. That understates what
// DuckDB holds and correspondingly overstates the untracked gap, which is the
// single number this series exists to compute. It also leaves the last good
// values in place when a read fails, reporting them as current: the reading
// that was meant to reveal an impending out-of-memory kill then looks normal.
//
// A Collector has neither problem. Each scrape builds its own metrics from its
// own read, and a read that returns nothing exports nothing -- the series goes
// absent, which is visibly different from "DuckDB is holding nothing".
// Both series come from one immutable sample, so concurrent scrapes never see
// a partial tag set or a timestamp belonging to a different sample.
var duckDBMemoryDesc = prometheus.NewDesc(
"fanout_duckdb_memory_bytes",
"DuckDB memory usage by tag, as DuckDB itself accounts for it",
"Last sampled DuckDB memory usage by tag, as DuckDB itself accounts for it",
[]string{"tag"}, nil,
)

var duckDBMemorySampleDesc = prometheus.NewDesc(
"fanout_duckdb_memory_last_success_timestamp_seconds",
"Unix timestamp of the last successful DuckDB memory sample, or zero before the first success",
nil, nil,
)

type duckDBMemoryCollector struct{}

func (duckDBMemoryCollector) Describe(ch chan<- *prometheus.Desc) { ch <- duckDBMemoryDesc }
func (duckDBMemoryCollector) Describe(ch chan<- *prometheus.Desc) {
ch <- duckDBMemoryDesc
ch <- duckDBMemorySampleDesc
}

func (duckDBMemoryCollector) Collect(ch chan<- prometheus.Metric) {
fn := duckDBMemoryStats.Load()
if fn == nil {
source := duckDBMemoryStats.Load()
if source == nil {
return
}
for tag, bytes := range (*fn)() {
ch <- prometheus.MustNewConstMetric(duckDBMemoryDesc, prometheus.GaugeValue, float64(bytes), tag)
at := 0.0
if sample := source.sample.Load(); sample != nil {
at = float64(sample.at.UnixNano()) / 1e9
if time.Since(sample.at) <= source.maxAge {
for tag, bytes := range sample.tags {
ch <- prometheus.MustNewConstMetric(duckDBMemoryDesc, prometheus.GaugeValue, float64(bytes), tag)
}
}
}
ch <- prometheus.MustNewConstMetric(duckDBMemorySampleDesc, prometheus.GaugeValue, at)
}

func init() {
prometheus.DefaultRegisterer.MustRegister(duckDBMemoryCollector{})
}

// SetDuckDBPoolSource points the connection-pool gauges at a pool and returns
// the function that stops reading it. The last caller wins; the returned
// release only clears the source if it is still the one it installed, so a
// second Duck closing in a test binary cannot blank the gauges of one that is
// still serving.
// SetDuckDBPoolSource reads database/sql bookkeeping on each scrape without
// querying the engine. Releasing an older Duck cannot clear a newer source.
func SetDuckDBPoolSource(stats func() sql.DBStats) (release func()) {
if stats == nil {
return func() {}
}
installed := &stats
installed := &duckDBPoolSource{stats: stats}
duckDBPoolStats.Store(installed)
return func() { duckDBPoolStats.CompareAndSwap(installed, nil) }
}

func poolStats() sql.DBStats {
if fn := duckDBPoolStats.Load(); fn != nil {
return (*fn)()
if source := duckDBPoolStats.Load(); source != nil {
return source.stats()
}
return sql.DBStats{}
}
Expand Down Expand Up @@ -571,8 +588,8 @@ type compactionBacklogSource struct{ count func() int }

var compactionBacklog atomic.Pointer[compactionBacklogSource]

// SetCompactionBacklogSource reads the exact inventory rather than a periodic
// file-stat snapshot. Closing an older repository cannot clear a newer source.
// SetCompactionBacklogSource reads the cached exact inventory. The source must
// not take the catalog lock. Closing an older repository cannot clear a newer source.
func SetCompactionBacklogSource(count func() int) func() {
source := &compactionBacklogSource{count: count}
compactionBacklog.Store(source)
Expand Down
Loading
Loading