From 4cfd60847eed2269e9a955fbd715d073510e4469 Mon Sep 17 00:00:00 2001 From: Vishal Rana Date: Sun, 15 Nov 2026 17:41:47 -0800 Subject: [PATCH] perf(engine): cache Parquet footers on read connections The engine binds an explicit per-file Parquet list on every read statement and a panel issues several, so with parquet_metadata_cache off DuckDB re-read each file's footer on every statement. parquet_metadata_cache is a GLOBAL-scoped option that the duckdb-go driver applies only to the connection that issues the SET, so enabling it once at boot does not reach the pooled read connections. It is now set on each connection in the init hook, next to the UTC setting, and kept in the policy allowed_configs so it still applies on connections opened after lock_configuration. The lockdown is unchanged: the cache only holds immutable Parquet footers and cannot widen file or external access. Measured with an engine-internal A/B over immutable fixtures (same files, one pinned read transaction, AB/BA alternation, 100 samples per arm). Median read latency drops with the cache on: 128 files: 11.83 -> 10.69 ms (-9.6%) 512 files: 33.02 -> 31.02 ms (-6.1%) 1024 files: 62.67 -> 58.61 ms (-6.5%) p95 and p99 improve similarly at 512 and 1024 files. The footer cache costs bounded memory (about 150 MB of OBJECT_CACHE at 1024 fixture files). It keys on path, mtime and size, and our batch paths are unique and immutable (uuid ingest ids and compact- compaction ids; retired paths are deleted, never overwritten in place), so a cached footer can never be stale. Closes #484 --- internal/duckdb/policy.go | 9 +- internal/engine/duck.go | 11 + internal/engine/duck_test.go | 41 +++ internal/engine/metadata_cache_bench_test.go | 254 +++++++++++++++++++ 4 files changed, 312 insertions(+), 3 deletions(-) create mode 100644 internal/engine/metadata_cache_bench_test.go diff --git a/internal/duckdb/policy.go b/internal/duckdb/policy.go index 6986223e9..3e9e6a204 100644 --- a/internal/duckdb/policy.go +++ b/internal/duckdb/policy.go @@ -26,9 +26,12 @@ func ConfigurePolicy(ctx context.Context, db *sql.DB, directories ...string) err "SET allow_community_extensions=false", "SET allowed_directories=[" + strings.Join(paths, ",") + "]", "SET enable_external_access=false", - // UTC is set on each newly pooled connection. Public SQL cannot SET - // options; memory and worker limits remain available to the operator. - "SET allowed_configs=['memory_limit','threads','TimeZone']", + // UTC and parquet_metadata_cache are set on each newly pooled connection, + // so both must stay settable after the lock. Public SQL cannot change any + // other option; memory and worker limits remain available to the operator. + // parquet_metadata_cache only caches immutable Parquet footers, so allowing + // it does not widen file access or the configuration lockdown. + "SET allowed_configs=['memory_limit','threads','TimeZone','parquet_metadata_cache']", "SET lock_configuration=true", } for _, statement := range statements { diff --git a/internal/engine/duck.go b/internal/engine/duck.go index 9fcaed39d..68090723b 100644 --- a/internal/engine/duck.go +++ b/internal/engine/duck.go @@ -502,6 +502,17 @@ func openDuckDB(dsn, tempDir string, maxConns int) (*sql.DB, *sql.DB, *sql.DB, * if _, err := execer.ExecContext(ctx, "SET TimeZone='UTC'", nil); err != nil { return fmt.Errorf("set timezone: %w", err) } + // parquet_metadata_cache is a GLOBAL-scoped option, but duckdb-go applies a + // SET to the issuing connection only — it does not reach the other pooled + // connections, so it must be set on each one here rather than once at boot. + // Every read statement binds an explicit per-file Parquet list and a panel + // issues several, so caching footers by path+mtime+size avoids re-reading + // them per statement. Our batch files are immutable with unique paths, so a + // cached footer is never stale. The policy keeps it in allowed_configs so + // this SET still succeeds on connections opened after lock_configuration. + if _, err := execer.ExecContext(ctx, "SET parquet_metadata_cache=true", nil); err != nil { + return fmt.Errorf("set parquet_metadata_cache: %w", err) + } tempDirOnce.Do(func() { _, tempDirErr = execer.ExecContext(ctx, "SET temp_directory="+sqlLiteral(tempDir), nil) }) diff --git a/internal/engine/duck_test.go b/internal/engine/duck_test.go index a20592bca..f3ff9f211 100644 --- a/internal/engine/duck_test.go +++ b/internal/engine/duck_test.go @@ -147,6 +147,47 @@ func TestNewDuckReconnectsInUTCAfterStartupContextCancelled(t *testing.T) { } } +// Every pooled read connection must enable parquet_metadata_cache so repeated +// statements over the same immutable files reuse their footers instead of +// re-reading them. The setting is applied per connection and kept in the policy +// allowed_configs, so it must also hold on a connection opened after the +// configuration lock, and the lock must still refuse other configuration. +func TestReadConnectionsEnableParquetMetadataCache(t *testing.T) { + ctx := context.Background() + cfg := config.Config{DataDir: t.TempDir(), DuckDBMemory: "128MB", DuckDBThreads: 2, DuckDBMaxConns: 4} + repository, err := telemetrystore.Open(cfg.TelemetryDir()) + if err != nil { + t.Fatal(err) + } + defer repository.Close() + d, err := NewDuck(ctx, cfg, repository) + if err != nil { + t.Fatalf("NewDuck() error = %v", err) + } + defer d.Close() + + for name, db := range map[string]*sql.DB{"read": d.DB, "write": d.writeDB, "memory": d.memoryDB, "parse": d.parseDB} { + t.Run(name, func(t *testing.T) { + // Retire idle connections so this query runs on a freshly opened one, + // whose init hook runs after the configuration was locked. + db.SetMaxIdleConns(0) + var enabled bool + if err := db.QueryRowContext(ctx, "SELECT current_setting('parquet_metadata_cache')::BOOLEAN").Scan(&enabled); err != nil { + t.Fatal(err) + } + if !enabled { + t.Fatalf("parquet_metadata_cache = false on a %s connection, want true", name) + } + }) + } + + // The lock still refuses configuration outside allowed_configs, so public SQL + // cannot re-open file or external access. + if _, err := d.DB.ExecContext(ctx, "SET enable_external_access=true"); err == nil { + t.Fatal("expected locked configuration to refuse SET enable_external_access") + } +} + // A cancelled statement's interrupt can reach the ROLLBACK that ends its read // transaction, leaving the transaction open. The read must roll back itself // after cancellation, and a connection whose rollback failed must leave the diff --git a/internal/engine/metadata_cache_bench_test.go b/internal/engine/metadata_cache_bench_test.go new file mode 100644 index 000000000..56dfedcf0 --- /dev/null +++ b/internal/engine/metadata_cache_bench_test.go @@ -0,0 +1,254 @@ +//go:build filebench + +package engine + +import ( + "context" + "database/sql" + "database/sql/driver" + "encoding/json" + "errors" + "fmt" + "path/filepath" + "sort" + "testing" + "time" + + duckdb "github.com/duckdb/duckdb-go/v2" + "github.com/labstack/pitot/internal/config" + "github.com/labstack/pitot/internal/queryrows" + "github.com/labstack/pitot/internal/telemetry" + telemetrystore "github.com/labstack/pitot/internal/telemetry/store" +) + +// A/B for parquet_metadata_cache (#484). The engine passes an explicit per-file +// Parquet list on every statement, and a panel issues several. With the footer +// cache OFF, each statement re-reads every file footer; with it ON, footers are +// cached by path+mtime+size. Both arms read the SAME immutable fixtures and run +// the same explicit-list statement; the only difference is the per-connection +// cache SET. Both engines are opened and warmed up front, then sampled with +// AB/BA alternation so OS page-cache and time drift affect both arms equally. +// This is an engine-internal control, not a product code path and not capacity +// evidence. +// +// go test -tags=filebench ./internal/engine -run '^TestParquetMetadataCacheComparisonBenchmark$' -count=1 -v +func TestParquetMetadataCacheComparisonBenchmark(t *testing.T) { + ctx, cancel := context.WithTimeout(t.Context(), 10*time.Minute) + defer cancel() + + cfg := config.Config{DataDir: t.TempDir()} + repo, err := telemetrystore.Open(cfg.TelemetryDir()) + if err != nil { + t.Fatal(err) + } + defer repo.Close() + + base := time.Date(2026, 11, 1, 12, 0, 0, 123, time.UTC) + w := queryrows.Window{Start: base, End: base.Add(time.Minute)} + signal, _ := telemetry.LookupSignal("spans") + + published := 0 + for _, files := range []int{128, 512, 1024} { + for ; published < files; published++ { + batch := telemetrystore.Batch{ID: fmt.Sprintf("mdc_%05d", published)} + for row := range 32 { + service := []string{"api", "worker", "db"}[row%3] + n := base.Add(time.Duration(row) * time.Millisecond).UnixNano() + if row == 0 { + n = w.Start.UnixNano() + } + if row == 31 { + n = w.End.UnixNano() - 1 + } + status := "STATUS_CODE_OK" + if row%7 == 0 { + status = "STATUS_CODE_ERROR" + } + span := telemetry.Span{Namespace: "prod", ServiceName: service, TraceID: fmt.Sprintf("mdc_trace_%05d_%02d", published, row/2), SpanID: fmt.Sprint(row), StartUnixNanos: n, EndUnixNanos: n + int64(row+1)*int64(time.Millisecond), DurationMS: float64(row + 1), StatusCode: status, IngestedAt: base.UnixNano() + int64(published)} + batch.Spans = append(batch.Spans, span) + } + if err := repo.Commit(ctx, batch); err != nil { + t.Fatal(err) + } + } + batches := repo.Parquet.BatchMetadata() + if len(batches) != files { + t.Fatalf("captured files=%d expected=%d", len(batches), files) + } + + sort.Slice(batches, func(i, j int) bool { return batches[i].ID < batches[j].ID }) + paths := make([]string, 0, len(batches)) + for _, b := range batches { + paths = append(paths, sqlLiteral(filepath.ToSlash(filepath.Join(repo.Parquet.Dir(), "batches", b.ID+".batch", "spans.parquet")))) + } + physical := "SELECT " + rawProjection(signal) + " FROM read_parquet([" + joinCSV(paths) + "], union_by_name=false, hive_partitioning=false)" + statement := `SELECT service,count(*)::BIGINT,count(DISTINCT trace_id)::BIGINT, + sum(end_unix_nano-start_unix_nano)::BIGINT, + sum(CASE WHEN status='STATUS_CODE_ERROR' THEN 1 ELSE 0 END)::BIGINT + FROM (` + cleanSource("spans", physical) + `) WHERE start_time>=?::TIMESTAMP_NS::TIMESTAMPTZ_NS AND start_time 0 { + out += "," + } + out += s + } + return out +} + +type mdcConnector struct { + *duckdb.Connector + cache bool +} + +func (c *mdcConnector) Connect(ctx context.Context) (driver.Conn, error) { + conn, err := c.Connector.Connect(ctx) + if err != nil { + return nil, err + } + ex := conn.(driver.ExecerContext) + stmts := []string{"LOAD parquet", "SET TimeZone='UTC'"} + if c.cache { + stmts = append(stmts, "SET parquet_metadata_cache=true") + } + for _, stmt := range stmts { + if _, err := ex.ExecContext(ctx, stmt, nil); err != nil { + return nil, errors.Join(err, conn.Close()) + } + } + return conn, nil +} + +type metadataCacheArm struct { + t *testing.T + ctx context.Context + pool *sql.DB + conn *sql.Conn + tx *sql.Tx + statement string + w queryrows.Window + cache bool + timings []float64 +} + +func newMetadataCacheArm(t *testing.T, ctx context.Context, statement string, w queryrows.Window, cache bool) *metadataCacheArm { + t.Helper() + dsn, err := duckDSN(filepath.Join(t.TempDir(), "arm.duckdb"), "4GB", 4) + if err != nil { + t.Fatal(err) + } + native, err := duckdb.NewConnector(dsn, nil) + if err != nil { + t.Fatal(err) + } + pool := sql.OpenDB(&mdcConnector{Connector: native, cache: cache}) + pool.SetMaxOpenConns(1) + pool.SetMaxIdleConns(1) + conn, err := pool.Conn(ctx) + if err != nil { + t.Fatal(err) + } + var got bool + if err := conn.QueryRowContext(ctx, "SELECT current_setting('parquet_metadata_cache')::BOOLEAN").Scan(&got); err != nil { + t.Fatal(err) + } + if got != cache { + t.Fatalf("arm cache=%v but connection reports %v", cache, got) + } + tx, err := conn.BeginTx(context.WithoutCancel(ctx), nil) + if err != nil { + t.Fatal(err) + } + return &metadataCacheArm{t: t, ctx: ctx, pool: pool, conn: conn, tx: tx, statement: statement, w: w, cache: cache} +} + +func (a *metadataCacheArm) exec() { + rows, err := a.tx.QueryContext(a.ctx, a.statement, a.w.Start, a.w.End) + if err != nil { + a.t.Fatal(err) + } + for rows.Next() { + var s string + var x, y, z, q int64 + if err := rows.Scan(&s, &x, &y, &z, &q); err != nil { + a.t.Fatal(err) + } + } + if err := rows.Err(); err != nil { + a.t.Fatal(err) + } + rows.Close() +} + +func (a *metadataCacheArm) sample() { + started := time.Now() + a.exec() + a.timings = append(a.timings, float64(time.Since(started))/float64(time.Millisecond)) +} + +func (a *metadataCacheArm) report() map[string]any { + sort.Float64s(a.timings) + var sum float64 + for _, v := range a.timings { + sum += v + } + var objectCacheBytes int64 + _ = a.tx.QueryRowContext(a.ctx, "SELECT coalesce(sum(memory_usage_bytes),0)::BIGINT FROM duckdb_memory() WHERE tag='OBJECT_CACHE'").Scan(&objectCacheBytes) + n := len(a.timings) + return map[string]any{ + "arm": map[bool]string{false: "cache_off", true: "cache_on"}[a.cache], + "cache_enabled": a.cache, + "samples": n, + "mean_ms": sum / float64(n), + "median_ms": (a.timings[n/2-1] + a.timings[n/2]) / 2, + "p95_ms": a.timings[int(float64(n)*0.95)-1], + "p99_ms": a.timings[int(float64(n)*0.99)-1], + "min_ms": a.timings[0], + "max_ms": a.timings[n-1], + "object_cache_bytes": objectCacheBytes, + "unit": "native_execute_scan_same_tx_ab_ba", + } +} + +func (a *metadataCacheArm) close() { + _ = a.tx.Rollback() + _ = a.conn.Close() + _ = a.pool.Close() +}