Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
49 changes: 41 additions & 8 deletions internal/api/observability.go
Original file line number Diff line number Diff line change
Expand Up @@ -29,15 +29,42 @@ func NewObservabilityHandler(queries ObservabilityQueries) *ObservabilityHandler
return &ObservabilityHandler{queries: queries, now: time.Now}
}

// Register mounts the deterministic product API. These endpoints are for
// dashboards and drill-downs; the browser does not need an agent for ordinary
// telemetry navigation.
// observabilityDeadline bounds one dashboard query.
//
// The SQL API has always been bounded; these endpoints, which every widget
// calls, had no deadline of their own — a query that went pathological held a
// connection until the client gave up, and the client's own timeout is not
// something the server can rely on.
//
// Twenty seconds is well clear of the slowest query measured under load here,
// about two seconds, and stricter than the SQL API's own ceiling. It is
// deliberately shorter than the worst legitimate wait behind a stuck
// publication — the drain remainder plus the swap budget can exceed a minute —
// so a publication that goes wrong answers the dashboard instead of holding
// every widget's connection until it finishes.
const observabilityDeadline = 20 * time.Second

// Register mounts the deterministic product API, each endpoint under a
// deadline. These endpoints are for dashboards and drill-downs; the browser
// does not need an agent for ordinary telemetry navigation.
func (h *ObservabilityHandler) Register(group *echo.Group) {
group.GET("/overview", h.overview)
group.GET("/topology", h.topology)
group.GET("/performance", h.performance)
group.GET("/trace", h.trace)
group.GET("/logs", h.logs)
group.GET("/overview", h.bounded(h.overview))
group.GET("/topology", h.bounded(h.topology))
group.GET("/performance", h.bounded(h.performance))
group.GET("/trace", h.bounded(h.trace))
group.GET("/logs", h.bounded(h.logs))
}

// bounded gives a handler a deadline. The context reaches DuckDB through
// QueryContext, so an expired one stops the statement rather than only the
// reply — and releases the pool connection it was holding.
func (h *ObservabilityHandler) bounded(handler func(*echo.Context) error) func(*echo.Context) error {
return func(c *echo.Context) error {
ctx, cancel := context.WithTimeout(c.Request().Context(), observabilityDeadline)
defer cancel()
c.SetRequest(c.Request().WithContext(ctx))
return handler(c)
}
}

func (h *ObservabilityHandler) performance(c *echo.Context) error {
Expand Down Expand Up @@ -129,5 +156,11 @@ func mapQueryError(err error) error {
if errors.Is(err, observability.ErrInvalidScope) || errors.Is(err, observability.ErrInvalidLimit) {
return echo.NewHTTPError(http.StatusBadRequest, err.Error())
}
// A query that ran out of time is a load condition, not a fault: reporting
// it as a server error logs it at ERROR and counts it towards the 5xx rate
// that pages someone.
if errors.Is(err, context.DeadlineExceeded) {
return echo.NewHTTPError(http.StatusServiceUnavailable, "telemetry query took too long").Wrap(err)
}
return echo.NewHTTPError(http.StatusInternalServerError, "telemetry query failed").Wrap(err)
}
97 changes: 97 additions & 0 deletions internal/api/observability_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,8 @@ package api
import (
"context"
"encoding/json"
"errors"
"fmt"
"net/http"
"net/http/httptest"
"testing"
Expand Down Expand Up @@ -91,3 +93,98 @@ func TestRouteRejectsInvalidWindow(t *testing.T) {
t.Fatalf("status = %d, want 400", rec.Code)
}
}

// A dashboard query that runs out of time is a load condition. Reporting it as
// a server error logs it at ERROR and counts towards the 5xx rate that pages
// someone; the deadline itself exists so a pathological query stops holding a
// connection once the answer can no longer be useful.
func TestObservabilityQueryDeadlineIsNotAServerError(t *testing.T) {
err := mapQueryError(fmt.Errorf("query endpoints: %w", context.DeadlineExceeded))
var httpErr *echo.HTTPError
if !errors.As(err, &httpErr) {
t.Fatalf("mapQueryError returned %T, want an *echo.HTTPError", err)
}
if httpErr.Code != http.StatusServiceUnavailable {
t.Fatalf("a timed-out query maps to %d, want %d", httpErr.Code, http.StatusServiceUnavailable)
}
// A genuine failure still reads as one.
var failure *echo.HTTPError
if !errors.As(mapQueryError(errors.New("boom")), &failure) || failure.Code != http.StatusInternalServerError {
t.Fatal("a real query failure no longer maps to 500")
}
}

// Every registered route carries the deadline, not just the wrapper in
// isolation: dropping bounded() from Register is the mistake a refactor makes,
// and it left the whole suite green.
func TestRegisteredObservabilityRoutesCarryADeadline(t *testing.T) {
deadlines := map[string]bool{}
handler := &ObservabilityHandler{queries: deadlineProbe{seen: deadlines}, now: time.Now}
e := echo.New()
handler.Register(e.Group("/api/observability"))
for _, path := range []string{"/api/observability/overview", "/api/observability/topology", "/api/observability/performance", "/api/observability/logs", "/api/observability/trace"} {
recorder := httptest.NewRecorder()
e.ServeHTTP(recorder, httptest.NewRequest(http.MethodGet, path+"?window=1h", nil))
if !deadlines[path] {
t.Errorf("%s reached the query layer with no deadline", path)
}
}
}

// deadlineProbe records, per route, whether the context that reached it had a
// deadline. It answers every query with an empty result.
type deadlineProbe struct{ seen map[string]bool }

func (p deadlineProbe) note(ctx context.Context, path string) {
if _, ok := ctx.Deadline(); ok {
p.seen[path] = true
}
}

func (p deadlineProbe) Overview(ctx context.Context, scope observability.Scope, limit int) (observability.Result[observability.Overview], error) {
p.note(ctx, "/api/observability/overview")
return observability.Result[observability.Overview]{}, nil
}

func (p deadlineProbe) Topology(ctx context.Context, scope observability.Scope, limit int) (observability.Result[observability.Topology], error) {
p.note(ctx, "/api/observability/topology")
return observability.Result[observability.Topology]{}, nil
}

func (p deadlineProbe) Performance(ctx context.Context, scope observability.Scope, service string, limit int) (observability.Result[observability.Performance], error) {
p.note(ctx, "/api/observability/performance")
return observability.Result[observability.Performance]{}, nil
}

func (p deadlineProbe) Logs(ctx context.Context, scope observability.Scope, service, severity, search string, limit int) (observability.Result[observability.Logs], error) {
p.note(ctx, "/api/observability/logs")
return observability.Result[observability.Logs]{}, nil
}

func (p deadlineProbe) Trace(ctx context.Context, scope observability.Scope, traceID, service string, limit int) (observability.Result[observability.TraceDetail], error) {
p.note(ctx, "/api/observability/trace")
return observability.Result[observability.TraceDetail]{}, nil
}

// The handlers carry a deadline of their own, because the client's timeout is
// not something the server can rely on.
func TestObservabilityHandlersCarryADeadline(t *testing.T) {
var seen time.Duration
handler := (&ObservabilityHandler{}).bounded(func(c *echo.Context) error {
deadline, ok := c.Request().Context().Deadline()
if !ok {
t.Fatal("handler ran with no deadline")
}
seen = time.Until(deadline)
return nil
})
e := echo.New()
request := httptest.NewRequest(http.MethodGet, "/api/observability/overview", nil)
c := e.NewContext(request, httptest.NewRecorder())
if err := handler(c); err != nil {
t.Fatalf("bounded handler: %v", err)
}
if seen <= 0 || seen > observabilityDeadline {
t.Fatalf("deadline left %s, want a positive value no greater than %s", seen, observabilityDeadline)
}
}
152 changes: 152 additions & 0 deletions internal/metrics/metrics.go
Original file line number Diff line number Diff line change
@@ -1,10 +1,13 @@
package metrics

import (
"database/sql"
"math"
"sync/atomic"

"github.com/prometheus/client_golang/prometheus"
"github.com/prometheus/client_golang/prometheus/promauto"
dto "github.com/prometheus/client_model/go"
)

type RollupComponent string
Expand Down Expand Up @@ -204,6 +207,57 @@ var (
Help: "Parquet publications abandoned after reader wait timeout",
})

// The publisher's side of Parquet publication was measured and the
// reader's side was not, which left the one question a stalled query asks
// — "how long did I wait to enter the snapshot?" — unanswerable. These
// three complete the pair.
// An uncontended admission costs single-digit microseconds, so the ladder
// starts below that: a histogram whose healthy traffic all lands in the
// first bucket can tell you a stall happened but not what normal was.
snapshotBuckets = []float64{.000001, .000005, .00002, .0001, .0005, .001, .005, .025, .1, .5, 1, 2.5, 5, 10, 30, 60}

ParquetPublishHold = promauto.NewHistogram(prometheus.HistogramOpts{
Name: "fanout_parquet_publish_hold_seconds",
Help: "Time readers were excluded while a new Parquet file set was swapped in",
Buckets: snapshotBuckets,
})

parquetReadWait = promauto.NewHistogramVec(prometheus.HistogramOpts{
Name: "fanout_parquet_read_wait_seconds",
Help: "Time a reader spent waiting to enter the Parquet snapshot",
Buckets: snapshotBuckets,
}, []string{"reader"})

parquetReadRefusals = promauto.NewCounterVec(prometheus.CounterOpts{
Name: "fanout_parquet_read_refusals_total",
Help: "Readers that gave up waiting to enter the Parquet snapshot",
}, []string{"reader"})

// The wait to enter the snapshot and the wait for a connection are both
// measured; how long the statement itself takes was not, so a slow request
// could be attributed to queueing or to execution only by elimination.
// The connection is acquired before this timer starts, so the two do not
// overlap.
DuckDBStatement = promauto.NewHistogram(prometheus.HistogramOpts{
Name: "fanout_duckdb_statement_seconds",
Help: "Time a read statement spent executing in DuckDB, excluding any wait to start it",
Buckets: []float64{.0005, .001, .005, .01, .025, .05, .1, .25, .5, 1, 2.5, 5, 10, 30},
})

// An explicit CHECKPOINT blocks new transactions for its duration, and the
// non-FORCE form refuses while another write transaction is open — so both
// how long it takes and how often it does not happen are worth knowing.
DuckDBCheckpoint = promauto.NewHistogram(prometheus.HistogramOpts{
Name: "fanout_duckdb_checkpoint_seconds",
Help: "Duration of an explicit DuckDB checkpoint on the rollup cache",
Buckets: []float64{.001, .005, .01, .05, .1, .25, .5, 1, 2.5, 5, 10, 30},
})

DuckDBCheckpointFailures = promauto.NewCounter(prometheus.CounterOpts{
Name: "fanout_duckdb_checkpoint_failures_total",
Help: "Explicit DuckDB checkpoints that returned an error",
})

// HTTP metrics
HTTPRequestsTotal = promauto.NewCounterVec(prometheus.CounterOpts{
Name: "fanout_http_requests_total",
Expand Down Expand Up @@ -239,6 +293,104 @@ func RecordFlush(signal string, durationSec float64) {
FlushDuration.WithLabelValues(signal).Observe(durationSec)
}

// duckDBPoolStats is the live source for the connection-pool gauges below.
//
// A holder rather than a promauto.NewGaugeFunc closure over the pool: the
// gauges are declared once for the process, while a *Duck is constructed many
// times in a test binary, and registering a collector per construction panics
// on the second one. Unset, every gauge reads zero.
var duckDBPoolStats atomic.Pointer[func() sql.DBStats]

// SetDuckDBPoolSource points the connection-pool gauges at a pool and returns
// the function that stops reading it. The last caller wins; the returned
// release only clears the source if it is still the one it installed, so a
// second Duck closing in a test binary cannot blank the gauges of one that is
// still serving.
func SetDuckDBPoolSource(stats func() sql.DBStats) (release func()) {
if stats == nil {
return func() {}
}
installed := &stats
duckDBPoolStats.Store(installed)
return func() { duckDBPoolStats.CompareAndSwap(installed, nil) }
}

func poolStats() sql.DBStats {
if fn := duckDBPoolStats.Load(); fn != nil {
return (*fn)()
}
return sql.DBStats{}
}

// The read pool is the one that can starve: background writers, the detector,
// the alert engine and every dashboard widget draw from it, and a request that
// waits for a connection waits invisibly — database/sql counts that wait and
// nothing exported it.
var (
_ = promauto.NewGaugeFunc(prometheus.GaugeOpts{
Name: "fanout_duckdb_pool_in_use",
Help: "DuckDB read connections currently in use",
}, func() float64 { return float64(poolStats().InUse) })

_ = promauto.NewGaugeFunc(prometheus.GaugeOpts{
Name: "fanout_duckdb_pool_idle",
Help: "DuckDB read connections currently idle",
}, func() float64 { return float64(poolStats().Idle) })

_ = promauto.NewGaugeFunc(prometheus.GaugeOpts{
Name: "fanout_duckdb_pool_max_open",
Help: "Configured DuckDB read connection limit",
}, func() float64 { return float64(poolStats().MaxOpenConnections) })

_ = promauto.NewCounterFunc(prometheus.CounterOpts{
Name: "fanout_duckdb_pool_waits_total",
Help: "Times a caller waited for a free DuckDB read connection",
}, func() float64 { return float64(poolStats().WaitCount) })

_ = promauto.NewCounterFunc(prometheus.CounterOpts{
Name: "fanout_duckdb_pool_wait_seconds_total",
Help: "Total time callers spent waiting for a free DuckDB read connection",
}, func() float64 { return poolStats().WaitDuration.Seconds() })
)

// RecordParquetRead records one attempt to enter the Parquet snapshot.
//
// Both series are behind this function for the same reason RecordWriteGate
// exists: the label is a fixed reader class, and a vector left exported is a
// vector something eventually calls with a service name in it.
func RecordParquetRead(reader string, waitSec float64, admitted bool) {
if admitted {
parquetReadWait.WithLabelValues(reader).Observe(waitSec)
return
}
parquetReadRefusals.WithLabelValues(reader).Inc()
}

// ParquetReadWaitForTest and ParquetReadRefusalsForTest hand the two series to
// a test in another package, which asserts on them rather than on the call —
// the call is the easy half. They return collectors rather than the vectors so
// production code still cannot set an arbitrary label, and they keep
// prometheus/testutil out of the release dependency graph.
func ParquetReadWaitForTest() prometheus.Collector { return parquetReadWait }

func ParquetReadRefusalsForTest(reader string) prometheus.Counter {
return parquetReadRefusals.WithLabelValues(reader)
}

// DuckDBStatementSecondsForTest reports the statement histogram's running
// total, so a test can prove what the timer does and does not cover.
func DuckDBStatementSecondsForTest() float64 {
metric := &dto.Metric{}
if err := DuckDBStatement.Write(metric); err != nil {
return 0
}
return metric.GetHistogram().GetSampleSum()
}

// DuckDBPoolStatsForTest exposes what the pool gauges are reading, so a test
// can prove they point at the read pool rather than the write handle.
func DuckDBPoolStatsForTest() sql.DBStats { return poolStats() }

// RecordWriteGate records one complete Telemetry catalog write critical section.
// Callers constrain operation to the fixed writegate.WriteOperation set so this
// metric cannot grow with tenant or telemetry cardinality.
Expand Down
26 changes: 21 additions & 5 deletions internal/observability/logs.go
Original file line number Diff line number Diff line change
Expand Up @@ -17,12 +17,28 @@ WHERE time >= ? AND time < ?
// DuckDB answers the two questions the API asks: the newest `limit` entries
// and per-bucket counts. LIMIT bounds the entry stream and GROUP BY bounds the
// histogram stream regardless of the retained Parquet row count.
//
// Redaction runs outside the LIMIT, and that placement is the whole cost of
// this query. A projection in the same SELECT as ORDER BY ... LIMIT is
// evaluated below the top-N operator, so four chained regexp_replace ran over
// every row in the window to return a hundred: measured at 140ms for a page of
// 100 out of 372,000 rows, and 347ms with four DuckDB threads, against 8ms
// once the top-N runs first. It was the one statement here whose cost grew
// with retained rows rather than with the answer.
//
// The search predicate stays inside, on purpose: a search has to match what
// the reader will be shown, so it matches the redacted text. It is skipped
// entirely when no search term is given.
var logEntriesQuery = `
SELECT time, severity, coalesce(service, ''), ` + redactLogBodySQL("body") + `,
coalesce(trace_id, ''), coalesce(span_id, '')
FROM logs` + logFilters + `
ORDER BY time DESC
LIMIT ?`
SELECT time, severity, service, ` + redactLogBodySQL("body") + `, trace_id, span_id
FROM (
SELECT time, severity, coalesce(service, '') AS service, body,
coalesce(trace_id, '') AS trace_id, coalesce(span_id, '') AS span_id
FROM logs` + logFilters + `
ORDER BY time DESC
LIMIT ?
)
ORDER BY time DESC`

var logBucketsQueryTemplate = `
SELECT time_bucket(INTERVAL '%s', time) AS point_time,
Expand Down
Loading
Loading