Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion internal/mcp/apps/overview.html

Large diffs are not rendered by default.

20 changes: 10 additions & 10 deletions internal/mcp/apps/performance.html

Large diffs are not rendered by default.

2 changes: 1 addition & 1 deletion internal/mcp/apps/topology.html

Large diffs are not rendered by default.

2 changes: 1 addition & 1 deletion internal/mcp/apps/trace.html

Large diffs are not rendered by default.

104 changes: 104 additions & 0 deletions internal/observability/latency_read_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,7 @@ import (
"context"
"database/sql"
"math"
"strings"
"testing"
"time"

Expand Down Expand Up @@ -89,3 +90,106 @@ func TestTotalsTakeLatencyFromHalvesThatServedSomething(t *testing.T) {
t.Fatalf("p95 = %.0f, want the subscription figure when there is nothing else", onlyWaiting.P95MS)
}
}

func TestHistogramQuantileInterpolatesInsideTheBucket(t *testing.T) {
// Six calls at 5, 10, 100, 500, 1000 and 2000ms.
cumulative := []float64{0, 0, 0, 0, 1, 2, 2, 2, 3, 3, 4, 4, 5, 6, 6, 6, 6}
if got := histogramQuantile(cumulative, 6, 0.50); math.Abs(got-100) > 1e-6 {
t.Fatalf("p50 = %v, want 100", got)
}
// The bound alone would report 2000 for both, which is the tell that a
// table is reporting buckets rather than latencies.
p95, p99 := histogramQuantile(cumulative, 6, 0.95), histogramQuantile(cumulative, 6, 0.99)
if math.Abs(p95-1700) > 1e-6 || math.Abs(p99-1940) > 1e-6 {
t.Fatalf("p95/p99 = %v/%v, want 1700/1940", p95, p99)
}
if p95 >= p99 {
t.Fatalf("p95 %v should sit below p99 %v", p95, p99)
}
}

func TestHistogramQuantileHandlesEmptyAndOverflow(t *testing.T) {
if got := histogramQuantile(nil, 0, 0.95); got != 0 {
t.Fatalf("no data = %v, want 0", got)
}
if got := histogramQuantile([]float64{0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0}, 5, 0.95); got != 300000 {
t.Fatalf("everything past the last boundary = %v, want the boundary itself", got)
}
// Ten calls somewhere in the 2.5–5ms bucket: the 95th sits near the top of
// that bucket rather than exactly on its boundary, which is the point of
// interpolating at all.
if got := histogramQuantile([]float64{0, 0, 0, 0, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10}, 10, 0.95); math.Abs(got-4.875) > 1e-9 {
t.Fatalf("quantile inside the last populated bucket = %v, want 4.875", got)
}
}

func TestComparisonIgnoresSubMillisecondLatencyMoves(t *testing.T) {
// 3.9ms to 4.0ms is 2.2%, which was enough to paint a red regression arrow
// beside a number that had not meaningfully moved.
quiet := comparisonMetric("P50 latency", "ms", 3.9, 4.0, true)
if quiet.Direction != DirectionStable {
t.Fatalf("direction = %q, want stable for a 0.1ms move", quiet.Direction)
}
if quiet.Significant {
t.Fatalf("a 0.1ms move should never be significant: %#v", quiet)
}
// A real move still reads as one.
real := comparisonMetric("P95 latency", "ms", 13690, 10410, true)
if real.Direction != DirectionImprovement || !real.Significant {
t.Fatalf("13.7s to 10.4s = %#v, want a significant improvement", real)
}
// The floor is for latency only; a rate moving by less than a point is
// still a rate that moved.
rate := comparisonMetric("Error rate", "%", 0.10, 0.40, true)
if rate.Direction != DirectionRegression {
t.Fatalf("error rate = %#v, want a regression", rate)
}
}

func TestEndpointDurationBucketsAreConsistent(t *testing.T) {
previous := 0.0
for _, bucket := range endpointDurationBuckets {
if bucket.Bound <= previous {
t.Fatalf("bound %v follows %v; the boundaries must increase or interpolation reads a negative bucket width", bucket.Bound, previous)
}
previous = bucket.Bound
}
// The query reads these counts positionally, so its SELECT list has to be
// this list — a reordering would report one bucket's count against
// another's boundary and quietly shift every percentile.
if !strings.Contains(endpointRollupQuery, endpointDurationColumns()) {
t.Fatalf("the rollup query does not select %q in order", endpointDurationColumns())
}
}

func TestComparisonFloorAlsoSilencesThePercentage(t *testing.T) {
// No traffic in the earlier half sends change through the divide-by-zero
// branch and out at 100%. Printed inside a grey "stable" badge, that is the
// same number-against-colour contradiction the floor exists to remove.
quiet := comparisonMetric("P50 latency", "ms", 0, 0.5, true)
if quiet.Direction != DirectionStable || quiet.ChangePct != 0 {
t.Fatalf("metric = %#v, want a stable reading with no percentage", quiet)
}
if quiet.Before != 0 || quiet.After != 0.5 {
t.Fatalf("metric = %#v, the underlying figures should still be reported", quiet)
}
}

// Interpolating inside the bucket moves an endpoint whose calls sit under a
// boundary out of the class the bound alone put it in. 100 calls at 700ms are
// healthy; reporting the 750ms bound called them degraded.
func TestInterpolationGradesAnEndpointOnItsCallsNotItsBucket(t *testing.T) {
cumulative := make([]float64, len(endpointDurationBuckets))
for i, bucket := range endpointDurationBuckets {
if bucket.Bound >= 750 {
cumulative[i] = 100
}
}
p95 := histogramQuantile(cumulative, 100, 0.95)
if p95 >= latencyDegradedThresholdMS {
t.Fatalf("p95 = %v, want it under the %vms boundary its calls sit below", p95, latencyDegradedThresholdMS)
}
if classify(0, p95) != HealthHealthy {
t.Fatalf("health = %q, want healthy for calls that never reached the boundary", classify(0, p95))
}
}
139 changes: 101 additions & 38 deletions internal/observability/performance.go
Original file line number Diff line number Diff line change
Expand Up @@ -54,7 +54,7 @@ FROM (SELECT DISTINCT bucket FROM endpoint_rollup LIMIT 5)`
// endpointRollupQuery uses the minute cache only through its current watermark.
// Raw spans cover both partial boundary minutes and any newer complete minutes,
// so the cache lag cannot appear as zero traffic and raw work stays bounded.
const endpointRollupQuery = `
var endpointRollupQuery = `
WITH params AS (
SELECT
?::TIMESTAMP AS start_time,
Expand Down Expand Up @@ -151,42 +151,11 @@ SELECT
t.method,
t.path,
t.calls,
CASE
WHEN duration_count = 0 THEN 0
WHEN le_0_1 >= duration_count * 0.50 THEN 0.1 WHEN le_0_5 >= duration_count * 0.50 THEN 0.5
WHEN le_1 >= duration_count * 0.50 THEN 1 WHEN le_2_5 >= duration_count * 0.50 THEN 2.5
WHEN le_5 >= duration_count * 0.50 THEN 5 WHEN le_10 >= duration_count * 0.50 THEN 10
WHEN le_25 >= duration_count * 0.50 THEN 25 WHEN le_50 >= duration_count * 0.50 THEN 50
WHEN le_100 >= duration_count * 0.50 THEN 100 WHEN le_250 >= duration_count * 0.50 THEN 250
WHEN le_500 >= duration_count * 0.50 THEN 500 WHEN le_750 >= duration_count * 0.50 THEN 750
WHEN le_1000 >= duration_count * 0.50 THEN 1000 WHEN le_2000 >= duration_count * 0.50 THEN 2000
WHEN le_5000 >= duration_count * 0.50 THEN 5000 WHEN le_30000 >= duration_count * 0.50 THEN 30000
ELSE 300000 END AS p50_ms,
CASE
WHEN duration_count = 0 THEN 0
WHEN le_0_1 >= duration_count * 0.95 THEN 0.1 WHEN le_0_5 >= duration_count * 0.95 THEN 0.5
WHEN le_1 >= duration_count * 0.95 THEN 1 WHEN le_2_5 >= duration_count * 0.95 THEN 2.5
WHEN le_5 >= duration_count * 0.95 THEN 5 WHEN le_10 >= duration_count * 0.95 THEN 10
WHEN le_25 >= duration_count * 0.95 THEN 25 WHEN le_50 >= duration_count * 0.95 THEN 50
WHEN le_100 >= duration_count * 0.95 THEN 100 WHEN le_250 >= duration_count * 0.95 THEN 250
WHEN le_500 >= duration_count * 0.95 THEN 500 WHEN le_750 >= duration_count * 0.95 THEN 750
WHEN le_1000 >= duration_count * 0.95 THEN 1000 WHEN le_2000 >= duration_count * 0.95 THEN 2000
WHEN le_5000 >= duration_count * 0.95 THEN 5000 WHEN le_30000 >= duration_count * 0.95 THEN 30000
ELSE 300000 END AS p95_ms,
CASE
WHEN duration_count = 0 THEN 0
WHEN le_0_1 >= duration_count * 0.99 THEN 0.1 WHEN le_0_5 >= duration_count * 0.99 THEN 0.5
WHEN le_1 >= duration_count * 0.99 THEN 1 WHEN le_2_5 >= duration_count * 0.99 THEN 2.5
WHEN le_5 >= duration_count * 0.99 THEN 5 WHEN le_10 >= duration_count * 0.99 THEN 10
WHEN le_25 >= duration_count * 0.99 THEN 25 WHEN le_50 >= duration_count * 0.99 THEN 50
WHEN le_100 >= duration_count * 0.99 THEN 100 WHEN le_250 >= duration_count * 0.99 THEN 250
WHEN le_500 >= duration_count * 0.99 THEN 500 WHEN le_750 >= duration_count * 0.99 THEN 750
WHEN le_1000 >= duration_count * 0.99 THEN 1000 WHEN le_2000 >= duration_count * 0.99 THEN 2000
WHEN le_5000 >= duration_count * 0.99 THEN 5000 WHEN le_30000 >= duration_count * 0.99 THEN 30000
ELSE 300000 END AS p99_ms,
t.error_rate
t.error_rate,
t.duration_count,
` + endpointDurationColumns() + `
FROM endpoint_totals t
ORDER BY t.calls DESC, p95_ms DESC
ORDER BY t.calls DESC, (t.duration_count - le_100) DESC, t.method, t.path
LIMIT ?`

var performanceHeatmapQueryTemplate = `
Expand Down Expand Up @@ -318,7 +287,7 @@ func (s *Service) queryEndpoints(ctx context.Context, scope Scope, service strin
if ready {
query = endpointRollupQuery
args = []any{scope.Start, scope.End, watermark, scope.Namespace, service, limit}
source = "endpoint_rollup + raw spans (histogram upper-bound percentiles)"
source = "endpoint_rollup + raw spans (interpolated histogram percentiles)"
}
rows, err := s.db.QueryContext(ctx, query, args...)
if err != nil {
Expand All @@ -329,7 +298,20 @@ func (s *Service) queryEndpoints(ctx context.Context, scope Scope, service strin
endpoints := make([]Endpoint, 0)
for rows.Next() {
var endpoint Endpoint
if err := rows.Scan(&endpoint.Method, &endpoint.Path, &endpoint.Calls, &endpoint.P50MS, &endpoint.P95MS, &endpoint.P99MS, &endpoint.ErrorRate); err != nil {
if ready {
var total float64
cumulative := make([]float64, len(endpointDurationBuckets))
targets := []any{&endpoint.Method, &endpoint.Path, &endpoint.Calls, &endpoint.ErrorRate, &total}
for i := range cumulative {
targets = append(targets, &cumulative[i])
}
if err := rows.Scan(targets...); err != nil {
return nil, "", fmt.Errorf("scan endpoint: %w", err)
}
endpoint.P50MS = histogramQuantile(cumulative, total, 0.50)
endpoint.P95MS = histogramQuantile(cumulative, total, 0.95)
endpoint.P99MS = histogramQuantile(cumulative, total, 0.99)
} else if err := rows.Scan(&endpoint.Method, &endpoint.Path, &endpoint.Calls, &endpoint.P50MS, &endpoint.P95MS, &endpoint.P99MS, &endpoint.ErrorRate); err != nil {
return nil, "", fmt.Errorf("scan endpoint: %w", err)
}
endpoint.Health = classify(endpoint.ErrorRate, endpoint.P95MS)
Expand All @@ -341,6 +323,75 @@ func (s *Service) queryEndpoints(ctx context.Context, scope Scope, service strin
return endpoints, source, nil
}

// endpointDurationBuckets pairs each histogram boundary with the column that
// counts it, and both the query's column list and the interpolation read from
// here.
//
// The two were written out separately at first, which couples them by position
// alone: reordering one list, or adding a boundary to the rollup without adding
// it here, would go on compiling and quietly report one bucket's count against
// another's boundary. Percentiles would shift and nothing would look broken.
// The boundaries themselves must stay in step with the histogram the rollup
// writes (endpointRollupInsertSQL, in internal/query). Nothing here can check
// that at compile time; TestEndpointRollupQueryMergesBucketsAndExactBoundaries
// is what catches a drift, because it fills a real rollup row by column name
// and reads the percentiles back out.
var endpointDurationBuckets = []struct {
Column string
Bound float64
}{
{"le_0_1", 0.1}, {"le_0_5", 0.5}, {"le_1", 1}, {"le_2_5", 2.5},
{"le_5", 5}, {"le_10", 10}, {"le_25", 25}, {"le_50", 50},
{"le_100", 100}, {"le_250", 250}, {"le_500", 500}, {"le_750", 750},
{"le_1000", 1000}, {"le_2000", 2000}, {"le_5000", 5000},
{"le_30000", 30000}, {"le_300000", 300000},
}

// endpointDurationColumns is the SELECT list for those counts, in the order
// histogramQuantile expects to read them.
func endpointDurationColumns() string {
names := make([]string, len(endpointDurationBuckets))
for i, bucket := range endpointDurationBuckets {
names[i] = bucket.Column
}
return strings.Join(names, ", ")
}

// histogramQuantile reads a quantile out of cumulative bucket counts,
// interpolating inside the bucket it lands in.
//
// Reporting the bucket's upper bound instead — which is what the rollup query
// used to do — makes a table that reads as broken: every endpoint whose P95 and
// P99 fall in one bucket shows the same number twice, and a page of "25.0ms /
// 25.0ms" invites the reader to distrust the whole view. Interpolation also
// stops a busy endpoint reporting 50ms when almost all of its calls sit just
// above 25ms.
//
// Values beyond the last boundary are reported at that boundary: the rollup
// stops counting there, so it is the most the data can support.
func histogramQuantile(cumulative []float64, total, quantile float64) float64 {
if total <= 0 || len(cumulative) == 0 {
return 0
}
rank := quantile * total
previousBound, previousCount := 0.0, 0.0
for i, count := range cumulative {
if i >= len(endpointDurationBuckets) {
break
}
bound := endpointDurationBuckets[i].Bound
if count >= rank {
width, inBucket := bound-previousBound, count-previousCount
if width <= 0 || inBucket <= 0 {
return bound
}
return previousBound + width*((rank-previousCount)/inBucket)
}
previousBound, previousCount = bound, count
}
return endpointDurationBuckets[len(endpointDurationBuckets)-1].Bound
}

func (s *Service) endpointCacheState(ctx context.Context) (bool, time.Time, error) {
rows, err := s.db.QueryContext(ctx, endpointRollupStatusQuery)
if err != nil {
Expand Down Expand Up @@ -463,6 +514,12 @@ func totalsOf(before, after performanceAggregate) PerformanceTotals {
return totals
}

// latencyNoiseMS is the smallest latency move worth calling a change. A p50
// that goes from 3.9ms to 4.0ms is not a regression, but as a percentage it is
// 2.2% — enough to have been painted red beside an arrow, which spends the
// colour that should mean "look here" on sampling noise.
const latencyNoiseMS = 1.0

func comparisonMetric(label, unit string, before, after float64, lowerIsBetter bool) ComparisonMetric {
change := 0.0
if before != 0 {
Expand All @@ -471,6 +528,12 @@ func comparisonMetric(label, unit string, before, after float64, lowerIsBetter b
change = 100
}
direction := DirectionStable
if unit == "ms" && math.Abs(after-before) < latencyNoiseMS {
// The percentage goes too. Passing it through printed "↑ 100.0%" inside
// a grey "stable" badge whenever the earlier half had no traffic — the
// same number-against-colour contradiction this floor exists to remove.
return ComparisonMetric{Label: label, Unit: unit, Before: before, After: after, Direction: direction}
}
if math.Abs(change) >= 1 {
improved := change > 0
if lowerIsBetter {
Expand Down
14 changes: 11 additions & 3 deletions internal/observability/performance_rollup_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -109,12 +109,20 @@ WHERE bucket >= ? AND bucket < ? AND namespace = 'prod' AND service = 'checkout'
if err != nil {
t.Fatalf("queryEndpoints: %v", err)
}
if source != "endpoint_rollup + raw spans (histogram upper-bound percentiles)" || len(got) != 1 {
if source != "endpoint_rollup + raw spans (interpolated histogram percentiles)" || len(got) != 1 {
t.Fatalf("queryEndpoints = (%#v, %q), want one rollup endpoint", got, source)
}
endpoint := got[0]
if endpoint.Calls != 6 || endpoint.P50MS != 100 || endpoint.P95MS != 2000 || endpoint.P99MS != 2000 {
t.Fatalf("endpoint = %#v, want calls=6 p50=100 p95=p99=2000", endpoint)
// The six merged calls are 5, 10, 100, 500, 1000 and 2000ms. Reporting the
// containing bucket's upper bound put P95 and P99 both at 2000ms — the same
// number twice, which reads as a broken table. Interpolating inside the
// 1000–2000ms bucket separates them: rank 5.7 of 6 lands at 1700ms, rank
// 5.94 at 1940ms.
if endpoint.Calls != 6 || endpoint.P50MS != 100 {
t.Fatalf("endpoint = %#v, want calls=6 p50=100", endpoint)
}
if math.Abs(endpoint.P95MS-1700) > 1e-6 || math.Abs(endpoint.P99MS-1940) > 1e-6 {
t.Fatalf("p95/p99 = %v/%v, want 1700/1940 interpolated inside the bucket", endpoint.P95MS, endpoint.P99MS)
}
if math.Abs(endpoint.ErrorRate-2.0/6.0) > 1e-9 {
t.Fatalf("error rate = %v, want %v", endpoint.ErrorRate, 2.0/6.0)
Expand Down
21 changes: 18 additions & 3 deletions internal/observability/service.go
Original file line number Diff line number Diff line change
Expand Up @@ -71,10 +71,16 @@ func (s *Service) normalizeScope(scope Scope) (Scope, error) {
return scope, nil
}

// timelineBucketWidth keeps a chart readable as the window grows. A day at five
// minutes is 288 points drawn across a card a few hundred pixels wide: the line
// becomes a band and its shape stops being legible, so the buckets widen with
// the window instead.
func timelineBucketWidth(window time.Duration) string {
switch {
case window <= 24*time.Hour:
case window <= 6*time.Hour:
return "5 minutes"
case window <= 24*time.Hour:
return "15 minutes"
case window <= 7*24*time.Hour:
return "30 minutes"
case window <= 30*24*time.Hour:
Expand Down Expand Up @@ -109,11 +115,20 @@ func (s *Service) provenanceFor(scope Scope, source string) Provenance {
}
}

// The thresholds a service or endpoint is graded against. ui/health.ts mirrors
// them so a table can colour each figure by the signal that produced it.
const (
latencyDegradedThresholdMS = 750.0
latencyUnhealthyThresholdMS = 2000.0
errorRateDegraded = 0.01
errorRateUnhealthy = 0.05
)

func classify(errorRate, p95MS float64) Health {
switch {
case errorRate >= 0.05 || p95MS >= 2000:
case errorRate >= errorRateUnhealthy || p95MS >= latencyUnhealthyThresholdMS:
return HealthUnhealthy
case errorRate >= 0.01 || p95MS >= 750:
case errorRate >= errorRateDegraded || p95MS >= latencyDegradedThresholdMS:
return HealthDegraded
default:
return HealthHealthy
Expand Down
Loading
Loading