From ad7cb2cddd866b4d834fdd6192a916f1067b08ba Mon Sep 17 00:00:00 2001 From: Vishal Rana Date: Fri, 16 Oct 2026 12:37:26 -0700 Subject: [PATCH] fix(intelligence): require a meaningful change before reporting latency and volume anomalies Latency and volume anomalies fired on the z-score alone. The baseline is the scatter of a few five-minute buckets, so a steady service has a tiny standard deviation and a 0.1 ms or 5% shift scored as a severe anomaly. One hour of the OpenTelemetry Demo produced 36 findings across 18 services, and the dashboard's anomaly overlay covered every panel for the whole hour. Both checks now also need practical significance, in the style of the existing error-rate floor: - latency: p95 must rise by at least 20% and at least 5 ms, with a 1 ms floor on the baseline scatter; - volume: the count must change by at least 25% in either direction, with a one-span floor on the scatter. A service present in the baseline but silent now is evaluated, so traffic stopping is reported. Only qualified findings affect severity, insights and the health score. Replaying the detector over one hour of a Demo snapshot reduced findings from 87 to 2; the remaining one is a frontend-web p95 rise from 202 to 243 ms. Claude-Session: https://claude.ai/code/session_01XA52nWyQ3R5f5kK8kjpAQW --- internal/intelligence/detector.go | 43 ++++- internal/intelligence/effect_size_test.go | 211 ++++++++++++++++++++++ 2 files changed, 245 insertions(+), 9 deletions(-) create mode 100644 internal/intelligence/effect_size_test.go diff --git a/internal/intelligence/detector.go b/internal/intelligence/detector.go index f7abd0547..4a684c8f7 100644 --- a/internal/intelligence/detector.go +++ b/internal/intelligence/detector.go @@ -297,6 +297,26 @@ func (d *Detector) detectAnomalies(ctx context.Context, start, end time.Time) ([ // percentage point. See errorRateAnomalySQL. const minErrorRateStddev = 0.01 +// minLatencyIncreaseRatio requires a 20% slowdown, keeping small proportional +// shifts in slow services quiet while allowing substantial regressions through. +const minLatencyIncreaseRatio = 0.20 + +// minLatencyIncreaseMS requires at least 5ms of extra p95 latency: even a large +// ratio on a sub-millisecond service is noise, but 20ms to 60ms still qualifies. +const minLatencyIncreaseMS = 5.0 + +// minLatencyStddev assumes at least 1ms of timing scatter, so a flat baseline +// can still detect regressions without amplifying fractional-millisecond jitter. +const minLatencyStddev = 1.0 + +// minVolumeChangeRatio requires a 25% traffic change in either direction, +// suppressing small count fluctuations while retaining 30% drops and doubling. +const minVolumeChangeRatio = 0.25 + +// minVolumeStddev assumes at least one span of counting uncertainty, allowing +// flat baselines to detect traffic stopping or surging without dividing by zero. +const minVolumeStddev = 1.0 + // errorRateAnomalySQL compares the error rate in [startNano, endNano) against // the window of equal length immediately before it. // @@ -428,7 +448,9 @@ func (d *Detector) detectLatencyAnomalies(ctx context.Context, start, end time.T baselineP95 := row.baseline zScore := row.zScore - if zScore >= d.config.LatencyThreshold { + increase := currentP95 - baselineP95 + if zScore >= d.config.LatencyThreshold && + increase >= minLatencyIncreaseMS && increase >= baselineP95*minLatencyIncreaseRatio { anomalies = append(anomalies, Anomaly{ Type: AnomalyLatencyDegradation, ServiceName: serviceName, @@ -458,6 +480,8 @@ func (d *Detector) detectLatencyAnomalies(ctx context.Context, start, end time.T // the traffic did. Every service was permanently a critical volume anomaly: // steady traffic scored z=75 in TestVolumeAnomalyComparesEqualWindows, and the // live demo sat at health score 0 with 17 meaningless critical insights. +// Retain baseline-only services with a zero current count so traffic stopping +// is evaluated even though the rollup has no positive-span rows for that window. func volumeAnomalySQL(startNano, endNano int64, scope string) string { return fmt.Sprintf(` WITH current_period AS ( @@ -492,16 +516,16 @@ func volumeAnomalySQL(startNano, endNano int64, scope string) string { GROUP BY service_name ) SELECT - c.service_name, - c.span_count::DOUBLE AS current_count, + COALESCE(c.service_name, b.service_name) AS service_name, + COALESCE(c.span_count, 0.0)::DOUBLE AS current_count, COALESCE(b.avg_count, 0.0) AS baseline_count, CASE - WHEN b.count_stddev > 0 THEN (c.span_count - b.avg_count) / b.count_stddev + WHEN b.avg_count > 0 THEN (COALESCE(c.span_count, 0.0) - b.avg_count) / GREATEST(COALESCE(b.count_stddev, 0.0), %f) ELSE 0.0 END AS z_score FROM current_period c - LEFT JOIN baseline_period b ON c.service_name = b.service_name - `, startNano, endNano, scope, startNano-endNano+startNano, startNano, scope) + FULL OUTER JOIN baseline_period b ON c.service_name = b.service_name + `, startNano, endNano, scope, startNano-endNano+startNano, startNano, scope, minVolumeStddev) } // detectVolumeAnomalies detects unusual traffic volume changes @@ -525,7 +549,8 @@ func (d *Detector) detectVolumeAnomalies(ctx context.Context, start, end time.Ti baselineCount := row.baseline zScore := row.zScore - if math.Abs(zScore) >= d.config.VolumeThreshold { + if math.Abs(zScore) >= d.config.VolumeThreshold && baselineCount > 0 && + math.Abs(currentCount-baselineCount) >= baselineCount*minVolumeChangeRatio { anomalies = append(anomalies, Anomaly{ Type: AnomalyVolumeChange, ServiceName: serviceName, @@ -581,12 +606,12 @@ func latencyAnomalySQL(startNano, endNano int64, scope string) string { c.p95_latency AS current_p95, COALESCE(b.avg_p95, 0.0) AS baseline_p95, CASE - WHEN b.p95_stddev > 0 THEN (c.p95_latency - b.avg_p95) / b.p95_stddev + WHEN b.avg_p95 IS NOT NULL THEN (c.p95_latency - b.avg_p95) / GREATEST(COALESCE(b.p95_stddev, 0.0), %f) ELSE 0.0 END AS z_score FROM current_period c LEFT JOIN baseline_period b ON c.service_name = b.service_name - `, startNano, endNano, scope, startNano-endNano+startNano, startNano, scope) + `, startNano, endNano, scope, startNano-endNano+startNano, startNano, scope, minLatencyStddev) } diff --git a/internal/intelligence/effect_size_test.go b/internal/intelligence/effect_size_test.go new file mode 100644 index 000000000..5e094f750 --- /dev/null +++ b/internal/intelligence/effect_size_test.go @@ -0,0 +1,211 @@ +package intelligence + +import ( + "database/sql" + "fmt" + "math" + "testing" + "time" + + "github.com/labstack/fanout/internal/query" +) + +func TestLatencyAnomaliesRequireStatisticalAndPracticalSignificance(t *testing.T) { + db := effectSizeDB(t) + start := time.Date(2026, 10, 1, 12, 15, 0, 0, time.UTC) + end := start.Add(15 * time.Minute) + tests := []struct { + name string + baseline, current, stddev float64 + want bool + }{ + {"ad", 1.3, 1.4, .01, false}, + {"flagd", .3, .3001, .00001, false}, + {"email", 5.8, 5.8001, .00001, false}, + {"sub_millisecond_doubling", .3, .6, .001, false}, + {"real_regression", 20, 60, 1, true}, + {"flat_baseline_regression", 20, 60, 0, true}, + {"absolute_below", 10, 15 - 1e-6, .01, false}, + {"absolute_at", 10, 15, .01, true}, + {"absolute_above", 10, 15 + 1e-6, .01, true}, + {"relative_below", 100, 120 - 1e-6, .01, false}, + {"relative_at", 100, 120, .01, true}, + {"relative_above", 100, 120 + 1e-6, .01, true}, + {"noisy_baseline", 100, 125, 30, false}, + {"statistical_below", 20, 30 - 1e-6, 5, false}, + {"statistical_at", 20, 30, 5, true}, + {"improvement", 60, 20, 1, false}, + } + for _, tt := range tests { + for i := range 6 { + p95 := tt.current + if i < 3 { + p95 = tt.baseline + float64(i-1)*tt.stddev + } + if _, err := db.Exec(`INSERT INTO service_rollup VALUES (?, 'default', ?, 100, ?)`, tt.name, start.Add(time.Duration(i-3)*5*time.Minute), p95); err != nil { + t.Fatal(err) + } + } + } + d := NewDetector(&query.Duck{DB: db}, DefaultDetectorConfig()) + anomalies, err := d.detectLatencyAnomalies(t.Context(), start, end) + if err != nil { + t.Fatal(err) + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + assertEffectSizeFinding(t, d, anomalies, tt.name, AnomalyLatencyDegradation, tt.want, start, end) + }) + } +} + +func TestVolumeAnomaliesRequireStatisticalAndPracticalSignificance(t *testing.T) { + db := effectSizeDB(t) + start := time.Date(2026, 10, 1, 12, 15, 0, 0, time.UTC) + end := start.Add(15 * time.Minute) + tests := []struct { + name string + baseline, current, jitter int + want bool + }{ + {"payment", 250, 263, 1, false}, + {"ad", 730, 779, 1, false}, + {"currency", 1274, 935, 12, true}, + {"frontend_proxy", 32031, 21573, 481, true}, + {"silent", 100, 0, 1, true}, + {"flat_baseline_silent", 100, 0, 0, true}, + {"doubling", 100, 200, 1, true}, + {"flat_baseline_doubling", 100, 200, 0, true}, + {"thirty_percent_drop", 100, 70, 1, true}, + {"surge_below", 100000, 124999, 1, false}, + {"surge_at", 100000, 125000, 1, true}, + {"surge_above", 100000, 125001, 1, true}, + {"drop_below", 100000, 75001, 1, false}, + {"drop_at", 100000, 75000, 1, true}, + {"drop_above", 100000, 74999, 1, true}, + {"noisy_baseline", 100, 130, 30, false}, + {"statistical_below", 100, 149, 20, false}, + {"statistical_at", 100, 150, 20, true}, + {"steady", 100, 100, 0, false}, + {"no_baseline", 0, 100, 0, false}, + } + for _, tt := range tests { + for i := range 6 { + count := tt.current + if i < 3 { + count = tt.baseline + (i-1)*tt.jitter + } + // Silence has no current rollup rows, rather than synthetic zeros. + if count == 0 { + continue + } + if _, err := db.Exec(`INSERT INTO service_rollup VALUES (?, 'default', ?, ?, 10)`, tt.name, start.Add(time.Duration(i-3)*5*time.Minute), count); err != nil { + t.Fatal(err) + } + } + } + d := NewDetector(&query.Duck{DB: db}, DefaultDetectorConfig()) + anomalies, err := d.detectVolumeAnomalies(t.Context(), start, end) + if err != nil { + t.Fatal(err) + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + assertEffectSizeFinding(t, d, anomalies, tt.name, AnomalyVolumeChange, tt.want, start, end) + }) + } +} + +func TestAnomalyScatterFloorsRetainFlatAndSingleBucketBaselines(t *testing.T) { + for _, kind := range []string{"latency", "volume"} { + for _, scatter := range []int{0, 1, 2} { + for _, buckets := range []int{1, 3} { + t.Run(fmt.Sprintf("%s/scatter_%d/buckets_%d", kind, scatter, buckets), func(t *testing.T) { + db := effectSizeDB(t) + start := time.Date(2026, 10, 1, 12, 15, 0, 0, time.UTC) + end := start.Add(15 * time.Minute) + for i := range buckets { + value := 100 + (i-buckets/2)*scatter + spans, p95 := 100, 100 + if kind == "latency" { + p95 = value + } else { + spans = value + } + if _, err := db.Exec(`INSERT INTO service_rollup VALUES ('api', 'default', ?, ?, ?)`, start.Add(time.Duration(i-3)*5*time.Minute), spans, p95); err != nil { + t.Fatal(err) + } + } + if _, err := db.Exec(`INSERT INTO service_rollup VALUES ('api', 'default', ?, 150, 150)`, start); err != nil { + t.Fatal(err) + } + statement := latencyAnomalySQL(start.UnixNano(), end.UnixNano(), "") + if kind == "volume" { + statement = volumeAnomalySQL(start.UnixNano(), end.UnixNano(), "") + } + var service string + var current, baseline, z float64 + if err := db.QueryRow(statement).Scan(&service, ¤t, &baseline, &z); err != nil { + t.Fatal(err) + } + denominator := 1.0 + if buckets == 3 { + denominator = math.Max(float64(scatter), 1) + } + if want := 50 / denominator; math.Abs(z-want) > 1e-9 { + t.Fatalf("z=%g, want %g", z, want) + } + }) + } + } + } +} + +func effectSizeDB(t *testing.T) *sql.DB { + t.Helper() + db, err := sql.Open("duckdb", "") + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { _ = db.Close() }) + if _, err := db.Exec(`CREATE TABLE service_rollup (service TEXT, namespace TEXT, bucket TIMESTAMP_NS, spans BIGINT, p95_ms DOUBLE)`); err != nil { + t.Fatal(err) + } + return db +} + +func assertEffectSizeFinding(t *testing.T, d *Detector, anomalies []Anomaly, service string, kind AnomalyType, want bool, start, end time.Time) { + t.Helper() + var found []Anomaly + for _, a := range anomalies { + if a.ServiceName == service { + found = append(found, a) + } + } + if !want { + if len(found) != 0 { + t.Fatalf("noise emitted findings: %+v", found) + } + if score := d.calculateHealthScore(found, nil); score != 100 { + t.Fatalf("noise lowered health to %g", score) + } + if len(detectorAnnotations(IntelligenceSnapshot{Anomalies: found}, "default")) != 0 || len(d.generateInsights(found, nil)) != 0 { + t.Fatal("noise emitted annotations or insights") + } + return + } + if len(found) != 1 || found[0].Type != kind || !found[0].WindowStart.Equal(start) || !found[0].WindowEnd.Equal(end) { + t.Fatalf("expected one qualified finding in the evaluated window: %+v", found) + } + if score := d.calculateHealthScore(found, nil); score >= 100 { + t.Fatalf("qualified finding did not lower health: %g", score) + } + annotations := detectorAnnotations(IntelligenceSnapshot{Anomalies: found}, "default") + severity := "warn" + if math.Abs(found[0].ZScore) >= 4 { + severity = "bad" + } + if len(annotations) != 1 || annotations[0].Severity != severity { + t.Fatalf("qualified finding severity: %+v", annotations) + } +}