diff --git a/internal/intelligence/detector.go b/internal/intelligence/detector.go index f7abd0547..4a684c8f7 100644 --- a/internal/intelligence/detector.go +++ b/internal/intelligence/detector.go @@ -297,6 +297,26 @@ func (d *Detector) detectAnomalies(ctx context.Context, start, end time.Time) ([ // percentage point. See errorRateAnomalySQL. const minErrorRateStddev = 0.01 +// minLatencyIncreaseRatio requires a 20% slowdown, keeping small proportional +// shifts in slow services quiet while allowing substantial regressions through. +const minLatencyIncreaseRatio = 0.20 + +// minLatencyIncreaseMS requires at least 5ms of extra p95 latency: even a large +// ratio on a sub-millisecond service is noise, but 20ms to 60ms still qualifies. +const minLatencyIncreaseMS = 5.0 + +// minLatencyStddev assumes at least 1ms of timing scatter, so a flat baseline +// can still detect regressions without amplifying fractional-millisecond jitter. +const minLatencyStddev = 1.0 + +// minVolumeChangeRatio requires a 25% traffic change in either direction, +// suppressing small count fluctuations while retaining 30% drops and doubling. +const minVolumeChangeRatio = 0.25 + +// minVolumeStddev assumes at least one span of counting uncertainty, allowing +// flat baselines to detect traffic stopping or surging without dividing by zero. +const minVolumeStddev = 1.0 + // errorRateAnomalySQL compares the error rate in [startNano, endNano) against // the window of equal length immediately before it. // @@ -428,7 +448,9 @@ func (d *Detector) detectLatencyAnomalies(ctx context.Context, start, end time.T baselineP95 := row.baseline zScore := row.zScore - if zScore >= d.config.LatencyThreshold { + increase := currentP95 - baselineP95 + if zScore >= d.config.LatencyThreshold && + increase >= minLatencyIncreaseMS && increase >= baselineP95*minLatencyIncreaseRatio { anomalies = append(anomalies, Anomaly{ Type: AnomalyLatencyDegradation, ServiceName: serviceName, @@ -458,6 +480,8 @@ func (d *Detector) detectLatencyAnomalies(ctx context.Context, start, end time.T // the traffic did. Every service was permanently a critical volume anomaly: // steady traffic scored z=75 in TestVolumeAnomalyComparesEqualWindows, and the // live demo sat at health score 0 with 17 meaningless critical insights. +// Retain baseline-only services with a zero current count so traffic stopping +// is evaluated even though the rollup has no positive-span rows for that window. func volumeAnomalySQL(startNano, endNano int64, scope string) string { return fmt.Sprintf(` WITH current_period AS ( @@ -492,16 +516,16 @@ func volumeAnomalySQL(startNano, endNano int64, scope string) string { GROUP BY service_name ) SELECT - c.service_name, - c.span_count::DOUBLE AS current_count, + COALESCE(c.service_name, b.service_name) AS service_name, + COALESCE(c.span_count, 0.0)::DOUBLE AS current_count, COALESCE(b.avg_count, 0.0) AS baseline_count, CASE - WHEN b.count_stddev > 0 THEN (c.span_count - b.avg_count) / b.count_stddev + WHEN b.avg_count > 0 THEN (COALESCE(c.span_count, 0.0) - b.avg_count) / GREATEST(COALESCE(b.count_stddev, 0.0), %f) ELSE 0.0 END AS z_score FROM current_period c - LEFT JOIN baseline_period b ON c.service_name = b.service_name - `, startNano, endNano, scope, startNano-endNano+startNano, startNano, scope) + FULL OUTER JOIN baseline_period b ON c.service_name = b.service_name + `, startNano, endNano, scope, startNano-endNano+startNano, startNano, scope, minVolumeStddev) } // detectVolumeAnomalies detects unusual traffic volume changes @@ -525,7 +549,8 @@ func (d *Detector) detectVolumeAnomalies(ctx context.Context, start, end time.Ti baselineCount := row.baseline zScore := row.zScore - if math.Abs(zScore) >= d.config.VolumeThreshold { + if math.Abs(zScore) >= d.config.VolumeThreshold && baselineCount > 0 && + math.Abs(currentCount-baselineCount) >= baselineCount*minVolumeChangeRatio { anomalies = append(anomalies, Anomaly{ Type: AnomalyVolumeChange, ServiceName: serviceName, @@ -581,12 +606,12 @@ func latencyAnomalySQL(startNano, endNano int64, scope string) string { c.p95_latency AS current_p95, COALESCE(b.avg_p95, 0.0) AS baseline_p95, CASE - WHEN b.p95_stddev > 0 THEN (c.p95_latency - b.avg_p95) / b.p95_stddev + WHEN b.avg_p95 IS NOT NULL THEN (c.p95_latency - b.avg_p95) / GREATEST(COALESCE(b.p95_stddev, 0.0), %f) ELSE 0.0 END AS z_score FROM current_period c LEFT JOIN baseline_period b ON c.service_name = b.service_name - `, startNano, endNano, scope, startNano-endNano+startNano, startNano, scope) + `, startNano, endNano, scope, startNano-endNano+startNano, startNano, scope, minLatencyStddev) } diff --git a/internal/intelligence/effect_size_test.go b/internal/intelligence/effect_size_test.go new file mode 100644 index 000000000..5e094f750 --- /dev/null +++ b/internal/intelligence/effect_size_test.go @@ -0,0 +1,211 @@ +package intelligence + +import ( + "database/sql" + "fmt" + "math" + "testing" + "time" + + "github.com/labstack/fanout/internal/query" +) + +func TestLatencyAnomaliesRequireStatisticalAndPracticalSignificance(t *testing.T) { + db := effectSizeDB(t) + start := time.Date(2026, 10, 1, 12, 15, 0, 0, time.UTC) + end := start.Add(15 * time.Minute) + tests := []struct { + name string + baseline, current, stddev float64 + want bool + }{ + {"ad", 1.3, 1.4, .01, false}, + {"flagd", .3, .3001, .00001, false}, + {"email", 5.8, 5.8001, .00001, false}, + {"sub_millisecond_doubling", .3, .6, .001, false}, + {"real_regression", 20, 60, 1, true}, + {"flat_baseline_regression", 20, 60, 0, true}, + {"absolute_below", 10, 15 - 1e-6, .01, false}, + {"absolute_at", 10, 15, .01, true}, + {"absolute_above", 10, 15 + 1e-6, .01, true}, + {"relative_below", 100, 120 - 1e-6, .01, false}, + {"relative_at", 100, 120, .01, true}, + {"relative_above", 100, 120 + 1e-6, .01, true}, + {"noisy_baseline", 100, 125, 30, false}, + {"statistical_below", 20, 30 - 1e-6, 5, false}, + {"statistical_at", 20, 30, 5, true}, + {"improvement", 60, 20, 1, false}, + } + for _, tt := range tests { + for i := range 6 { + p95 := tt.current + if i < 3 { + p95 = tt.baseline + float64(i-1)*tt.stddev + } + if _, err := db.Exec(`INSERT INTO service_rollup VALUES (?, 'default', ?, 100, ?)`, tt.name, start.Add(time.Duration(i-3)*5*time.Minute), p95); err != nil { + t.Fatal(err) + } + } + } + d := NewDetector(&query.Duck{DB: db}, DefaultDetectorConfig()) + anomalies, err := d.detectLatencyAnomalies(t.Context(), start, end) + if err != nil { + t.Fatal(err) + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + assertEffectSizeFinding(t, d, anomalies, tt.name, AnomalyLatencyDegradation, tt.want, start, end) + }) + } +} + +func TestVolumeAnomaliesRequireStatisticalAndPracticalSignificance(t *testing.T) { + db := effectSizeDB(t) + start := time.Date(2026, 10, 1, 12, 15, 0, 0, time.UTC) + end := start.Add(15 * time.Minute) + tests := []struct { + name string + baseline, current, jitter int + want bool + }{ + {"payment", 250, 263, 1, false}, + {"ad", 730, 779, 1, false}, + {"currency", 1274, 935, 12, true}, + {"frontend_proxy", 32031, 21573, 481, true}, + {"silent", 100, 0, 1, true}, + {"flat_baseline_silent", 100, 0, 0, true}, + {"doubling", 100, 200, 1, true}, + {"flat_baseline_doubling", 100, 200, 0, true}, + {"thirty_percent_drop", 100, 70, 1, true}, + {"surge_below", 100000, 124999, 1, false}, + {"surge_at", 100000, 125000, 1, true}, + {"surge_above", 100000, 125001, 1, true}, + {"drop_below", 100000, 75001, 1, false}, + {"drop_at", 100000, 75000, 1, true}, + {"drop_above", 100000, 74999, 1, true}, + {"noisy_baseline", 100, 130, 30, false}, + {"statistical_below", 100, 149, 20, false}, + {"statistical_at", 100, 150, 20, true}, + {"steady", 100, 100, 0, false}, + {"no_baseline", 0, 100, 0, false}, + } + for _, tt := range tests { + for i := range 6 { + count := tt.current + if i < 3 { + count = tt.baseline + (i-1)*tt.jitter + } + // Silence has no current rollup rows, rather than synthetic zeros. + if count == 0 { + continue + } + if _, err := db.Exec(`INSERT INTO service_rollup VALUES (?, 'default', ?, ?, 10)`, tt.name, start.Add(time.Duration(i-3)*5*time.Minute), count); err != nil { + t.Fatal(err) + } + } + } + d := NewDetector(&query.Duck{DB: db}, DefaultDetectorConfig()) + anomalies, err := d.detectVolumeAnomalies(t.Context(), start, end) + if err != nil { + t.Fatal(err) + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + assertEffectSizeFinding(t, d, anomalies, tt.name, AnomalyVolumeChange, tt.want, start, end) + }) + } +} + +func TestAnomalyScatterFloorsRetainFlatAndSingleBucketBaselines(t *testing.T) { + for _, kind := range []string{"latency", "volume"} { + for _, scatter := range []int{0, 1, 2} { + for _, buckets := range []int{1, 3} { + t.Run(fmt.Sprintf("%s/scatter_%d/buckets_%d", kind, scatter, buckets), func(t *testing.T) { + db := effectSizeDB(t) + start := time.Date(2026, 10, 1, 12, 15, 0, 0, time.UTC) + end := start.Add(15 * time.Minute) + for i := range buckets { + value := 100 + (i-buckets/2)*scatter + spans, p95 := 100, 100 + if kind == "latency" { + p95 = value + } else { + spans = value + } + if _, err := db.Exec(`INSERT INTO service_rollup VALUES ('api', 'default', ?, ?, ?)`, start.Add(time.Duration(i-3)*5*time.Minute), spans, p95); err != nil { + t.Fatal(err) + } + } + if _, err := db.Exec(`INSERT INTO service_rollup VALUES ('api', 'default', ?, 150, 150)`, start); err != nil { + t.Fatal(err) + } + statement := latencyAnomalySQL(start.UnixNano(), end.UnixNano(), "") + if kind == "volume" { + statement = volumeAnomalySQL(start.UnixNano(), end.UnixNano(), "") + } + var service string + var current, baseline, z float64 + if err := db.QueryRow(statement).Scan(&service, ¤t, &baseline, &z); err != nil { + t.Fatal(err) + } + denominator := 1.0 + if buckets == 3 { + denominator = math.Max(float64(scatter), 1) + } + if want := 50 / denominator; math.Abs(z-want) > 1e-9 { + t.Fatalf("z=%g, want %g", z, want) + } + }) + } + } + } +} + +func effectSizeDB(t *testing.T) *sql.DB { + t.Helper() + db, err := sql.Open("duckdb", "") + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { _ = db.Close() }) + if _, err := db.Exec(`CREATE TABLE service_rollup (service TEXT, namespace TEXT, bucket TIMESTAMP_NS, spans BIGINT, p95_ms DOUBLE)`); err != nil { + t.Fatal(err) + } + return db +} + +func assertEffectSizeFinding(t *testing.T, d *Detector, anomalies []Anomaly, service string, kind AnomalyType, want bool, start, end time.Time) { + t.Helper() + var found []Anomaly + for _, a := range anomalies { + if a.ServiceName == service { + found = append(found, a) + } + } + if !want { + if len(found) != 0 { + t.Fatalf("noise emitted findings: %+v", found) + } + if score := d.calculateHealthScore(found, nil); score != 100 { + t.Fatalf("noise lowered health to %g", score) + } + if len(detectorAnnotations(IntelligenceSnapshot{Anomalies: found}, "default")) != 0 || len(d.generateInsights(found, nil)) != 0 { + t.Fatal("noise emitted annotations or insights") + } + return + } + if len(found) != 1 || found[0].Type != kind || !found[0].WindowStart.Equal(start) || !found[0].WindowEnd.Equal(end) { + t.Fatalf("expected one qualified finding in the evaluated window: %+v", found) + } + if score := d.calculateHealthScore(found, nil); score >= 100 { + t.Fatalf("qualified finding did not lower health: %g", score) + } + annotations := detectorAnnotations(IntelligenceSnapshot{Anomalies: found}, "default") + severity := "warn" + if math.Abs(found[0].ZScore) >= 4 { + severity = "bad" + } + if len(annotations) != 1 || annotations[0].Severity != severity { + t.Fatalf("qualified finding severity: %+v", annotations) + } +}