Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
43 changes: 34 additions & 9 deletions internal/intelligence/detector.go
Original file line number Diff line number Diff line change
Expand Up @@ -297,6 +297,26 @@ func (d *Detector) detectAnomalies(ctx context.Context, start, end time.Time) ([
// percentage point. See errorRateAnomalySQL.
const minErrorRateStddev = 0.01

// minLatencyIncreaseRatio requires a 20% slowdown, keeping small proportional
// shifts in slow services quiet while allowing substantial regressions through.
const minLatencyIncreaseRatio = 0.20

// minLatencyIncreaseMS requires at least 5ms of extra p95 latency: even a large
// ratio on a sub-millisecond service is noise, but 20ms to 60ms still qualifies.
const minLatencyIncreaseMS = 5.0

// minLatencyStddev assumes at least 1ms of timing scatter, so a flat baseline
// can still detect regressions without amplifying fractional-millisecond jitter.
const minLatencyStddev = 1.0

// minVolumeChangeRatio requires a 25% traffic change in either direction,
// suppressing small count fluctuations while retaining 30% drops and doubling.
const minVolumeChangeRatio = 0.25

// minVolumeStddev assumes at least one span of counting uncertainty, allowing
// flat baselines to detect traffic stopping or surging without dividing by zero.
const minVolumeStddev = 1.0

// errorRateAnomalySQL compares the error rate in [startNano, endNano) against
// the window of equal length immediately before it.
//
Expand Down Expand Up @@ -428,7 +448,9 @@ func (d *Detector) detectLatencyAnomalies(ctx context.Context, start, end time.T
baselineP95 := row.baseline
zScore := row.zScore

if zScore >= d.config.LatencyThreshold {
increase := currentP95 - baselineP95
if zScore >= d.config.LatencyThreshold &&
increase >= minLatencyIncreaseMS && increase >= baselineP95*minLatencyIncreaseRatio {
anomalies = append(anomalies, Anomaly{
Type: AnomalyLatencyDegradation,
ServiceName: serviceName,
Expand Down Expand Up @@ -458,6 +480,8 @@ func (d *Detector) detectLatencyAnomalies(ctx context.Context, start, end time.T
// the traffic did. Every service was permanently a critical volume anomaly:
// steady traffic scored z=75 in TestVolumeAnomalyComparesEqualWindows, and the
// live demo sat at health score 0 with 17 meaningless critical insights.
// Retain baseline-only services with a zero current count so traffic stopping
// is evaluated even though the rollup has no positive-span rows for that window.
func volumeAnomalySQL(startNano, endNano int64, scope string) string {
return fmt.Sprintf(`
WITH current_period AS (
Expand Down Expand Up @@ -492,16 +516,16 @@ func volumeAnomalySQL(startNano, endNano int64, scope string) string {
GROUP BY service_name
)
SELECT
c.service_name,
c.span_count::DOUBLE AS current_count,
COALESCE(c.service_name, b.service_name) AS service_name,
COALESCE(c.span_count, 0.0)::DOUBLE AS current_count,
COALESCE(b.avg_count, 0.0) AS baseline_count,
CASE
WHEN b.count_stddev > 0 THEN (c.span_count - b.avg_count) / b.count_stddev
WHEN b.avg_count > 0 THEN (COALESCE(c.span_count, 0.0) - b.avg_count) / GREATEST(COALESCE(b.count_stddev, 0.0), %f)
ELSE 0.0
END AS z_score
FROM current_period c
LEFT JOIN baseline_period b ON c.service_name = b.service_name
`, startNano, endNano, scope, startNano-endNano+startNano, startNano, scope)
FULL OUTER JOIN baseline_period b ON c.service_name = b.service_name
`, startNano, endNano, scope, startNano-endNano+startNano, startNano, scope, minVolumeStddev)
}

// detectVolumeAnomalies detects unusual traffic volume changes
Expand All @@ -525,7 +549,8 @@ func (d *Detector) detectVolumeAnomalies(ctx context.Context, start, end time.Ti
baselineCount := row.baseline
zScore := row.zScore

if math.Abs(zScore) >= d.config.VolumeThreshold {
if math.Abs(zScore) >= d.config.VolumeThreshold && baselineCount > 0 &&
math.Abs(currentCount-baselineCount) >= baselineCount*minVolumeChangeRatio {
anomalies = append(anomalies, Anomaly{
Type: AnomalyVolumeChange,
ServiceName: serviceName,
Expand Down Expand Up @@ -581,12 +606,12 @@ func latencyAnomalySQL(startNano, endNano int64, scope string) string {
c.p95_latency AS current_p95,
COALESCE(b.avg_p95, 0.0) AS baseline_p95,
CASE
WHEN b.p95_stddev > 0 THEN (c.p95_latency - b.avg_p95) / b.p95_stddev
WHEN b.avg_p95 IS NOT NULL THEN (c.p95_latency - b.avg_p95) / GREATEST(COALESCE(b.p95_stddev, 0.0), %f)
ELSE 0.0
END AS z_score
FROM current_period c
LEFT JOIN baseline_period b ON c.service_name = b.service_name
`, startNano, endNano, scope, startNano-endNano+startNano, startNano, scope)
`, startNano, endNano, scope, startNano-endNano+startNano, startNano, scope, minLatencyStddev)

}

Expand Down
211 changes: 211 additions & 0 deletions internal/intelligence/effect_size_test.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,211 @@
package intelligence

import (
"database/sql"
"fmt"
"math"
"testing"
"time"

"github.com/labstack/fanout/internal/query"
)

func TestLatencyAnomaliesRequireStatisticalAndPracticalSignificance(t *testing.T) {
db := effectSizeDB(t)
start := time.Date(2026, 10, 1, 12, 15, 0, 0, time.UTC)
end := start.Add(15 * time.Minute)
tests := []struct {
name string
baseline, current, stddev float64
want bool
}{
{"ad", 1.3, 1.4, .01, false},
{"flagd", .3, .3001, .00001, false},
{"email", 5.8, 5.8001, .00001, false},
{"sub_millisecond_doubling", .3, .6, .001, false},
{"real_regression", 20, 60, 1, true},
{"flat_baseline_regression", 20, 60, 0, true},
{"absolute_below", 10, 15 - 1e-6, .01, false},
{"absolute_at", 10, 15, .01, true},
{"absolute_above", 10, 15 + 1e-6, .01, true},
{"relative_below", 100, 120 - 1e-6, .01, false},
{"relative_at", 100, 120, .01, true},
{"relative_above", 100, 120 + 1e-6, .01, true},
{"noisy_baseline", 100, 125, 30, false},
{"statistical_below", 20, 30 - 1e-6, 5, false},
{"statistical_at", 20, 30, 5, true},
{"improvement", 60, 20, 1, false},
}
for _, tt := range tests {
for i := range 6 {
p95 := tt.current
if i < 3 {
p95 = tt.baseline + float64(i-1)*tt.stddev
}
if _, err := db.Exec(`INSERT INTO service_rollup VALUES (?, 'default', ?, 100, ?)`, tt.name, start.Add(time.Duration(i-3)*5*time.Minute), p95); err != nil {
t.Fatal(err)
}
}
}
d := NewDetector(&query.Duck{DB: db}, DefaultDetectorConfig())
anomalies, err := d.detectLatencyAnomalies(t.Context(), start, end)
if err != nil {
t.Fatal(err)
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
assertEffectSizeFinding(t, d, anomalies, tt.name, AnomalyLatencyDegradation, tt.want, start, end)
})
}
}

func TestVolumeAnomaliesRequireStatisticalAndPracticalSignificance(t *testing.T) {
db := effectSizeDB(t)
start := time.Date(2026, 10, 1, 12, 15, 0, 0, time.UTC)
end := start.Add(15 * time.Minute)
tests := []struct {
name string
baseline, current, jitter int
want bool
}{
{"payment", 250, 263, 1, false},
{"ad", 730, 779, 1, false},
{"currency", 1274, 935, 12, true},
{"frontend_proxy", 32031, 21573, 481, true},
{"silent", 100, 0, 1, true},
{"flat_baseline_silent", 100, 0, 0, true},
{"doubling", 100, 200, 1, true},
{"flat_baseline_doubling", 100, 200, 0, true},
{"thirty_percent_drop", 100, 70, 1, true},
{"surge_below", 100000, 124999, 1, false},
{"surge_at", 100000, 125000, 1, true},
{"surge_above", 100000, 125001, 1, true},
{"drop_below", 100000, 75001, 1, false},
{"drop_at", 100000, 75000, 1, true},
{"drop_above", 100000, 74999, 1, true},
{"noisy_baseline", 100, 130, 30, false},
{"statistical_below", 100, 149, 20, false},
{"statistical_at", 100, 150, 20, true},
{"steady", 100, 100, 0, false},
{"no_baseline", 0, 100, 0, false},
}
for _, tt := range tests {
for i := range 6 {
count := tt.current
if i < 3 {
count = tt.baseline + (i-1)*tt.jitter
}
// Silence has no current rollup rows, rather than synthetic zeros.
if count == 0 {
continue
}
if _, err := db.Exec(`INSERT INTO service_rollup VALUES (?, 'default', ?, ?, 10)`, tt.name, start.Add(time.Duration(i-3)*5*time.Minute), count); err != nil {
t.Fatal(err)
}
}
}
d := NewDetector(&query.Duck{DB: db}, DefaultDetectorConfig())
anomalies, err := d.detectVolumeAnomalies(t.Context(), start, end)
if err != nil {
t.Fatal(err)
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
assertEffectSizeFinding(t, d, anomalies, tt.name, AnomalyVolumeChange, tt.want, start, end)
})
}
}

func TestAnomalyScatterFloorsRetainFlatAndSingleBucketBaselines(t *testing.T) {
for _, kind := range []string{"latency", "volume"} {
for _, scatter := range []int{0, 1, 2} {
for _, buckets := range []int{1, 3} {
t.Run(fmt.Sprintf("%s/scatter_%d/buckets_%d", kind, scatter, buckets), func(t *testing.T) {
db := effectSizeDB(t)
start := time.Date(2026, 10, 1, 12, 15, 0, 0, time.UTC)
end := start.Add(15 * time.Minute)
for i := range buckets {
value := 100 + (i-buckets/2)*scatter
spans, p95 := 100, 100
if kind == "latency" {
p95 = value
} else {
spans = value
}
if _, err := db.Exec(`INSERT INTO service_rollup VALUES ('api', 'default', ?, ?, ?)`, start.Add(time.Duration(i-3)*5*time.Minute), spans, p95); err != nil {
t.Fatal(err)
}
}
if _, err := db.Exec(`INSERT INTO service_rollup VALUES ('api', 'default', ?, 150, 150)`, start); err != nil {
t.Fatal(err)
}
statement := latencyAnomalySQL(start.UnixNano(), end.UnixNano(), "")
if kind == "volume" {
statement = volumeAnomalySQL(start.UnixNano(), end.UnixNano(), "")
}
var service string
var current, baseline, z float64
if err := db.QueryRow(statement).Scan(&service, &current, &baseline, &z); err != nil {
t.Fatal(err)
}
denominator := 1.0
if buckets == 3 {
denominator = math.Max(float64(scatter), 1)
}
if want := 50 / denominator; math.Abs(z-want) > 1e-9 {
t.Fatalf("z=%g, want %g", z, want)
}
})
}
}
}
}

func effectSizeDB(t *testing.T) *sql.DB {
t.Helper()
db, err := sql.Open("duckdb", "")
if err != nil {
t.Fatal(err)
}
t.Cleanup(func() { _ = db.Close() })
if _, err := db.Exec(`CREATE TABLE service_rollup (service TEXT, namespace TEXT, bucket TIMESTAMP_NS, spans BIGINT, p95_ms DOUBLE)`); err != nil {
t.Fatal(err)
}
return db
}

func assertEffectSizeFinding(t *testing.T, d *Detector, anomalies []Anomaly, service string, kind AnomalyType, want bool, start, end time.Time) {
t.Helper()
var found []Anomaly
for _, a := range anomalies {
if a.ServiceName == service {
found = append(found, a)
}
}
if !want {
if len(found) != 0 {
t.Fatalf("noise emitted findings: %+v", found)
}
if score := d.calculateHealthScore(found, nil); score != 100 {
t.Fatalf("noise lowered health to %g", score)
}
if len(detectorAnnotations(IntelligenceSnapshot{Anomalies: found}, "default")) != 0 || len(d.generateInsights(found, nil)) != 0 {
t.Fatal("noise emitted annotations or insights")
}
return
}
if len(found) != 1 || found[0].Type != kind || !found[0].WindowStart.Equal(start) || !found[0].WindowEnd.Equal(end) {
t.Fatalf("expected one qualified finding in the evaluated window: %+v", found)
}
if score := d.calculateHealthScore(found, nil); score >= 100 {
t.Fatalf("qualified finding did not lower health: %g", score)
}
annotations := detectorAnnotations(IntelligenceSnapshot{Anomalies: found}, "default")
severity := "warn"
if math.Abs(found[0].ZScore) >= 4 {
severity = "bad"
}
if len(annotations) != 1 || annotations[0].Severity != severity {
t.Fatalf("qualified finding severity: %+v", annotations)
}
}
Loading