|
| 1 | +package intelligence |
| 2 | + |
| 3 | +import ( |
| 4 | + "database/sql" |
| 5 | + "fmt" |
| 6 | + "math" |
| 7 | + "testing" |
| 8 | + "time" |
| 9 | + |
| 10 | + "github.com/labstack/fanout/internal/query" |
| 11 | +) |
| 12 | + |
| 13 | +func TestLatencyAnomaliesRequireStatisticalAndPracticalSignificance(t *testing.T) { |
| 14 | + db := effectSizeDB(t) |
| 15 | + start := time.Date(2026, 10, 1, 12, 15, 0, 0, time.UTC) |
| 16 | + end := start.Add(15 * time.Minute) |
| 17 | + tests := []struct { |
| 18 | + name string |
| 19 | + baseline, current, stddev float64 |
| 20 | + want bool |
| 21 | + }{ |
| 22 | + {"ad", 1.3, 1.4, .01, false}, |
| 23 | + {"flagd", .3, .3001, .00001, false}, |
| 24 | + {"email", 5.8, 5.8001, .00001, false}, |
| 25 | + {"sub_millisecond_doubling", .3, .6, .001, false}, |
| 26 | + {"real_regression", 20, 60, 1, true}, |
| 27 | + {"flat_baseline_regression", 20, 60, 0, true}, |
| 28 | + {"absolute_below", 10, 15 - 1e-6, .01, false}, |
| 29 | + {"absolute_at", 10, 15, .01, true}, |
| 30 | + {"absolute_above", 10, 15 + 1e-6, .01, true}, |
| 31 | + {"relative_below", 100, 120 - 1e-6, .01, false}, |
| 32 | + {"relative_at", 100, 120, .01, true}, |
| 33 | + {"relative_above", 100, 120 + 1e-6, .01, true}, |
| 34 | + {"noisy_baseline", 100, 125, 30, false}, |
| 35 | + {"statistical_below", 20, 30 - 1e-6, 5, false}, |
| 36 | + {"statistical_at", 20, 30, 5, true}, |
| 37 | + {"improvement", 60, 20, 1, false}, |
| 38 | + } |
| 39 | + for _, tt := range tests { |
| 40 | + for i := range 6 { |
| 41 | + p95 := tt.current |
| 42 | + if i < 3 { |
| 43 | + p95 = tt.baseline + float64(i-1)*tt.stddev |
| 44 | + } |
| 45 | + if _, err := db.Exec(`INSERT INTO service_rollup VALUES (?, 'default', ?, 100, ?)`, tt.name, start.Add(time.Duration(i-3)*5*time.Minute), p95); err != nil { |
| 46 | + t.Fatal(err) |
| 47 | + } |
| 48 | + } |
| 49 | + } |
| 50 | + d := NewDetector(&query.Duck{DB: db}, DefaultDetectorConfig()) |
| 51 | + anomalies, err := d.detectLatencyAnomalies(t.Context(), start, end) |
| 52 | + if err != nil { |
| 53 | + t.Fatal(err) |
| 54 | + } |
| 55 | + for _, tt := range tests { |
| 56 | + t.Run(tt.name, func(t *testing.T) { |
| 57 | + assertEffectSizeFinding(t, d, anomalies, tt.name, AnomalyLatencyDegradation, tt.want, start, end) |
| 58 | + }) |
| 59 | + } |
| 60 | +} |
| 61 | + |
| 62 | +func TestVolumeAnomaliesRequireStatisticalAndPracticalSignificance(t *testing.T) { |
| 63 | + db := effectSizeDB(t) |
| 64 | + start := time.Date(2026, 10, 1, 12, 15, 0, 0, time.UTC) |
| 65 | + end := start.Add(15 * time.Minute) |
| 66 | + tests := []struct { |
| 67 | + name string |
| 68 | + baseline, current, jitter int |
| 69 | + want bool |
| 70 | + }{ |
| 71 | + {"payment", 250, 263, 1, false}, |
| 72 | + {"ad", 730, 779, 1, false}, |
| 73 | + {"currency", 1274, 935, 12, true}, |
| 74 | + {"frontend_proxy", 32031, 21573, 481, true}, |
| 75 | + {"silent", 100, 0, 1, true}, |
| 76 | + {"flat_baseline_silent", 100, 0, 0, true}, |
| 77 | + {"doubling", 100, 200, 1, true}, |
| 78 | + {"flat_baseline_doubling", 100, 200, 0, true}, |
| 79 | + {"thirty_percent_drop", 100, 70, 1, true}, |
| 80 | + {"surge_below", 100000, 124999, 1, false}, |
| 81 | + {"surge_at", 100000, 125000, 1, true}, |
| 82 | + {"surge_above", 100000, 125001, 1, true}, |
| 83 | + {"drop_below", 100000, 75001, 1, false}, |
| 84 | + {"drop_at", 100000, 75000, 1, true}, |
| 85 | + {"drop_above", 100000, 74999, 1, true}, |
| 86 | + {"noisy_baseline", 100, 130, 30, false}, |
| 87 | + {"statistical_below", 100, 149, 20, false}, |
| 88 | + {"statistical_at", 100, 150, 20, true}, |
| 89 | + {"steady", 100, 100, 0, false}, |
| 90 | + {"no_baseline", 0, 100, 0, false}, |
| 91 | + } |
| 92 | + for _, tt := range tests { |
| 93 | + for i := range 6 { |
| 94 | + count := tt.current |
| 95 | + if i < 3 { |
| 96 | + count = tt.baseline + (i-1)*tt.jitter |
| 97 | + } |
| 98 | + // Silence has no current rollup rows, rather than synthetic zeros. |
| 99 | + if count == 0 { |
| 100 | + continue |
| 101 | + } |
| 102 | + if _, err := db.Exec(`INSERT INTO service_rollup VALUES (?, 'default', ?, ?, 10)`, tt.name, start.Add(time.Duration(i-3)*5*time.Minute), count); err != nil { |
| 103 | + t.Fatal(err) |
| 104 | + } |
| 105 | + } |
| 106 | + } |
| 107 | + d := NewDetector(&query.Duck{DB: db}, DefaultDetectorConfig()) |
| 108 | + anomalies, err := d.detectVolumeAnomalies(t.Context(), start, end) |
| 109 | + if err != nil { |
| 110 | + t.Fatal(err) |
| 111 | + } |
| 112 | + for _, tt := range tests { |
| 113 | + t.Run(tt.name, func(t *testing.T) { |
| 114 | + assertEffectSizeFinding(t, d, anomalies, tt.name, AnomalyVolumeChange, tt.want, start, end) |
| 115 | + }) |
| 116 | + } |
| 117 | +} |
| 118 | + |
| 119 | +func TestAnomalyScatterFloorsRetainFlatAndSingleBucketBaselines(t *testing.T) { |
| 120 | + for _, kind := range []string{"latency", "volume"} { |
| 121 | + for _, scatter := range []int{0, 1, 2} { |
| 122 | + for _, buckets := range []int{1, 3} { |
| 123 | + t.Run(fmt.Sprintf("%s/scatter_%d/buckets_%d", kind, scatter, buckets), func(t *testing.T) { |
| 124 | + db := effectSizeDB(t) |
| 125 | + start := time.Date(2026, 10, 1, 12, 15, 0, 0, time.UTC) |
| 126 | + end := start.Add(15 * time.Minute) |
| 127 | + for i := range buckets { |
| 128 | + value := 100 + (i-buckets/2)*scatter |
| 129 | + spans, p95 := 100, 100 |
| 130 | + if kind == "latency" { |
| 131 | + p95 = value |
| 132 | + } else { |
| 133 | + spans = value |
| 134 | + } |
| 135 | + if _, err := db.Exec(`INSERT INTO service_rollup VALUES ('api', 'default', ?, ?, ?)`, start.Add(time.Duration(i-3)*5*time.Minute), spans, p95); err != nil { |
| 136 | + t.Fatal(err) |
| 137 | + } |
| 138 | + } |
| 139 | + if _, err := db.Exec(`INSERT INTO service_rollup VALUES ('api', 'default', ?, 150, 150)`, start); err != nil { |
| 140 | + t.Fatal(err) |
| 141 | + } |
| 142 | + statement := latencyAnomalySQL(start.UnixNano(), end.UnixNano(), "") |
| 143 | + if kind == "volume" { |
| 144 | + statement = volumeAnomalySQL(start.UnixNano(), end.UnixNano(), "") |
| 145 | + } |
| 146 | + var service string |
| 147 | + var current, baseline, z float64 |
| 148 | + if err := db.QueryRow(statement).Scan(&service, ¤t, &baseline, &z); err != nil { |
| 149 | + t.Fatal(err) |
| 150 | + } |
| 151 | + denominator := 1.0 |
| 152 | + if buckets == 3 { |
| 153 | + denominator = math.Max(float64(scatter), 1) |
| 154 | + } |
| 155 | + if want := 50 / denominator; math.Abs(z-want) > 1e-9 { |
| 156 | + t.Fatalf("z=%g, want %g", z, want) |
| 157 | + } |
| 158 | + }) |
| 159 | + } |
| 160 | + } |
| 161 | + } |
| 162 | +} |
| 163 | + |
| 164 | +func effectSizeDB(t *testing.T) *sql.DB { |
| 165 | + t.Helper() |
| 166 | + db, err := sql.Open("duckdb", "") |
| 167 | + if err != nil { |
| 168 | + t.Fatal(err) |
| 169 | + } |
| 170 | + t.Cleanup(func() { _ = db.Close() }) |
| 171 | + if _, err := db.Exec(`CREATE TABLE service_rollup (service TEXT, namespace TEXT, bucket TIMESTAMP_NS, spans BIGINT, p95_ms DOUBLE)`); err != nil { |
| 172 | + t.Fatal(err) |
| 173 | + } |
| 174 | + return db |
| 175 | +} |
| 176 | + |
| 177 | +func assertEffectSizeFinding(t *testing.T, d *Detector, anomalies []Anomaly, service string, kind AnomalyType, want bool, start, end time.Time) { |
| 178 | + t.Helper() |
| 179 | + var found []Anomaly |
| 180 | + for _, a := range anomalies { |
| 181 | + if a.ServiceName == service { |
| 182 | + found = append(found, a) |
| 183 | + } |
| 184 | + } |
| 185 | + if !want { |
| 186 | + if len(found) != 0 { |
| 187 | + t.Fatalf("noise emitted findings: %+v", found) |
| 188 | + } |
| 189 | + if score := d.calculateHealthScore(found, nil); score != 100 { |
| 190 | + t.Fatalf("noise lowered health to %g", score) |
| 191 | + } |
| 192 | + if len(detectorAnnotations(IntelligenceSnapshot{Anomalies: found}, "default")) != 0 || len(d.generateInsights(found, nil)) != 0 { |
| 193 | + t.Fatal("noise emitted annotations or insights") |
| 194 | + } |
| 195 | + return |
| 196 | + } |
| 197 | + if len(found) != 1 || found[0].Type != kind || !found[0].WindowStart.Equal(start) || !found[0].WindowEnd.Equal(end) { |
| 198 | + t.Fatalf("expected one qualified finding in the evaluated window: %+v", found) |
| 199 | + } |
| 200 | + if score := d.calculateHealthScore(found, nil); score >= 100 { |
| 201 | + t.Fatalf("qualified finding did not lower health: %g", score) |
| 202 | + } |
| 203 | + annotations := detectorAnnotations(IntelligenceSnapshot{Anomalies: found}, "default") |
| 204 | + severity := "warn" |
| 205 | + if math.Abs(found[0].ZScore) >= 4 { |
| 206 | + severity = "bad" |
| 207 | + } |
| 208 | + if len(annotations) != 1 || annotations[0].Severity != severity { |
| 209 | + t.Fatalf("qualified finding severity: %+v", annotations) |
| 210 | + } |
| 211 | +} |
0 commit comments