Skip to content

Commit 95acb6d

Browse files
committed
fix(intelligence): require a meaningful change before reporting latency and volume anomalies
Latency and volume anomalies fired on the z-score alone. The baseline is the scatter of a few five-minute buckets, so a steady service has a tiny standard deviation and a 0.1 ms or 5% shift scored as a severe anomaly. One hour of the OpenTelemetry Demo produced 36 findings across 18 services, and the dashboard's anomaly overlay covered every panel for the whole hour. Both checks now also need practical significance, in the style of the existing error-rate floor: - latency: p95 must rise by at least 20% and at least 5 ms, with a 1 ms floor on the baseline scatter; - volume: the count must change by at least 25% in either direction, with a one-span floor on the scatter. A service present in the baseline but silent now is evaluated, so traffic stopping is reported. Only qualified findings affect severity, insights and the health score. Replaying the detector over one hour of a Demo snapshot reduced findings from 87 to 2; the remaining one is a frontend-web p95 rise from 202 to 243 ms. Claude-Session: https://claude.ai/code/session_01XA52nWyQ3R5f5kK8kjpAQW
1 parent 0902483 commit 95acb6d

2 files changed

Lines changed: 245 additions & 9 deletions

File tree

‎internal/intelligence/detector.go‎

Lines changed: 34 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -297,6 +297,26 @@ func (d *Detector) detectAnomalies(ctx context.Context, start, end time.Time) ([
297297
// percentage point. See errorRateAnomalySQL.
298298
const minErrorRateStddev = 0.01
299299

300+
// minLatencyIncreaseRatio requires a 20% slowdown, keeping small proportional
301+
// shifts in slow services quiet while allowing substantial regressions through.
302+
const minLatencyIncreaseRatio = 0.20
303+
304+
// minLatencyIncreaseMS requires at least 5ms of extra p95 latency: even a large
305+
// ratio on a sub-millisecond service is noise, but 20ms to 60ms still qualifies.
306+
const minLatencyIncreaseMS = 5.0
307+
308+
// minLatencyStddev assumes at least 1ms of timing scatter, so a flat baseline
309+
// can still detect regressions without amplifying fractional-millisecond jitter.
310+
const minLatencyStddev = 1.0
311+
312+
// minVolumeChangeRatio requires a 25% traffic change in either direction,
313+
// suppressing small count fluctuations while retaining 30% drops and doubling.
314+
const minVolumeChangeRatio = 0.25
315+
316+
// minVolumeStddev assumes at least one span of counting uncertainty, allowing
317+
// flat baselines to detect traffic stopping or surging without dividing by zero.
318+
const minVolumeStddev = 1.0
319+
300320
// errorRateAnomalySQL compares the error rate in [startNano, endNano) against
301321
// the window of equal length immediately before it.
302322
//
@@ -428,7 +448,9 @@ func (d *Detector) detectLatencyAnomalies(ctx context.Context, start, end time.T
428448
baselineP95 := row.baseline
429449
zScore := row.zScore
430450

431-
if zScore >= d.config.LatencyThreshold {
451+
increase := currentP95 - baselineP95
452+
if zScore >= d.config.LatencyThreshold &&
453+
increase >= minLatencyIncreaseMS && increase >= baselineP95*minLatencyIncreaseRatio {
432454
anomalies = append(anomalies, Anomaly{
433455
Type: AnomalyLatencyDegradation,
434456
ServiceName: serviceName,
@@ -458,6 +480,8 @@ func (d *Detector) detectLatencyAnomalies(ctx context.Context, start, end time.T
458480
// the traffic did. Every service was permanently a critical volume anomaly:
459481
// steady traffic scored z=75 in TestVolumeAnomalyComparesEqualWindows, and the
460482
// live demo sat at health score 0 with 17 meaningless critical insights.
483+
// Retain baseline-only services with a zero current count so traffic stopping
484+
// is evaluated even though the rollup has no positive-span rows for that window.
461485
func volumeAnomalySQL(startNano, endNano int64, scope string) string {
462486
return fmt.Sprintf(`
463487
WITH current_period AS (
@@ -492,16 +516,16 @@ func volumeAnomalySQL(startNano, endNano int64, scope string) string {
492516
GROUP BY service_name
493517
)
494518
SELECT
495-
c.service_name,
496-
c.span_count::DOUBLE AS current_count,
519+
COALESCE(c.service_name, b.service_name) AS service_name,
520+
COALESCE(c.span_count, 0.0)::DOUBLE AS current_count,
497521
COALESCE(b.avg_count, 0.0) AS baseline_count,
498522
CASE
499-
WHEN b.count_stddev > 0 THEN (c.span_count - b.avg_count) / b.count_stddev
523+
WHEN b.avg_count > 0 THEN (COALESCE(c.span_count, 0.0) - b.avg_count) / GREATEST(COALESCE(b.count_stddev, 0.0), %f)
500524
ELSE 0.0
501525
END AS z_score
502526
FROM current_period c
503-
LEFT JOIN baseline_period b ON c.service_name = b.service_name
504-
`, startNano, endNano, scope, startNano-endNano+startNano, startNano, scope)
527+
FULL OUTER JOIN baseline_period b ON c.service_name = b.service_name
528+
`, startNano, endNano, scope, startNano-endNano+startNano, startNano, scope, minVolumeStddev)
505529
}
506530

507531
// detectVolumeAnomalies detects unusual traffic volume changes
@@ -525,7 +549,8 @@ func (d *Detector) detectVolumeAnomalies(ctx context.Context, start, end time.Ti
525549
baselineCount := row.baseline
526550
zScore := row.zScore
527551

528-
if math.Abs(zScore) >= d.config.VolumeThreshold {
552+
if math.Abs(zScore) >= d.config.VolumeThreshold && baselineCount > 0 &&
553+
math.Abs(currentCount-baselineCount) >= baselineCount*minVolumeChangeRatio {
529554
anomalies = append(anomalies, Anomaly{
530555
Type: AnomalyVolumeChange,
531556
ServiceName: serviceName,
@@ -581,12 +606,12 @@ func latencyAnomalySQL(startNano, endNano int64, scope string) string {
581606
c.p95_latency AS current_p95,
582607
COALESCE(b.avg_p95, 0.0) AS baseline_p95,
583608
CASE
584-
WHEN b.p95_stddev > 0 THEN (c.p95_latency - b.avg_p95) / b.p95_stddev
609+
WHEN b.avg_p95 IS NOT NULL THEN (c.p95_latency - b.avg_p95) / GREATEST(COALESCE(b.p95_stddev, 0.0), %f)
585610
ELSE 0.0
586611
END AS z_score
587612
FROM current_period c
588613
LEFT JOIN baseline_period b ON c.service_name = b.service_name
589-
`, startNano, endNano, scope, startNano-endNano+startNano, startNano, scope)
614+
`, startNano, endNano, scope, startNano-endNano+startNano, startNano, scope, minLatencyStddev)
590615

591616
}
592617

Lines changed: 211 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,211 @@
1+
package intelligence
2+
3+
import (
4+
"database/sql"
5+
"fmt"
6+
"math"
7+
"testing"
8+
"time"
9+
10+
"github.com/labstack/fanout/internal/query"
11+
)
12+
13+
func TestLatencyAnomaliesRequireStatisticalAndPracticalSignificance(t *testing.T) {
14+
db := effectSizeDB(t)
15+
start := time.Date(2026, 10, 1, 12, 15, 0, 0, time.UTC)
16+
end := start.Add(15 * time.Minute)
17+
tests := []struct {
18+
name string
19+
baseline, current, stddev float64
20+
want bool
21+
}{
22+
{"ad", 1.3, 1.4, .01, false},
23+
{"flagd", .3, .3001, .00001, false},
24+
{"email", 5.8, 5.8001, .00001, false},
25+
{"sub_millisecond_doubling", .3, .6, .001, false},
26+
{"real_regression", 20, 60, 1, true},
27+
{"flat_baseline_regression", 20, 60, 0, true},
28+
{"absolute_below", 10, 15 - 1e-6, .01, false},
29+
{"absolute_at", 10, 15, .01, true},
30+
{"absolute_above", 10, 15 + 1e-6, .01, true},
31+
{"relative_below", 100, 120 - 1e-6, .01, false},
32+
{"relative_at", 100, 120, .01, true},
33+
{"relative_above", 100, 120 + 1e-6, .01, true},
34+
{"noisy_baseline", 100, 125, 30, false},
35+
{"statistical_below", 20, 30 - 1e-6, 5, false},
36+
{"statistical_at", 20, 30, 5, true},
37+
{"improvement", 60, 20, 1, false},
38+
}
39+
for _, tt := range tests {
40+
for i := range 6 {
41+
p95 := tt.current
42+
if i < 3 {
43+
p95 = tt.baseline + float64(i-1)*tt.stddev
44+
}
45+
if _, err := db.Exec(`INSERT INTO service_rollup VALUES (?, 'default', ?, 100, ?)`, tt.name, start.Add(time.Duration(i-3)*5*time.Minute), p95); err != nil {
46+
t.Fatal(err)
47+
}
48+
}
49+
}
50+
d := NewDetector(&query.Duck{DB: db}, DefaultDetectorConfig())
51+
anomalies, err := d.detectLatencyAnomalies(t.Context(), start, end)
52+
if err != nil {
53+
t.Fatal(err)
54+
}
55+
for _, tt := range tests {
56+
t.Run(tt.name, func(t *testing.T) {
57+
assertEffectSizeFinding(t, d, anomalies, tt.name, AnomalyLatencyDegradation, tt.want, start, end)
58+
})
59+
}
60+
}
61+
62+
func TestVolumeAnomaliesRequireStatisticalAndPracticalSignificance(t *testing.T) {
63+
db := effectSizeDB(t)
64+
start := time.Date(2026, 10, 1, 12, 15, 0, 0, time.UTC)
65+
end := start.Add(15 * time.Minute)
66+
tests := []struct {
67+
name string
68+
baseline, current, jitter int
69+
want bool
70+
}{
71+
{"payment", 250, 263, 1, false},
72+
{"ad", 730, 779, 1, false},
73+
{"currency", 1274, 935, 12, true},
74+
{"frontend_proxy", 32031, 21573, 481, true},
75+
{"silent", 100, 0, 1, true},
76+
{"flat_baseline_silent", 100, 0, 0, true},
77+
{"doubling", 100, 200, 1, true},
78+
{"flat_baseline_doubling", 100, 200, 0, true},
79+
{"thirty_percent_drop", 100, 70, 1, true},
80+
{"surge_below", 100000, 124999, 1, false},
81+
{"surge_at", 100000, 125000, 1, true},
82+
{"surge_above", 100000, 125001, 1, true},
83+
{"drop_below", 100000, 75001, 1, false},
84+
{"drop_at", 100000, 75000, 1, true},
85+
{"drop_above", 100000, 74999, 1, true},
86+
{"noisy_baseline", 100, 130, 30, false},
87+
{"statistical_below", 100, 149, 20, false},
88+
{"statistical_at", 100, 150, 20, true},
89+
{"steady", 100, 100, 0, false},
90+
{"no_baseline", 0, 100, 0, false},
91+
}
92+
for _, tt := range tests {
93+
for i := range 6 {
94+
count := tt.current
95+
if i < 3 {
96+
count = tt.baseline + (i-1)*tt.jitter
97+
}
98+
// Silence has no current rollup rows, rather than synthetic zeros.
99+
if count == 0 {
100+
continue
101+
}
102+
if _, err := db.Exec(`INSERT INTO service_rollup VALUES (?, 'default', ?, ?, 10)`, tt.name, start.Add(time.Duration(i-3)*5*time.Minute), count); err != nil {
103+
t.Fatal(err)
104+
}
105+
}
106+
}
107+
d := NewDetector(&query.Duck{DB: db}, DefaultDetectorConfig())
108+
anomalies, err := d.detectVolumeAnomalies(t.Context(), start, end)
109+
if err != nil {
110+
t.Fatal(err)
111+
}
112+
for _, tt := range tests {
113+
t.Run(tt.name, func(t *testing.T) {
114+
assertEffectSizeFinding(t, d, anomalies, tt.name, AnomalyVolumeChange, tt.want, start, end)
115+
})
116+
}
117+
}
118+
119+
func TestAnomalyScatterFloorsRetainFlatAndSingleBucketBaselines(t *testing.T) {
120+
for _, kind := range []string{"latency", "volume"} {
121+
for _, scatter := range []int{0, 1, 2} {
122+
for _, buckets := range []int{1, 3} {
123+
t.Run(fmt.Sprintf("%s/scatter_%d/buckets_%d", kind, scatter, buckets), func(t *testing.T) {
124+
db := effectSizeDB(t)
125+
start := time.Date(2026, 10, 1, 12, 15, 0, 0, time.UTC)
126+
end := start.Add(15 * time.Minute)
127+
for i := range buckets {
128+
value := 100 + (i-buckets/2)*scatter
129+
spans, p95 := 100, 100
130+
if kind == "latency" {
131+
p95 = value
132+
} else {
133+
spans = value
134+
}
135+
if _, err := db.Exec(`INSERT INTO service_rollup VALUES ('api', 'default', ?, ?, ?)`, start.Add(time.Duration(i-3)*5*time.Minute), spans, p95); err != nil {
136+
t.Fatal(err)
137+
}
138+
}
139+
if _, err := db.Exec(`INSERT INTO service_rollup VALUES ('api', 'default', ?, 150, 150)`, start); err != nil {
140+
t.Fatal(err)
141+
}
142+
statement := latencyAnomalySQL(start.UnixNano(), end.UnixNano(), "")
143+
if kind == "volume" {
144+
statement = volumeAnomalySQL(start.UnixNano(), end.UnixNano(), "")
145+
}
146+
var service string
147+
var current, baseline, z float64
148+
if err := db.QueryRow(statement).Scan(&service, &current, &baseline, &z); err != nil {
149+
t.Fatal(err)
150+
}
151+
denominator := 1.0
152+
if buckets == 3 {
153+
denominator = math.Max(float64(scatter), 1)
154+
}
155+
if want := 50 / denominator; math.Abs(z-want) > 1e-9 {
156+
t.Fatalf("z=%g, want %g", z, want)
157+
}
158+
})
159+
}
160+
}
161+
}
162+
}
163+
164+
func effectSizeDB(t *testing.T) *sql.DB {
165+
t.Helper()
166+
db, err := sql.Open("duckdb", "")
167+
if err != nil {
168+
t.Fatal(err)
169+
}
170+
t.Cleanup(func() { _ = db.Close() })
171+
if _, err := db.Exec(`CREATE TABLE service_rollup (service TEXT, namespace TEXT, bucket TIMESTAMP_NS, spans BIGINT, p95_ms DOUBLE)`); err != nil {
172+
t.Fatal(err)
173+
}
174+
return db
175+
}
176+
177+
func assertEffectSizeFinding(t *testing.T, d *Detector, anomalies []Anomaly, service string, kind AnomalyType, want bool, start, end time.Time) {
178+
t.Helper()
179+
var found []Anomaly
180+
for _, a := range anomalies {
181+
if a.ServiceName == service {
182+
found = append(found, a)
183+
}
184+
}
185+
if !want {
186+
if len(found) != 0 {
187+
t.Fatalf("noise emitted findings: %+v", found)
188+
}
189+
if score := d.calculateHealthScore(found, nil); score != 100 {
190+
t.Fatalf("noise lowered health to %g", score)
191+
}
192+
if len(detectorAnnotations(IntelligenceSnapshot{Anomalies: found}, "default")) != 0 || len(d.generateInsights(found, nil)) != 0 {
193+
t.Fatal("noise emitted annotations or insights")
194+
}
195+
return
196+
}
197+
if len(found) != 1 || found[0].Type != kind || !found[0].WindowStart.Equal(start) || !found[0].WindowEnd.Equal(end) {
198+
t.Fatalf("expected one qualified finding in the evaluated window: %+v", found)
199+
}
200+
if score := d.calculateHealthScore(found, nil); score >= 100 {
201+
t.Fatalf("qualified finding did not lower health: %g", score)
202+
}
203+
annotations := detectorAnnotations(IntelligenceSnapshot{Anomalies: found}, "default")
204+
severity := "warn"
205+
if math.Abs(found[0].ZScore) >= 4 {
206+
severity = "bad"
207+
}
208+
if len(annotations) != 1 || annotations[0].Severity != severity {
209+
t.Fatalf("qualified finding severity: %+v", annotations)
210+
}
211+
}

0 commit comments

Comments
 (0)