Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
7 changes: 5 additions & 2 deletions internal/alert/engine.go
Original file line number Diff line number Diff line change
Expand Up @@ -421,15 +421,18 @@ func (e *Engine) buildEnvs(ctx context.Context) map[string]AlertEnv {
WITH current AS (
SELECT service,
avg(error_rate) as error_rate,
avg(p50_ms) as p50, avg(p95_ms) as p95,
COALESCE(avg(p50_ms) FILTER (WHERE served_spans > 0), avg(p50_ms)) as p50,
COALESCE(avg(p95_ms) FILTER (WHERE served_spans > 0), avg(p95_ms)) as p95,
sum(spans) as throughput, sum(log_count) as log_count
FROM service_rollup
WHERE bucket >= (SELECT max(bucket) FROM service_rollup) - INTERVAL '5 minutes'
GROUP BY service
),
previous AS (
SELECT service,
avg(error_rate) as error_rate, avg(p95_ms) as p95, sum(spans) as throughput
avg(error_rate) as error_rate,
COALESCE(avg(p95_ms) FILTER (WHERE served_spans > 0), avg(p95_ms)) as p95,
sum(spans) as throughput
FROM service_rollup
WHERE bucket >= (SELECT max(bucket) FROM service_rollup) - INTERVAL '10 minutes'
AND bucket < (SELECT max(bucket) FROM service_rollup) - INTERVAL '5 minutes'
Expand Down
6 changes: 3 additions & 3 deletions internal/alert/engine_env_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -27,15 +27,15 @@ func TestBuildEnvs_IntegerSumsSurviveHugeint(t *testing.T) {
if _, err := db.Exec(`
CREATE TABLE service_rollup (
namespace VARCHAR, bucket TIMESTAMP, service VARCHAR,
spans BIGINT, error_rate DOUBLE, p50_ms DOUBLE, p95_ms DOUBLE,
spans BIGINT, served_spans BIGINT, error_rate DOUBLE, p50_ms DOUBLE, p95_ms DOUBLE,
log_count BIGINT, metric_count BIGINT
)`); err != nil {
t.Fatalf("create service_rollup: %v", err)
}
if _, err := db.Exec(`
INSERT INTO service_rollup VALUES
('', now(), 'svc-a', 100, 0.0, 10.0, 20.0, 5, 0),
('', now() - INTERVAL 1 MINUTE, 'svc-a', 100, 0.0, 10.0, 20.0, 5, 0)`); err != nil {
('', now(), 'svc-a', 100, 100, 0.0, 10.0, 20.0, 5, 0),
('', now() - INTERVAL 1 MINUTE, 'svc-a', 100, 100, 0.0, 10.0, 20.0, 5, 0)`); err != nil {
t.Fatalf("seed service_rollup: %v", err)
}

Expand Down
47 changes: 39 additions & 8 deletions internal/ingest/server.go
Original file line number Diff line number Diff line change
Expand Up @@ -703,13 +703,18 @@ func anyValueString(v *common.AnyValue) string {
}
}

// normalizeSeverity returns uppercase severity text. If text is empty,
// derives it from the OTel severity number (1-24).
// normalizeSeverity maps a record onto the six OTel severity names, so that a
// search for ERROR finds every error however its emitter spelled it.
//
// Passing SeverityText through unchanged is what produced a store holding
// INFO, INFORMATION and "" for the same level: .NET writes "Information",
// Envoy's access logs carry no severity at all, and each spelling then answered
// a different query. severity_number is preferred because the specification
// makes it the authoritative field and the text the display form, and a record
// that carries neither is stored as UNSPECIFIED rather than as an empty string
// — the log-bucket query already labels that case UNSPECIFIED, and a level you
// can see in the chart should be one you can filter on.
func normalizeSeverity(text string, number int32) string {
s := strings.ToUpper(text)
if s != "" {
return s
}
switch {
case number >= 21:
return "FATAL"
Expand All @@ -723,9 +728,35 @@ func normalizeSeverity(text string, number int32) string {
return "DEBUG"
case number >= 1:
return "TRACE"
default:
return ""
}
return severityFromText(text)
}

// severityFromText canonicalizes the spellings emitters use when they send no
// severity_number: OTel's own numbered short names (INFO2, WARN3), the syslog
// names, and the long forms favoured by .NET and Python.
func severityFromText(text string) string {
name := strings.ToUpper(strings.TrimSpace(text))
name = strings.TrimRight(name, "0123456789")
switch name {
case "FATAL", "CRIT", "CRITICAL", "EMERG", "EMERGENCY", "ALERT", "PANIC":
return "FATAL"
case "ERROR", "ERR", "SEVERE", "EXCEPTION":
return "ERROR"
case "WARN", "WARNING":
return "WARN"
case "INFO", "INFORMATION", "INFORMATIONAL", "NOTICE":
return "INFO"
case "DEBUG", "DBG", "FINE":
return "DEBUG"
case "TRACE", "VERBOSE", "FINEST":
return "TRACE"
case "":
return "UNSPECIFIED"
}
// An unrecognised level is still the emitter's own word for it, and
// discarding it would lose the only grading the record carries.
return name
}

func getServiceName(r *resourcepb.Resource) string {
Expand Down
25 changes: 25 additions & 0 deletions internal/ingest/server_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -551,6 +551,31 @@ func TestTraceExportContextCancellation(t *testing.T) {
}
}

func TestNormalizeSeverityCanonicalizesEmitterSpellings(t *testing.T) {
tests := []struct {
name string
text string
number int32
want string
}{
{"otel short name", "INFO", 9, "INFO"},
{"number wins over display text", "Information", 9, "INFO"},
{"dotnet long form without a number", "Information", 0, "INFO"},
{"python style warning", "WARNING", 0, "WARN"},
{"syslog critical", "crit", 0, "FATAL"},
{"numbered short name", "ERROR2", 0, "ERROR"},
{"envoy access log carries neither", "", 0, "UNSPECIFIED"},
{"unrecognised level is kept", "audit", 0, "AUDIT"},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
if got := normalizeSeverity(tt.text, tt.number); got != tt.want {
t.Errorf("normalizeSeverity(%q, %d) = %q, want %q", tt.text, tt.number, got, tt.want)
}
})
}
}

func TestTraceExportAcceptsSpanWithoutStatus(t *testing.T) {
// status is optional in OTLP and most SDKs omit it for an unset status.
// Reading it through the struct field panicked the handler, so a single
Expand Down
4 changes: 2 additions & 2 deletions internal/mcp/apps/overview.html

Large diffs are not rendered by default.

4 changes: 2 additions & 2 deletions internal/mcp/apps/performance.html

Large diffs are not rendered by default.

18 changes: 18 additions & 0 deletions internal/observability/contracts.go
Original file line number Diff line number Diff line change
Expand Up @@ -47,6 +47,11 @@ const (
HealthHealthy Health = "healthy"
HealthDegraded Health = "degraded"
HealthUnhealthy Health = "unhealthy"
// HealthUnknown is what a window with no telemetry reports. Silence is not
// health: a namespace filter that matches nothing, an agent that stopped
// exporting and an interval before onboarding all land here, and calling
// any of them "healthy" states the opposite of what is known.
HealthUnknown Health = "unknown"
)

type ServiceHealth struct {
Expand Down Expand Up @@ -136,8 +141,21 @@ type ComparisonMetric struct {
Significant bool `json:"significant"`
}

// PerformanceTotals covers the whole requested window, so a headline figure
// agrees with the window the view is labelled with. Points are per-bucket and
// the last one is still filling, so reading a headline off it reports a
// fraction of the traffic and whatever latency the final seconds happened to
// see.
type PerformanceTotals struct {
Spans int64 `json:"spans"`
ErrorRate float64 `json:"error_rate"`
P50MS float64 `json:"p50_ms"`
P95MS float64 `json:"p95_ms"`
}

type Performance struct {
Service string `json:"service,omitempty"`
Totals PerformanceTotals `json:"totals"`
Points []PerformancePoint `json:"points"`
Endpoints []Endpoint `json:"endpoints"`
Heatmap []HeatmapPoint `json:"heatmap"`
Expand Down
91 changes: 91 additions & 0 deletions internal/observability/latency_read_test.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,91 @@
package observability

import (
"context"
"database/sql"
"math"
"testing"
"time"

_ "github.com/duckdb/duckdb-go/v2"
)

// A bucket holding nothing but calls the service was waiting on must not decide
// the window. Excluding those spans inside the bucket is not enough on its own:
// a minute in which a ten-minute subscription is the only span still has to
// report something, and that something would otherwise become the service's
// worst bucket and its health.
func TestOverviewPrefersBucketsWhereTheServiceServedSomething(t *testing.T) {
db, err := sql.Open("duckdb", "")
if err != nil {
t.Fatalf("open duckdb: %v", err)
}
t.Cleanup(func() { _ = db.Close() })
if _, err := db.Exec(`
CREATE TABLE service_rollup (
namespace VARCHAR, bucket TIMESTAMP, service VARCHAR,
spans BIGINT, served_spans BIGINT, error_rate DOUBLE, p50_ms DOUBLE, p95_ms DOUBLE,
log_count BIGINT, metric_count BIGINT
)`); err != nil {
t.Fatalf("create service_rollup: %v", err)
}

now := time.Now().UTC().Truncate(time.Minute)
if _, err := db.Exec(`INSERT INTO service_rollup VALUES
('', ?, 'reviews', 1, 0, 0, 600000, 600000, 0, 0),
('', ?, 'reviews', 20, 20, 0, 30, 40, 0, 0),
('', ?, 'generator', 5, 0, 0, 250, 250, 0, 0)`,
now.Add(-3*time.Minute), now.Add(-time.Minute), now.Add(-time.Minute)); err != nil {
t.Fatalf("seed service_rollup: %v", err)
}

svc := New(SQLDB(db), newTestRepository(t).Parquet, 30)
result, err := svc.Overview(context.Background(), Scope{Start: now.Add(-time.Hour), End: now.Add(time.Minute)}, 50)
if err != nil {
t.Fatalf("Overview: %v", err)
}
got := map[string]ServiceHealth{}
for _, service := range result.Data.Services {
got[service.Service] = service
}
if p95 := got["reviews"].P95MS; p95 > 100 {
t.Fatalf("reviews p95 = %.0fms, want the minute it served (40ms), not the subscription it was holding", p95)
}
if got["reviews"].Health != HealthHealthy {
t.Fatalf("reviews health = %q, want healthy", got["reviews"].Health)
}
// A service that served nothing all window keeps the only number it has.
if p95 := got["generator"].P95MS; p95 != 250 {
t.Fatalf("generator p95 = %.0fms, want 250ms rather than a flat zero", p95)
}
}

// The card's headline figures are folded from the two comparison halves. A half
// spent holding a subscription open has no latency of its own, and letting it
// decide the number would put the card back in disagreement with the overview
// beside it.
func TestTotalsTakeLatencyFromHalvesThatServedSomething(t *testing.T) {
served := performanceAggregate{Spans: 100, ServedSpans: 100, ErrorRate: 0.02, P50MS: 30, P95MS: 40}
waiting := performanceAggregate{Spans: 3, ServedSpans: 0, ErrorRate: 0, P50MS: 600000, P95MS: 600000}

totals := totalsOf(served, waiting)
if totals.P95MS != 40 {
t.Fatalf("p95 = %.0f, want the served half's 40ms rather than the subscription's 600000ms", totals.P95MS)
}
if totals.P50MS != 30 {
t.Fatalf("p50 = %.0f, want 30ms", totals.P50MS)
}
// Counts and error rate still describe everything the service did.
if totals.Spans != 103 {
t.Fatalf("spans = %d, want every span in the window", totals.Spans)
}
if math.Abs(totals.ErrorRate-(0.02*100/103)) > 1e-9 {
t.Fatalf("error rate = %v, want it weighted across all spans", totals.ErrorRate)
}

// With nothing served all window, the only numbers available are kept.
onlyWaiting := totalsOf(waiting, waiting)
if onlyWaiting.P95MS != 600000 {
t.Fatalf("p95 = %.0f, want the subscription figure when there is nothing else", onlyWaiting.P95MS)
}
}
25 changes: 25 additions & 0 deletions internal/observability/latency_sql.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,25 @@
package observability

// A service's latency is read from the buckets in which it actually served
// something. service_rollup stores percentiles that already exclude the calls a
// service was waiting on, but a bucket holding nothing except such calls has no
// served latency to report and falls back to them — a subscription held open
// for ten minutes would otherwise decide the service's whole window.
//
// Preferring served buckets keeps that bucket out of the answer while leaving a
// service that served nothing all window — a load generator, a cron worker —
// measured on what it does have instead of reporting a flat zero.
const (
// windowP95SQL aggregates p95 across buckets. MAX, not an average: the
// worst bucket is the one worth knowing about.
windowP95SQL = `COALESCE(MAX(p95_ms) FILTER (WHERE served_spans > 0), MAX(p95_ms), 0)`

// windowP50SQL weights each bucket by the spans the figure describes. A
// bucket's p50 is computed over served spans, so weighting it by the
// bucket's total would let one served request sitting beside a thousand
// outbound calls outweigh a bucket that served hundreds.
windowP50SQL = `COALESCE(
SUM(p50_ms * served_spans) FILTER (WHERE served_spans > 0) / NULLIF(SUM(served_spans) FILTER (WHERE served_spans > 0), 0),
SUM(p50_ms * spans) / NULLIF(SUM(spans), 0),
0)`
)
5 changes: 3 additions & 2 deletions internal/observability/namespace_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -22,6 +22,7 @@ CREATE TABLE service_rollup (
namespace VARCHAR,
service VARCHAR,
spans BIGINT,
served_spans BIGINT,
error_rate DOUBLE,
p50_ms DOUBLE,
p95_ms DOUBLE,
Expand All @@ -33,8 +34,8 @@ CREATE TABLE service_rollup (

stamp := time.Date(2026, 8, 23, 20, 0, 0, 0, time.UTC)
if _, err := db.Exec(`INSERT INTO service_rollup VALUES
(?, 'prod', 'checkout', 2, 0, 10, 20, 1, 1),
(?, 'staging', 'catalog', 3, 0, 10, 20, 1, 1)`, stamp, stamp); err != nil {
(?, 'prod', 'checkout', 2, 2, 0, 10, 20, 1, 1),
(?, 'staging', 'catalog', 3, 3, 0, 10, 20, 1, 1)`, stamp, stamp); err != nil {
t.Fatalf("insert service rollups: %v", err)
}

Expand Down
15 changes: 11 additions & 4 deletions internal/observability/overview.go
Original file line number Diff line number Diff line change
Expand Up @@ -5,13 +5,13 @@ import (
"fmt"
)

const overviewQuery = `
var overviewQuery = `
SELECT
service,
CAST(SUM(spans) AS BIGINT) AS spans,
COALESCE(SUM(error_rate * spans) / NULLIF(SUM(spans), 0), 0) AS error_rate,
COALESCE(SUM(p50_ms * spans) / NULLIF(SUM(spans), 0), 0) AS p50_ms,
COALESCE(MAX(p95_ms), 0) AS p95_ms,
` + windowP50SQL + ` AS p50_ms,
` + windowP95SQL + ` AS p95_ms,
CAST(SUM(log_count) AS BIGINT) AS log_count,
CAST(SUM(metric_count) AS BIGINT) AS metric_count
FROM service_rollup
Expand Down Expand Up @@ -54,9 +54,16 @@ func (s *Service) Overview(ctx context.Context, scope Scope, limit int) (Result[
}
data.Health = overallHealth(data.Counts)

// An empty window is reported as such rather than as a clean bill of
// health, so a model reading the summary cannot conclude the system is
// fine from the absence of evidence that it is.
summary := fmt.Sprintf("%d services: %d unhealthy, %d degraded, %d healthy", data.ServiceCount, data.Counts.Unhealthy, data.Counts.Degraded, data.Counts.Healthy)
if data.ServiceCount == 0 {
summary = "no services reported telemetry in this window"
}
return Result[Overview]{
Schema: OverviewSchema,
Summary: fmt.Sprintf("%d services: %d unhealthy, %d degraded, %d healthy", data.ServiceCount, data.Counts.Unhealthy, data.Counts.Degraded, data.Counts.Healthy),
Summary: summary,
Data: data,
Provenance: s.provenance(scope),
}, nil
Expand Down
Loading
Loading