From 05ebef4a11e40a321b58b8a30f66d5318c0b197c Mon Sep 17 00:00:00 2001 From: Vishal Rana Date: Fri, 28 Aug 2026 15:29:59 -0700 Subject: [PATCH 1/2] chore: remove bundled benchmark tooling --- README.md | 22 +- cmd/bench/adaptive.go | 205 ---- cmd/bench/adaptive_test.go | 229 ---- cmd/bench/config.go | 57 - cmd/bench/config_test.go | 87 -- cmd/bench/main.go | 1006 ----------------- cmd/bench/main_test.go | 124 -- cmd/bench/manifest.go | 155 --- cmd/bench/manifest_test.go | 100 -- cmd/bench/metrics_report.go | 515 --------- cmd/bench/metrics_report_test.go | 219 ---- cmd/bench/verdict.go | 104 -- cmd/bench/verdict_test.go | 192 ---- docs/benchmarking.md | 33 - internal/config/config.go | 3 +- .../performance_benchmark_test.go | 118 -- justfile | 3 +- .../content/docs/explanation/performance.mdx | 24 +- 18 files changed, 27 insertions(+), 3169 deletions(-) delete mode 100644 cmd/bench/adaptive.go delete mode 100644 cmd/bench/adaptive_test.go delete mode 100644 cmd/bench/config.go delete mode 100644 cmd/bench/config_test.go delete mode 100644 cmd/bench/main.go delete mode 100644 cmd/bench/main_test.go delete mode 100644 cmd/bench/manifest.go delete mode 100644 cmd/bench/manifest_test.go delete mode 100644 cmd/bench/metrics_report.go delete mode 100644 cmd/bench/metrics_report_test.go delete mode 100644 cmd/bench/verdict.go delete mode 100644 cmd/bench/verdict_test.go delete mode 100644 docs/benchmarking.md delete mode 100644 internal/observability/performance_benchmark_test.go diff --git a/README.md b/README.md index de570e599..eec6c59a7 100644 --- a/README.md +++ b/README.md @@ -45,13 +45,12 @@ database. ## Performance -The bundled [`cmd/bench`](cmd/bench) driver measures authenticated ingest and -optional dashboard read load against your hardware. Ingest, DuckDB queries, -and native Parquet maintenance have separate coordination paths but still -compete for the same CPU, memory bandwidth, filesystem cache, and disk. Fanout -does not publish a throughput headline until the raw reports and exact driver -revision can ship with it; see [the benchmark publication -standard](docs/benchmarking.md). +The independent [Fanout Bench](https://github.com/labstack/fanout-bench) +project measures authenticated ingest and optional dashboard read load against +your hardware. It uses the official OpenTelemetry generator and publishes raw, +reproducible evidence separately from the production binary. Ingest, DuckDB +queries, and native Parquet maintenance have separate coordination paths but +still compete for the same CPU, memory bandwidth, filesystem cache, and disk. ## How it compares @@ -197,11 +196,9 @@ Fanout serves the UI on , accepts OTLP/gRPC on created becomes the administrator and receives the ingest token once. Point any OpenTelemetry collector or SDK at either OTLP endpoint with the -ingest token, or generate load with the bundled gRPC benchmark driver: - -```sh -./bin/bench -endpoint localhost:4317 -token "$INGEST_TOKEN" -``` +ingest token. Use the separate +[Fanout Bench](https://github.com/labstack/fanout-bench) project for controlled +capacity tests. ## Configuration @@ -293,7 +290,6 @@ the same `just check` unconditionally, and that is what actually enforces it. ```text cmd/fanout/ process composition and the single entry point -cmd/bench/ load generator and benchmark reporter internal/ ingest, storage, query, agent, MCP, auth, alerts ui/host/ React AG-UI browser host (build-time) ui/apps/ portable React MCP Apps (build-time) diff --git a/cmd/bench/adaptive.go b/cmd/bench/adaptive.go deleted file mode 100644 index 3cb91f652..000000000 --- a/cmd/bench/adaptive.go +++ /dev/null @@ -1,205 +0,0 @@ -package main - -import "runtime" - -// numCPU is the driver's usable parallelism. GOMAXPROCS rather than NumCPU so -// a container CPU limit is respected: sizing to the host's cores inside a -// two-core cgroup would oversubscribe the driver and inflate its own latency. -func numCPU() int { return runtime.GOMAXPROCS(0) } - -// Adaptive ramping. The harness discovers what a machine sustains instead of -// being told a rate that was calibrated on different hardware. -// -// The shape is deliberately simple: offer a rate, measure what comes back, -// multiply, repeat until the server stops keeping up or latency turns the -// corner. Everything here is a pure function of the steps already run, so the -// policy can be tested without a server. - -// maxAutoWorkers bounds self-sizing. Past this the driver is mostly queueing -// work inside itself, and reporting that queue delay as the server's latency -// would be measuring the wrong process. -const maxAutoWorkers = 256 - -// rampStep is one completed fixed-rate trial. -type rampStep struct { - TargetRate float64 `json:"target_traces_per_sec"` - AchievedRate float64 `json:"achieved_traces_per_sec"` - ExportP95Ms float64 `json:"export_p95_ms"` - QueryP95Ms float64 `json:"query_p95_ms,omitempty"` -} - -// held reports whether the server actually accepted what this step offered. -func (s rampStep) held(floor float64) bool { - return s.TargetRate > 0 && s.AchievedRate >= s.TargetRate*floor -} - -// rampPolicy is the ramp's stopping rule. -type rampPolicy struct { - // Growth multiplies the target after a step the server kept up with. - Growth float64 - // MaxSteps bounds the ramp so a fast server cannot run indefinitely. - MaxSteps int - // DeliveryFloor is the fraction of the offered rate that must actually - // arrive for the step to count as sustained. - DeliveryFloor float64 - // LatencyKnee is the multiple of the best observed latency past which - // throughput is being bought with unacceptable delay. - LatencyKnee float64 - // RefineTolerance stops the bisection once the bracket is this tight, - // relative to its upper bound. Chasing precision past this measures noise. - RefineTolerance float64 -} - -// bracket returns the highest offered rate the server sustained without -// crossing the latency knee, and the lowest rate that failed either test. A -// zero upper bound means nothing has missed yet. -func bracket(history []rampStep, p rampPolicy) (lastGood, firstBad float64) { - best := bestHeldExportP95(history, p.DeliveryFloor) - for _, s := range history { - if s.held(p.DeliveryFloor) && !kneeExceeded(best, s.ExportP95Ms, p.LatencyKnee) { - if s.TargetRate > lastGood { - lastGood = s.TargetRate - } - continue - } - if firstBad == 0 || s.TargetRate < firstBad { - firstBad = s.TargetRate - } - } - return lastGood, firstBad -} - -func defaultRampPolicy() rampPolicy { - return rampPolicy{ - Growth: 2, - MaxSteps: 8, - // 0.95, not 0.90. A step that scrapes past a looser floor gets reported - // as sustainable and then fails to reproduce in the confirmation pass: - // measured on two dedicated cores, 90.8% delivery was accepted as - // sustainable at 7267/s and the confirmation managed 6503/s. Requiring - // near-complete delivery makes the quoted rate one the machine repeats. - DeliveryFloor: 0.95, - LatencyKnee: 10, - RefineTolerance: 0.15, - } -} - -type rampDecision struct { - NextRate float64 - Stop bool - Reason string -} - -// decideNextStep chooses the next offered rate, or stops the ramp. -func decideNextStep(history []rampStep, p rampPolicy) rampDecision { - if len(history) == 0 { - return rampDecision{NextRate: seedRate(numCPU())} - } - if len(history) >= p.MaxSteps { - return rampDecision{Stop: true, Reason: "reached the step limit"} - } - - last := history[len(history)-1] - lastGood, firstBad := bracket(history, p) - - // Nothing has missed yet: keep doubling to find the upper bound quickly. - if firstBad == 0 { - return rampDecision{NextRate: last.TargetRate * p.Growth} - } - // The very first offer already missed. There is no lower bound to bisect - // toward, and halving indefinitely would just measure a smaller machine. - if lastGood == 0 { - return rampDecision{Stop: true, Reason: "server did not keep up even with the opening rate"} - } - // Bracketed. Bisect rather than stopping at the first miss: a single noisy - // step would otherwise end the ramp and report the last doubling, which on - // a geometric ramp understates capacity by up to half. - if (firstBad-lastGood)/firstBad <= p.RefineTolerance { - return rampDecision{Stop: true, Reason: "converged on the sustainable rate"} - } - return rampDecision{NextRate: (lastGood + firstBad) / 2} -} - -// bestHeldExportP95 is the lowest export p95 from a step that sustained its -// offered rate, i.e. the machine's behaviour before delivery fell behind. -// Failed steps only record successful exports, so their latency can look -// deceptively low and must not redefine the baseline used to judge good steps. -func bestHeldExportP95(history []rampStep, deliveryFloor float64) float64 { - best := 0.0 - for _, s := range history { - if s.held(deliveryFloor) && s.ExportP95Ms > 0 && (best == 0 || s.ExportP95Ms < best) { - best = s.ExportP95Ms - } - } - return best -} - -// kneeExceeded compares a loaded latency against an unloaded baseline. A -// baseline at or below zero is unmeasurable, and dividing by it would call -// every subsequent step a knee. -func kneeExceeded(baselineMs, loadedMs, knee float64) bool { - if baselineMs <= 0 { - return false - } - return loadedMs > baselineMs*knee -} - -// sustainableRate is the highest throughput the server both accepted and -// served without passing the latency knee. -func sustainableRate(history []rampStep, p rampPolicy) float64 { - best := bestHeldExportP95(history, p.DeliveryFloor) - sustained := 0.0 - for _, s := range history { - if !s.held(p.DeliveryFloor) || kneeExceeded(best, s.ExportP95Ms, p.LatencyKnee) { - continue - } - if s.AchievedRate > sustained { - sustained = s.AchievedRate - } - } - if sustained == 0 { - // Nothing held its target. The machine still did something, and - // reporting zero would misdescribe it as having done nothing. - return saturationRate(history) - } - return sustained -} - -// saturationRate is the most throughput observed at any offered rate, -// regardless of whether the server kept up or what it cost in latency. -func saturationRate(history []rampStep) float64 { - peak := 0.0 - for _, s := range history { - if s.AchievedRate > peak { - peak = s.AchievedRate - } - } - return peak -} - -// autoWorkers sizes in-flight senders from the driver's cores. Sending is -// round-trip bound rather than CPU bound, so this deliberately oversubscribes: -// one sender per core would leave a remote server mostly idle. -func autoWorkers(cores int) int { - if cores < 1 { - cores = 1 - } - workers := cores * 16 - if workers < 8 { - workers = 8 - } - if workers > maxAutoWorkers { - workers = maxAutoWorkers - } - return workers -} - -// seedRate is the ramp's first offer. Low enough that even a small machine -// clears it, so the first step establishes the unloaded latency baseline that -// every later knee check is measured against. -func seedRate(cores int) float64 { - if cores < 1 { - cores = 1 - } - return float64(cores) * 250 -} diff --git a/cmd/bench/adaptive_test.go b/cmd/bench/adaptive_test.go deleted file mode 100644 index 3f7f4686b..000000000 --- a/cmd/bench/adaptive_test.go +++ /dev/null @@ -1,229 +0,0 @@ -package main - -import ( - "math" - "testing" -) - -func TestAutoWorkersScalesWithCoresAndHasAFloor(t *testing.T) { - // A single-core driver still needs enough in-flight senders to keep a - // remote server busy: throughput here is round-trip bound, not CPU bound. - if got := autoWorkers(1); got < 8 { - t.Fatalf("autoWorkers(1) = %d, want at least 8", got) - } - if autoWorkers(8) <= autoWorkers(2) { - t.Fatal("autoWorkers must grow with core count") - } - // Unbounded growth would just queue work in the driver and inflate the - // latency it reports as the server's. - if got := autoWorkers(256); got > maxAutoWorkers { - t.Fatalf("autoWorkers(256) = %d, want <= %d", got, maxAutoWorkers) - } -} - -func TestDecideNextStepRampsWhileDeliveryHolds(t *testing.T) { - p := defaultRampPolicy() - history := []rampStep{{TargetRate: 1000, AchievedRate: 995, ExportP95Ms: 2}} - - got := decideNextStep(history, p) - - if got.Stop { - t.Fatalf("should keep ramping while delivery holds, stopped: %s", got.Reason) - } - if want := 1000 * p.Growth; got.NextRate != want { - t.Fatalf("NextRate = %v, want %v", got.NextRate, want) - } -} - -func TestDecideNextStepBisectsInsteadOfStoppingAtTheFirstMiss(t *testing.T) { - p := defaultRampPolicy() - // Asked for 8000, delivered 5000. Stopping here would report 4000 as the - // capacity of a machine that might well sustain 6000 — on a doubling ramp - // the first miss understates the answer by up to half. - history := []rampStep{ - {TargetRate: 4000, AchievedRate: 3980, ExportP95Ms: 3}, - {TargetRate: 8000, AchievedRate: 5000, ExportP95Ms: 6}, - } - - got := decideNextStep(history, p) - - if got.Stop { - t.Fatalf("should bisect the bracket, stopped: %s", got.Reason) - } - if got.NextRate <= 4000 || got.NextRate >= 8000 { - t.Fatalf("NextRate = %v, want a probe strictly inside (4000, 8000)", got.NextRate) - } -} - -func TestDecideNextStepIgnoresLatencyFromMissedStepWhenBuildingBaseline(t *testing.T) { - p := defaultRampPolicy() - // The failed step records latency only for successful exports, so its p95 - // can look better than the sustained lower bound. It must not make that - // known-good step appear to have crossed the latency knee. - history := []rampStep{ - {TargetRate: 1000, AchievedRate: 1000, ExportP95Ms: 100}, - {TargetRate: 2000, AchievedRate: 1500, ExportP95Ms: 1}, - } - - got := decideNextStep(history, p) - - if got.Stop { - t.Fatalf("should refine from the sustained lower bound, stopped: %s", got.Reason) - } - if got.NextRate != 1500 { - t.Fatalf("NextRate = %v, want midpoint 1500", got.NextRate) - } - if got := sustainableRate(history, p); got != 1000 { - t.Fatalf("sustainableRate = %v, want sustained lower bound 1000", got) - } -} - -func TestDecideNextStepStopsOnceTheBracketIsTight(t *testing.T) { - p := defaultRampPolicy() - // 3800 held, 4000 did not: a 5% bracket. Refining further measures noise. - history := []rampStep{ - {TargetRate: 3800, AchievedRate: 3790, ExportP95Ms: 3}, - {TargetRate: 4000, AchievedRate: 3000, ExportP95Ms: 6}, - } - - got := decideNextStep(history, p) - - if !got.Stop { - t.Fatalf("must stop once converged, got next rate %v", got.NextRate) - } - if got.Reason == "" { - t.Fatal("a stop must explain itself; the reason lands in the report") - } -} - -func TestDecideNextStepStopsWhenEvenTheOpeningRateMisses(t *testing.T) { - p := defaultRampPolicy() - // No lower bound exists, so there is nothing to bisect toward. - history := []rampStep{{TargetRate: 500, AchievedRate: 100, ExportP95Ms: 40}} - - if got := decideNextStep(history, p); !got.Stop { - t.Fatalf("must stop with no good lower bound, got %v", got.NextRate) - } -} - -func TestDecideNextStepRefinesTheLatencyKnee(t *testing.T) { - p := defaultRampPolicy() - // Delivery is fine, but latency has blown out relative to the best step. - // Treat the knee as the upper bracket rather than reporting the previous - // doubling as capacity without probing the large gap between them. - history := []rampStep{ - {TargetRate: 1000, AchievedRate: 1000, ExportP95Ms: 2}, - {TargetRate: 2000, AchievedRate: 1990, ExportP95Ms: 80}, - } - - got := decideNextStep(history, p) - - if got.Stop { - t.Fatalf("should refine the latency bracket, stopped: %s", got.Reason) - } - if got.NextRate != 1500 { - t.Fatalf("NextRate = %v, want midpoint 1500", got.NextRate) - } -} - -func TestDecideNextStepStopsAfterRefiningTheLatencyKnee(t *testing.T) { - p := defaultRampPolicy() - history := []rampStep{ - {TargetRate: 1000, AchievedRate: 1000, ExportP95Ms: 2}, - {TargetRate: 1100, AchievedRate: 1100, ExportP95Ms: 80}, - } - - got := decideNextStep(history, p) - - if !got.Stop { - t.Fatalf("must stop once the latency bracket is tight, got next rate %v", got.NextRate) - } -} - -func TestDecideNextStepStopsAtMaxSteps(t *testing.T) { - p := defaultRampPolicy() - history := make([]rampStep, p.MaxSteps) - for i := range history { - history[i] = rampStep{TargetRate: 1000, AchievedRate: 1000, ExportP95Ms: 1} - } - - if got := decideNextStep(history, p); !got.Stop { - t.Fatal("must stop at MaxSteps so a fast server cannot ramp forever") - } -} - -func TestDecideNextStepSeedsTheFirstStep(t *testing.T) { - got := decideNextStep(nil, defaultRampPolicy()) - - if got.Stop { - t.Fatal("an empty history must produce a first step, not a stop") - } - if got.NextRate <= 0 { - t.Fatalf("NextRate = %v, want positive", got.NextRate) - } -} - -func TestSustainableRateIgnoresStepsThatMissedTheirTarget(t *testing.T) { - p := defaultRampPolicy() - history := []rampStep{ - {TargetRate: 1000, AchievedRate: 1000, ExportP95Ms: 2}, - {TargetRate: 2000, AchievedRate: 1990, ExportP95Ms: 3}, - // Saturated: achieved is well past the last good step but the server - // never accepted what was asked, so it is not a sustainable rate. - {TargetRate: 4000, AchievedRate: 2600, ExportP95Ms: 90}, - } - - if got := sustainableRate(history, p); got != 1990 { - t.Fatalf("sustainableRate = %v, want 1990 (highest step that held its target)", got) - } -} - -func TestSaturationRateIsTheBestObservedThroughput(t *testing.T) { - history := []rampStep{ - {TargetRate: 1000, AchievedRate: 1000}, - {TargetRate: 4000, AchievedRate: 2600}, - {TargetRate: 8000, AchievedRate: 2550}, - } - - if got := saturationRate(history); got != 2600 { - t.Fatalf("saturationRate = %v, want 2600", got) - } -} - -func TestSustainableRateFallsBackWhenNoStepHeldItsTarget(t *testing.T) { - p := defaultRampPolicy() - // Even the very first step missed. Reporting zero would imply the server - // did nothing, when in fact it sustained 400/s. - history := []rampStep{{TargetRate: 1000, AchievedRate: 400, ExportP95Ms: 50}} - - if got := sustainableRate(history, p); got != 400 { - t.Fatalf("sustainableRate = %v, want the observed 400", got) - } -} - -func TestQueryKneeUsesUnloadedBaseline(t *testing.T) { - p := defaultRampPolicy() - // 60ms unloaded; 200ms under load is a normal cost of concurrency. - if kneeExceeded(60, 200, p.LatencyKnee) { - t.Fatal("a modest multiple of the baseline is not a knee") - } - // 60ms unloaded to 6s is queueing, not serving. - if !kneeExceeded(60, 6000, p.LatencyKnee) { - t.Fatal("a 100x baseline must register as a knee") - } -} - -func TestKneeIgnoresImmeasurablyFastBaselines(t *testing.T) { - // A sub-millisecond baseline would make any real latency look like a knee. - if kneeExceeded(0, 5, defaultRampPolicy().LatencyKnee) { - t.Fatal("a zero baseline must not trigger a knee") - } -} - -func TestSeedRateIsPositiveForAnyCoreCount(t *testing.T) { - for _, cores := range []int{0, 1, 2, 64} { - if got := seedRate(cores); got <= 0 || math.IsNaN(got) { - t.Fatalf("seedRate(%d) = %v, want positive", cores, got) - } - } -} diff --git a/cmd/bench/config.go b/cmd/bench/config.go deleted file mode 100644 index b7ed17e70..000000000 --- a/cmd/bench/config.go +++ /dev/null @@ -1,57 +0,0 @@ -package main - -import ( - "errors" - "strings" -) - -// validateConfig validates the command-level mode before adaptive execution -// replaces the zero rate with a concrete probe rate. -func validateConfig(cfg config) error { - if cfg.adaptive() { - if cfg.stepDuration <= 0 { - return errors.New("-step must be positive in adaptive mode") - } - cfg.rate = 1 - } - return validateTrialConfig(cfg) -} - -func validateTrialConfig(cfg config) error { - switch { - case strings.TrimSpace(cfg.token) == "": - return errors.New("-token is required") - // Zero is not "unset" here: by the time a trial runs, adaptive mode has - // resolved a concrete rate and worker count. A zero reaching this point - // means the ramp produced a step it cannot actually execute. - case cfg.rate <= 0: - return errors.New("-rate must be positive") - case cfg.workers <= 0: - return errors.New("-workers must be positive") - case cfg.services < 2: - return errors.New("-services must be >= 2 (cross-service edges need a caller and a callee)") - case cfg.namespaces <= 0: - return errors.New("-namespaces must be positive") - case cfg.cardinality <= 0: - return errors.New("-attr-cardinality must be positive") - case cfg.errorRate < 0 || cfg.errorRate > 1: - return errors.New("-error-rate must be between 0 and 1") - case cfg.msgRatio < 0 || cfg.msgRatio > 1: - return errors.New("-messaging-ratio must be between 0 and 1") - case cfg.duration < 0: - return errors.New("-duration cannot be negative") - case cfg.queryWorkers < 0: - return errors.New("-query-workers cannot be negative") - case cfg.queryWorkers > 0 && strings.TrimSpace(cfg.queryURL) == "": - return errors.New("-query-url is required when -query-workers is positive") - case cfg.queryWorkers > 0 && strings.TrimSpace(cfg.querySessionCookie) == "": - return errors.New("-query-session-cookie is required when -query-workers is positive") - case cfg.queryWorkers > 0 && cfg.queryRate <= 0: - return errors.New("-query-rate must be positive when query load is enabled") - case cfg.maxExportP95 < 0 || cfg.maxQueryP95 < 0: - return errors.New("latency thresholds cannot be negative") - case cfg.backfillHours < 0: - return errors.New("-backfill-hours cannot be negative") - } - return nil -} diff --git a/cmd/bench/config_test.go b/cmd/bench/config_test.go deleted file mode 100644 index e0302760d..000000000 --- a/cmd/bench/config_test.go +++ /dev/null @@ -1,87 +0,0 @@ -package main - -import ( - "strings" - "testing" - "time" -) - -func TestValidateTrialConfigRejectsUnusableInputs(t *testing.T) { - valid := config{ - token: "fo_test", - rate: 1000, workers: 8, services: 20, namespaces: 1, cardinality: 100, - errorRate: 0.05, msgRatio: 0.2, - } - tests := []struct { - name string - mutate func(*config) - }{ - {name: "zero rate", mutate: func(cfg *config) { cfg.rate = 0 }}, - {name: "zero workers", mutate: func(cfg *config) { cfg.workers = 0 }}, - {name: "missing ingest token", mutate: func(cfg *config) { cfg.token = "" }}, - {name: "one service", mutate: func(cfg *config) { cfg.services = 1 }}, - {name: "invalid error rate", mutate: func(cfg *config) { cfg.errorRate = 1.1 }}, - {name: "invalid messaging ratio", mutate: func(cfg *config) { cfg.msgRatio = -0.1 }}, - {name: "query workers without URL", mutate: func(cfg *config) { - cfg.queryWorkers = 1 - cfg.queryRate = 10 - cfg.querySessionCookie = "fanout_session=test" - }}, - {name: "query workers without session cookie", mutate: func(cfg *config) { cfg.queryWorkers = 1; cfg.queryRate = 10; cfg.queryURL = "http://x" }}, - {name: "query workers without rate", mutate: func(cfg *config) { - cfg.queryWorkers = 1 - cfg.queryURL = "http://x" - cfg.querySessionCookie = "fanout_session=test" - }}, - {name: "negative backfill", mutate: func(cfg *config) { cfg.backfillHours = -1 }}, - } - for _, test := range tests { - t.Run(test.name, func(t *testing.T) { - cfg := valid - test.mutate(&cfg) - if err := validateTrialConfig(cfg); err == nil { - t.Fatalf("config unexpectedly valid: %#v", cfg) - } - }) - } - if err := validateTrialConfig(valid); err != nil { - t.Fatalf("valid config rejected: %v", err) - } -} - -func TestValidateConfigRejectsNonPositiveAdaptiveStep(t *testing.T) { - cfg := config{ - token: "fo_test", - workers: 8, services: 20, namespaces: 1, cardinality: 100, - errorRate: 0.05, msgRatio: 0.2, stepDuration: 0, - } - - err := validateConfig(cfg) - if err == nil || !strings.Contains(err.Error(), "-step must be positive") { - t.Fatalf("validateConfig error = %v, want a positive-step error", err) - } -} - -func TestValidateConfigAllowsPositiveAdaptiveStep(t *testing.T) { - cfg := config{ - token: "fo_test", - workers: 8, services: 20, namespaces: 1, cardinality: 100, - errorRate: 0.05, msgRatio: 0.2, stepDuration: time.Second, - } - - if err := validateConfig(cfg); err != nil { - t.Fatalf("valid adaptive config rejected: %v", err) - } -} - -func TestValidateConfigDoesNotApplyStepToFixedRateRuns(t *testing.T) { - cfg := config{ - token: "fo_test", - rate: 1000, workers: 8, services: 20, namespaces: 1, cardinality: 100, - errorRate: 0.05, msgRatio: 0.2, stepDuration: 0, - } - - if err := validateConfig(cfg); err != nil { - t.Fatalf("fixed-rate config rejected because of unused -step: %v", err) - } -} diff --git a/cmd/bench/main.go b/cmd/bench/main.go deleted file mode 100644 index fbf9e5380..000000000 --- a/cmd/bench/main.go +++ /dev/null @@ -1,1006 +0,0 @@ -// Command bench is fanout's local stress/soak generator and benchmark -// reporter. It pushes synthetic OTLP (traces, logs, metrics) straight at the -// gRPC ingest port with knobs for rate, duration, service/namespace count, -// attribute cardinality, and error rate, then reports OTLP export latency -// percentiles and — when pointed at fanout's /-/metrics — the server-side -// deltas that matter (rows accepted/dropped, file growth, rollup/flush time). -// -// Unlike telemetrygen (single service per process), bench emits cross-service -// parent/child traces and producer/consumer pairs, so it exercises BOTH rollups -// — service_rollup and edge_rollup (call + messaging topology) — plus the -// GROUP BY cardinality cost. It reuses fanout's own vendored OTLP proto, so it -// adds no new dependencies. -// -// Example — a 10-minute soak at 2k traces/s across 50 services, with a report: -// -// go run ./cmd/bench -rate 2000 -duration 10m -services 50 -attr-cardinality 200 \ -// -token "$INGEST_TOKEN" \ -// -metrics-url http://localhost:7520/-/metrics -metrics-token "$FANOUT_METRICS_TOKEN" -report run.json -// -// The metrics endpoint requires -metrics-token unless FANOUT_METRICS_PUBLIC=true. -package main - -import ( - "context" - "encoding/json" - "flag" - "fmt" - "io" - "log" - "math" - "math/rand/v2" - "net/http" - "os" - "os/signal" - "sort" - "strings" - "sync" - "sync/atomic" - "syscall" - "time" - - collectorlogs "go.opentelemetry.io/proto/otlp/collector/logs/v1" - collectormetrics "go.opentelemetry.io/proto/otlp/collector/metrics/v1" - collectortrace "go.opentelemetry.io/proto/otlp/collector/trace/v1" - common "go.opentelemetry.io/proto/otlp/common/v1" - logspb "go.opentelemetry.io/proto/otlp/logs/v1" - metricspb "go.opentelemetry.io/proto/otlp/metrics/v1" - resourcepb "go.opentelemetry.io/proto/otlp/resource/v1" - tracepb "go.opentelemetry.io/proto/otlp/trace/v1" - "google.golang.org/grpc" - "google.golang.org/grpc/credentials/insecure" - "google.golang.org/grpc/metadata" -) - -type config struct { - endpoint string - token string - rate float64 - duration time.Duration - workers int - services int - namespaces int - cardinality int - errorRate float64 - msgRatio float64 - sendLogs bool - sendMetrics bool - metricsURL string - metricsToken string - reportPath string - // Query-under-load: drive the read path concurrently with ingest. - queryURL string - queryWorkers int - queryRate float64 - // querySessionCookie is the Cookie header from a signed-in Fanout account. - // It is deliberately omitted from reports and logs. - querySessionCookie string - // Pass/fail thresholds (0 = no threshold). Non-zero exit on violation so - // the harness can gate CI / releases. - maxExportP95 float64 - maxQueryP95 float64 - // backfillHours, when >0, spreads each event's timestamp uniformly over the - // last N hours (instead of "now"). Used to PRE-SEED a multi-hour dataset so - // time-window queries and rollups cover more than the current hour. - backfillHours float64 - // seed makes the synthetic workload reproducible: same seed, same services, - // endpoints, attributes, and error placement. Two runs are only comparable if - // they share it. - seed uint64 - // fanoutVersion is recorded in the report so a stored run.json says which - // server build produced it. Report-only; it never changes Fanout config. - fanoutVersion string - // stepDuration is how long each adaptive step runs. Unused when -rate is - // set explicitly, which pins the harness to a single fixed-rate trial. - stepDuration time.Duration -} - -// adaptive reports whether this run discovers its own rate. An explicit -rate -// pins the harness to one trial, which is what a regression gate wants; the -// default ramps, which is what a capacity question wants. -func (c config) adaptive() bool { return c.rate <= 0 } - -// version is set at build time via -ldflags "-X main.version=...". -var version = "dev" - -func main() { - var cfg config - flag.StringVar(&cfg.endpoint, "endpoint", "localhost:4317", "OTLP gRPC endpoint") - flag.StringVar(&cfg.token, "token", "", "ingest bearer token; required") - flag.Float64Var(&cfg.rate, "rate", 0, "target traces per second (aggregate); 0 ramps adaptively to find what this server sustains") - flag.DurationVar(&cfg.duration, "duration", time.Minute, "run duration for a fixed -rate run; 0 means run until interrupted") - flag.IntVar(&cfg.workers, "workers", 0, "concurrent senders; 0 sizes them from the driver's cores") - flag.DurationVar(&cfg.stepDuration, "step", 30*time.Second, "duration of each step while ramping adaptively") - flag.IntVar(&cfg.services, "services", 20, "number of distinct services") - flag.IntVar(&cfg.namespaces, "namespaces", 1, "number of distinct namespaces") - flag.IntVar(&cfg.cardinality, "attr-cardinality", 100, "distinct values per high-cardinality attribute") - flag.Float64Var(&cfg.errorRate, "error-rate", 0.05, "fraction of spans marked STATUS_CODE_ERROR (0..1)") - flag.Float64Var(&cfg.msgRatio, "messaging-ratio", 0.2, "fraction of traces that also emit a producer/consumer pair (0..1)") - flag.BoolVar(&cfg.sendLogs, "logs", true, "also emit logs") - flag.BoolVar(&cfg.sendMetrics, "metrics", true, "also emit metrics") - flag.StringVar(&cfg.metricsURL, "metrics-url", "", "fanout /-/metrics URL to capture server-side deltas (e.g. http://localhost:7520/-/metrics)") - flag.StringVar(&cfg.metricsToken, "metrics-token", "", "bearer token for a private fanout /-/metrics endpoint") - flag.StringVar(&cfg.reportPath, "report", "", "write a JSON performance report to this path") - flag.StringVar(&cfg.queryURL, "query-url", "", "fanout HTTP base URL to drive read load under ingest (e.g. http://localhost:7520)") - flag.IntVar(&cfg.queryWorkers, "query-workers", 0, "concurrent query workers (0 = ingest only)") - flag.Float64Var(&cfg.queryRate, "query-rate", 50, "target queries/sec (aggregate) when query-workers > 0") - flag.StringVar(&cfg.querySessionCookie, "query-session-cookie", "", "Cookie header from an authenticated Fanout account; required when query-workers > 0") - flag.Float64Var(&cfg.maxExportP95, "max-export-p95-ms", 0, "fail (exit 1) if ingest export p95 exceeds this") - flag.Float64Var(&cfg.maxQueryP95, "max-query-p95-ms", 0, "fail (exit 1) if query p95 exceeds this") - flag.Float64Var(&cfg.backfillHours, "backfill-hours", 0, "spread event timestamps over the last N hours (pre-seed a multi-hour dataset); 0 = use now()") - flag.Uint64Var(&cfg.seed, "seed", 1, "deterministic synthetic workload seed") - flag.StringVar(&cfg.fanoutVersion, "fanout-version", "unknown", "Fanout build/version identifier under test, recorded in the report") - flag.Parse() - - // Self-size before validation: a zero worker count is how the operator asks - // for auto-sizing, but every trial downstream needs a concrete number. - if cfg.workers <= 0 { - cfg.workers = autoWorkers(numCPU()) - } - if err := validateConfig(cfg); err != nil { - log.Fatal(err) - } - - // No global deadline: in adaptive mode the run is a sequence of steps, each - // bounded separately. Signals still cut the whole thing short. - ctx, stop := signal.NotifyContext(context.Background(), os.Interrupt, syscall.SIGTERM) - defer stop() - - conn, err := grpc.NewClient(cfg.endpoint, grpc.WithTransportCredentials(insecure.NewCredentials())) - if err != nil { - log.Fatalf("dial %s: %v", cfg.endpoint, err) - } - defer conn.Close() - - fmt.Printf("bench → %s | workers=%d services=%d namespaces=%d cardinality=%d error-rate=%.2f\n", - cfg.endpoint, cfg.workers, cfg.services, cfg.namespaces, cfg.cardinality, cfg.errorRate) - - var baseline *metricSnapshot - var infrastructureFailures []string - - // Session clock, spanning the ramp and the confirmation pass. Each trial - // keeps its own clock for its own rate; the manifest describes the whole run. - start := time.Now() - - // Ramp first to find what this server sustains, then confirm at that rate. - // The headline numbers therefore describe a rate the machine actually holds, - // not the saturated step that ended the ramp. - var steps []rampStep - var stopReason string - if cfg.adaptive() { - policy := defaultRampPolicy() - fmt.Printf("adaptive: ramping in %s steps (%d max) to find the sustainable rate\n", cfg.stepDuration, policy.MaxSteps) - for { - decision := decideNextStep(steps, policy) - if decision.Stop { - stopReason = decision.Reason - break - } - trial := cfg - trial.rate = decision.NextRate - trial.duration = cfg.stepDuration - sg, selapsed := runTrial(ctx, trial, conn, true) - step := rampStep{ - TargetRate: round2(decision.NextRate), - AchievedRate: round2(float64(sg.tracesSent.Load()) / selapsed), - ExportP95Ms: round2(sg.lat.quantile(0.95)), - } - if trial.queryURL != "" && trial.queryWorkers > 0 { - step.QueryP95Ms = round2(sg.queryLat.quantile(0.95)) - } - steps = append(steps, step) - fmt.Printf(" step %d: offered %.0f/s → sustained %.0f/s, export p95 %.1fms\n", - len(steps), step.TargetRate, step.AchievedRate, step.ExportP95Ms) - if ctx.Err() != nil { - stopReason = "interrupted" - break - } - } - cfg.rate = sustainableRate(steps, policy) - if cfg.rate <= 0 { - cfg.rate = seedRate(numCPU()) - } - fmt.Printf("adaptive: stopped because %s — sustainable %.0f/s, peak %.0f/s. Confirming at %.0f/s for %s.\n", - stopReason, cfg.rate, saturationRate(steps), cfg.rate, cfg.duration) - } - - // Baseline immediately before the confirmation pass, not before the ramp. - // Every server-side rate is a delta divided by the confirmation pass's - // elapsed time, so a baseline taken earlier would fold the entire ramp into - // the numerator and report, say, 3.26 busy cores on a two-core machine. - if cfg.metricsURL != "" { - if baseline, err = scrapeMetrics(cfg.metricsURL, cfg.metricsToken); err != nil { - fmt.Fprintf(os.Stderr, " warn: baseline metrics scrape failed: %v\n", err) - infrastructureFailures = append(infrastructureFailures, "baseline metrics scrape failed") - } - } - - g, elapsed := runTrial(ctx, cfg, conn, false) - - rep := report{ - Manifest: newRunManifest(cfg, start.UTC(), time.Now().UTC()), - Endpoint: cfg.endpoint, - DurationSec: round2(elapsed), - TargetRate: cfg.rate, - TracesSent: g.tracesSent.Load(), - SpansSent: g.spansSent.Load(), - LogsSent: g.logsSent.Load(), - MetricsSent: g.metricsSent.Load(), - SendErrors: g.sendErrs.Load(), - AvgTracesPerSec: round2(float64(g.tracesSent.Load()) / elapsed), - ExportLatencyMs: g.lat.snapshot(), - } - if cfg.queryURL != "" && cfg.queryWorkers > 0 { - ql := g.queryLat.snapshot() - rep.QueryLatencyMs = &ql - rep.QueryLatencyByOperation = make(map[string]latencyReport, len(g.queryLatByOperation)) - for operation, histogram := range g.queryLatByOperation { - rep.QueryLatencyByOperation[operation] = histogram.snapshot() - } - rep.QueriesRun = g.queriesRun.Load() - rep.QueryErrors = g.queryErrs.Load() - } - if cfg.metricsURL != "" { - if final, ferr := scrapeMetrics(cfg.metricsURL, cfg.metricsToken); ferr != nil { - fmt.Fprintf(os.Stderr, " warn: final metrics scrape failed: %v\n", ferr) - infrastructureFailures = append(infrastructureFailures, "final metrics scrape failed") - } else { - rep.Server = serverDelta(baseline, final, elapsed) - } - } - - if cfg.adaptive() || len(steps) > 0 { - policy := defaultRampPolicy() - rampEstimate := sustainableRate(steps, policy) - confirmed := rep.AvgTracesPerSec - sustainable := rampEstimate - if confirmed > 0 && confirmed < sustainable { - sustainable = confirmed - } - rep.Capacity = &capacityReport{ - Steps: steps, - SustainableTracesPerSec: round2(sustainable), - RampEstimateTracesPerSec: round2(rampEstimate), - ConfirmedTracesPerSec: round2(confirmed), - SaturationTracesPerSec: round2(saturationRate(steps)), - StopReason: stopReason, - Workers: cfg.workers, - DriverLogicalCPUs: numCPU(), - } - } - - rep.Failures = evaluateReport(cfg, rep, infrastructureFailures) - rep.Passed = len(rep.Failures) == 0 - printReport(rep) - - // A report that was asked for and not written is a failed run, not a warning. - // Callers that aggregate trials cannot tell "missing" from "fine" after the - // fact, so exiting 0 here turns a lost trial into a silent pass upstream. - reportWritten := true - if cfg.reportPath != "" { - if err := writeReport(cfg.reportPath, rep); err != nil { - fmt.Fprintf(os.Stderr, " error: write report: %v\n", err) - reportWritten = false - } else { - fmt.Printf("report written: %s\n", cfg.reportPath) - } - } - - if !rep.Passed { - fmt.Fprintf(os.Stderr, "FAIL: %s\n", strings.Join(rep.Failures, "; ")) - os.Exit(1) - } - if !reportWritten { - os.Exit(1) - } -} - -// runTrial executes one fixed-rate trial and returns its generator and the -// wall-clock seconds it covered. Each trial gets a fresh generator so its -// histograms describe that rate alone; a shared one would blend every step of -// a ramp into a single indistinguishable distribution. -// -// quiet suppresses per-5s progress lines, which are noise during a ramp and -// useful during the confirmation pass. -func runTrial(ctx context.Context, cfg config, conn *grpc.ClientConn, quiet bool) (*generator, float64) { - g := &generator{ - cfg: cfg, - traces: collectortrace.NewTraceServiceClient(conn), - logs: collectorlogs.NewLogsServiceClient(conn), - metrics: collectormetrics.NewMetricsServiceClient(conn), - lat: newHistogram(), - queryLat: newHistogram(), - queryLatByOperation: map[string]*histogram{ - "overview": newHistogram(), - "topology": newHistogram(), - "performance": newHistogram(), - "trace": newHistogram(), - "logs": newHistogram(), - }, - http: &http.Client{Timeout: 30 * time.Second}, - svcNames: func() []string { - names := make([]string, cfg.services) - for i := range names { - names[i] = fmt.Sprintf("svc-%02d", i) - } - return names - }(), - } - - trialCtx := ctx - if cfg.duration > 0 { - var cancel context.CancelFunc - trialCtx, cancel = context.WithTimeout(ctx, cfg.duration) - defer cancel() - } - - interval := time.Duration(float64(time.Second) * float64(cfg.workers) / cfg.rate) - start := time.Now() - - var wg sync.WaitGroup - for w := 0; w < cfg.workers; w++ { - worker := w - wg.Add(1) - go func() { - defer wg.Done() - rng := rand.New(rand.NewPCG(cfg.seed, uint64(worker)+1)) - g.run(trialCtx, interval, rng) - }() - } - if cfg.queryURL != "" && cfg.queryWorkers > 0 { - qInterval := time.Duration(float64(time.Second) * float64(cfg.queryWorkers) / cfg.queryRate) - if !quiet { - fmt.Printf("query load: %d workers @ %.0f q/s → %s/api/observability/{overview,topology,performance,trace,logs}\n", - cfg.queryWorkers, cfg.queryRate, strings.TrimRight(cfg.queryURL, "/")) - } - for w := 0; w < cfg.queryWorkers; w++ { - wg.Add(1) - go func() { - defer wg.Done() - g.runQueries(trialCtx, qInterval) - }() - } - } - - done := make(chan struct{}) - go func() { - t := time.NewTicker(5 * time.Second) - defer t.Stop() - for { - select { - case <-trialCtx.Done(): - close(done) - return - case <-t.C: - if quiet { - continue - } - elapsed := time.Since(start).Seconds() - sent := g.tracesSent.Load() - fmt.Printf(" %5.0fs traces=%d (%.0f/s) spans=%d p95=%.1fms errors=%d\n", - elapsed, sent, float64(sent)/elapsed, g.spansSent.Load(), g.lat.quantile(0.95), g.sendErrs.Load()) - } - } - }() - - wg.Wait() - <-done - return g, time.Since(start).Seconds() -} - -type generator struct { - cfg config - traces collectortrace.TraceServiceClient - logs collectorlogs.LogsServiceClient - metrics collectormetrics.MetricsServiceClient - lat *histogram - queryLat *histogram - // Populated before workers start and never mutated. Histograms use atomics. - queryLatByOperation map[string]*histogram - http *http.Client - svcNames []string - - tracesSent atomic.Int64 - spansSent atomic.Int64 - logsSent atomic.Int64 - metricsSent atomic.Int64 - sendErrs atomic.Int64 - lastErr atomic.Pointer[error] - queriesRun atomic.Int64 - queryErrs atomic.Int64 -} - -// queryWindows rotate so the read load hits both small and wide rollup windows, -// exercising the latency SLOs under ingest. The shared HTTP/MCP observability -// kernel accepts Go duration strings and caps the window at 24 hours. -var ( - queryOperations = []string{"overview", "topology", "performance", "trace", "logs"} - queryWindows = []string{"15m", "1h", "6h", "12h", "24h"} -) - -type queryTarget struct { - operation string - path string -} - -func queryTargetAt(i int) queryTarget { - operationIndex := i % len(queryOperations) - round := i / len(queryOperations) - // Offset each successive round so every operation exercises every window; - // using i for both dimensions locks equal-length lists into fixed pairs. - window := queryWindows[(operationIndex+round)%len(queryWindows)] - operation := queryOperations[operationIndex] - return queryTarget{ - operation: operation, - path: fmt.Sprintf("/api/observability/%s?window=%s&limit=100", operation, window), - } -} - -// maxSendErrorRate tolerates a handful of transient export errors (e.g. a gRPC -// DeadlineExceeded during a rollup pause) at high volume — they are noise, not a -// capacity failure. The real data-integrity gate is server-side rows-dropped. -// A genuine backpressure failure produces a send-error rate far above this. -const maxSendErrorRate = 0.001 // 0.1% - -// runQueries drives the shared HTTP/MCP observability read path concurrently -// with ingest, recording latency into queryLat. -func (g *generator) runQueries(ctx context.Context, interval time.Duration) { - ticker := time.NewTicker(interval) - defer ticker.Stop() - i := 0 - base := strings.TrimRight(g.cfg.queryURL, "/") - reportedFailure := make(map[string]bool) - reportFailure := func(operation, detail string) { - if reportedFailure[operation] { - return - } - reportedFailure[operation] = true - log.Printf("query %s first failure: %s", operation, detail) - } - for { - select { - case <-ctx.Done(): - return - case <-ticker.C: - target := queryTargetAt(i) - url := base + target.path - i++ - t0 := time.Now() - // Bound to the run context so a query issued just before the - // deadline is cancelled rather than running on for the client - // timeout, which both delays the final scrape and hides its own - // failure (ctx.Err() is non-nil by the time it returns). - req, reqErr := g.newQueryRequest(ctx, url) - if reqErr != nil { - g.queryErrs.Add(1) - reportFailure(target.operation, reqErr.Error()) - continue - } - resp, err := g.http.Do(req) - if err != nil { - if ctx.Err() == nil { - g.queryErrs.Add(1) - reportFailure(target.operation, err.Error()) - } - continue - } - contentType := resp.Header.Get("Content-Type") - body, readErr := io.ReadAll(io.LimitReader(resp.Body, 1<<20)) - resp.Body.Close() - if resp.StatusCode != http.StatusOK || readErr != nil || - !strings.HasPrefix(contentType, "application/json") || - !json.Valid(body) { - g.queryErrs.Add(1) - sample := body - if len(sample) > 256 { - sample = sample[:256] - } - reportFailure(target.operation, fmt.Sprintf("status=%d content_type=%q read_err=%v body=%q", resp.StatusCode, contentType, readErr, sample)) - continue - } - latency := time.Since(t0) - g.queryLat.record(latency) - g.queryLatByOperation[target.operation].record(latency) - g.queriesRun.Add(1) - } - } -} - -func (g *generator) newQueryRequest(ctx context.Context, url string) (*http.Request, error) { - req, err := http.NewRequestWithContext(ctx, http.MethodGet, url, nil) - if err != nil { - return nil, err - } - req.Header.Set("Cookie", g.cfg.querySessionCookie) - return req, nil -} - -func (g *generator) run(ctx context.Context, interval time.Duration, rng *rand.Rand) { - ticker := time.NewTicker(interval) - defer ticker.Stop() - for { - select { - case <-ctx.Done(): - return - case <-ticker.C: - g.sendTrace(ctx, rng) - if g.cfg.sendLogs { - g.sendLog(ctx, rng) - } - if g.cfg.sendMetrics { - g.sendMetric(ctx, rng) - } - } - } -} - -// countSendErr records a failed export, except when the run context is already -// done — in-flight requests cancelled at the duration deadline or on Ctrl-C are -// a clean shutdown, not a fanout failure, and shouldn't inflate the error count. -func (g *generator) countSendErr(ctx context.Context, err error) { - if ctx.Err() != nil { - return - } - g.sendErrs.Add(1) - if g.lastErr.CompareAndSwap(nil, &err) { // surface the first real error once - fmt.Fprintf(os.Stderr, " send error: %v\n", err) - } -} - -func (g *generator) outCtx(ctx context.Context) context.Context { - if g.cfg.token == "" { - return ctx - } - return metadata.AppendToOutgoingContext(ctx, "authorization", "Bearer "+g.cfg.token) -} - -// eventTime returns the timestamp for an emitted event: now(), or — when -// backfillHours>0 — a time spread uniformly over the last N hours so a pre-seed -// run exercises multi-hour time-window queries and rollups. -func (g *generator) eventTime(rng *rand.Rand) time.Time { - if g.cfg.backfillHours <= 0 { - return time.Now() - } - return time.Now().Add(-time.Duration(rng.Float64() * g.cfg.backfillHours * float64(time.Hour))) -} - -// sendTrace emits one trace: a SERVER span in the caller service and a CLIENT -// child span in a different callee service (a call edge), plus — for a fraction -// of traces — a PRODUCER/CONSUMER pair on a messaging destination (a messaging -// edge). Each ResourceSpans carries one service.name, matching how fanout keys -// service_rollup and edge_rollup. -func (g *generator) sendTrace(ctx context.Context, rng *rand.Rand) { - ns := g.namespace(rng) - caller := g.svcNames[rng.IntN(g.cfg.services)] - callee := g.svcNames[rng.IntN(g.cfg.services)] - for callee == caller { - callee = g.svcNames[rng.IntN(g.cfg.services)] - } - - traceID := randBytes(rng, 16) - parentID := randBytes(rng, 8) - childID := randBytes(rng, 8) - now := g.eventTime(rng) - startNano := uint64(now.UnixNano()) - parentEnd := uint64(now.Add(time.Duration(20+rng.IntN(400)) * time.Millisecond).UnixNano()) - childEnd := uint64(now.Add(time.Duration(5+rng.IntN(200)) * time.Millisecond).UnixNano()) - - resSpans := []*tracepb.ResourceSpans{ - g.resourceSpans(caller, ns, &tracepb.Span{ - TraceId: traceID, SpanId: parentID, - Name: "GET /" + g.route(rng), Kind: tracepb.Span_SPAN_KIND_SERVER, - StartTimeUnixNano: startNano, EndTimeUnixNano: parentEnd, - Attributes: g.spanAttrs(rng), Status: g.status(rng), - }), - g.resourceSpans(callee, ns, &tracepb.Span{ - TraceId: traceID, SpanId: childID, ParentSpanId: parentID, - Name: "rpc." + g.route(rng), Kind: tracepb.Span_SPAN_KIND_CLIENT, - StartTimeUnixNano: startNano, EndTimeUnixNano: childEnd, - Attributes: g.spanAttrs(rng), Status: g.status(rng), - }), - } - spans := 2 - - if rng.Float64() < g.cfg.msgRatio { - dest := fmt.Sprintf("topic-%d", rng.IntN(8)) - msgAttrs := []*common.KeyValue{ - strAttr("messaging.destination.name", dest), - strAttr("messaging.system", "kafka"), - } - producer := g.svcNames[rng.IntN(g.cfg.services)] - consumer := g.svcNames[rng.IntN(g.cfg.services)] - for consumer == producer { - consumer = g.svcNames[rng.IntN(g.cfg.services)] - } - resSpans = append(resSpans, - g.resourceSpans(producer, ns, &tracepb.Span{ - TraceId: randBytes(rng, 16), SpanId: randBytes(rng, 8), - Name: dest + " publish", Kind: tracepb.Span_SPAN_KIND_PRODUCER, - StartTimeUnixNano: startNano, EndTimeUnixNano: parentEnd, - Attributes: msgAttrs, Status: g.status(rng), - }), - g.resourceSpans(consumer, ns, &tracepb.Span{ - TraceId: randBytes(rng, 16), SpanId: randBytes(rng, 8), - Name: dest + " process", Kind: tracepb.Span_SPAN_KIND_CONSUMER, - StartTimeUnixNano: startNano, EndTimeUnixNano: childEnd, - Attributes: msgAttrs, Status: g.status(rng), - }), - ) - spans += 2 - } - - t0 := time.Now() - _, err := g.traces.Export(g.outCtx(ctx), &collectortrace.ExportTraceServiceRequest{ResourceSpans: resSpans}) - if err != nil { - g.countSendErr(ctx, err) - return - } - g.lat.record(time.Since(t0)) - g.tracesSent.Add(1) - g.spansSent.Add(int64(spans)) -} - -func (g *generator) resourceSpans(service, ns string, span *tracepb.Span) *tracepb.ResourceSpans { - return &tracepb.ResourceSpans{ - Resource: &resourcepb.Resource{Attributes: []*common.KeyValue{ - strAttr("service.name", service), - strAttr("service.namespace", ns), - }}, - ScopeSpans: []*tracepb.ScopeSpans{{ - Scope: &common.InstrumentationScope{Name: "bench"}, - Spans: []*tracepb.Span{span}, - }}, - } -} - -func (g *generator) sendLog(ctx context.Context, rng *rand.Rand) { - ns := g.namespace(rng) - svc := g.svcNames[rng.IntN(g.cfg.services)] - sev := "INFO" - sevNum := logspb.SeverityNumber_SEVERITY_NUMBER_INFO - if rng.Float64() < g.cfg.errorRate { - sev, sevNum = "ERROR", logspb.SeverityNumber_SEVERITY_NUMBER_ERROR - } - req := &collectorlogs.ExportLogsServiceRequest{ResourceLogs: []*logspb.ResourceLogs{{ - Resource: &resourcepb.Resource{Attributes: []*common.KeyValue{ - strAttr("service.name", svc), strAttr("service.namespace", ns), - }}, - ScopeLogs: []*logspb.ScopeLogs{{ - LogRecords: []*logspb.LogRecord{{ - TimeUnixNano: uint64(g.eventTime(rng).UnixNano()), - SeverityText: sev, - SeverityNumber: sevNum, - Body: &common.AnyValue{Value: &common.AnyValue_StringValue{StringValue: "request " + g.route(rng)}}, - Attributes: g.spanAttrs(rng), - }}, - }}, - }}} - t0 := time.Now() - if _, err := g.logs.Export(g.outCtx(ctx), req); err != nil { - g.countSendErr(ctx, err) - return - } - g.lat.record(time.Since(t0)) - g.logsSent.Add(1) -} - -func (g *generator) sendMetric(ctx context.Context, rng *rand.Rand) { - ns := g.namespace(rng) - svc := g.svcNames[rng.IntN(g.cfg.services)] - req := &collectormetrics.ExportMetricsServiceRequest{ResourceMetrics: []*metricspb.ResourceMetrics{{ - Resource: &resourcepb.Resource{Attributes: []*common.KeyValue{ - strAttr("service.name", svc), strAttr("service.namespace", ns), - }}, - ScopeMetrics: []*metricspb.ScopeMetrics{{ - Metrics: []*metricspb.Metric{{ - Name: "request.duration", - Unit: "ms", - Data: &metricspb.Metric_Gauge{Gauge: &metricspb.Gauge{ - DataPoints: []*metricspb.NumberDataPoint{{ - TimeUnixNano: uint64(g.eventTime(rng).UnixNano()), - Value: &metricspb.NumberDataPoint_AsDouble{AsDouble: rng.Float64() * 500}, - Attributes: g.spanAttrs(rng), - }}, - }}, - }}, - }}, - }}} - t0 := time.Now() - if _, err := g.metrics.Export(g.outCtx(ctx), req); err != nil { - g.countSendErr(ctx, err) - return - } - g.lat.record(time.Since(t0)) - g.metricsSent.Add(1) -} - -func (g *generator) namespace(rng *rand.Rand) string { - if g.cfg.namespaces <= 1 { - return "default" - } - return fmt.Sprintf("ns-%02d", rng.IntN(g.cfg.namespaces)) -} - -func (g *generator) route(rng *rand.Rand) string { - return fmt.Sprintf("r%d", rng.IntN(20)) -} - -// spanAttrs carries a high-cardinality key to stress the rollup GROUP BYs and -// attribute extraction. -func (g *generator) spanAttrs(rng *rand.Rand) []*common.KeyValue { - return []*common.KeyValue{ - strAttr("http.method", []string{"GET", "POST", "PUT", "DELETE"}[rng.IntN(4)]), - strAttr("user.id", fmt.Sprintf("u-%d", rng.IntN(g.cfg.cardinality))), - } -} - -func (g *generator) status(rng *rand.Rand) *tracepb.Status { - if rng.Float64() < g.cfg.errorRate { - return &tracepb.Status{Code: tracepb.Status_STATUS_CODE_ERROR} - } - return &tracepb.Status{Code: tracepb.Status_STATUS_CODE_OK} -} - -func strAttr(k, v string) *common.KeyValue { - return &common.KeyValue{Key: k, Value: &common.AnyValue{Value: &common.AnyValue_StringValue{StringValue: v}}} -} - -func randBytes(rng *rand.Rand, n int) []byte { - b := make([]byte, n) - for i := range b { - b[i] = byte(rng.IntN(256)) - } - return b -} - -// ── Export-latency histogram ─────────────────────────────────────────────── -// Lock-free, fixed-bucket (log-spaced ms) so it's safe and bounded under a -// multi-hour soak. Percentiles are bucket-granular approximations (the upper -// bound of the bucket the quantile falls in). - -// A 0.5% geometric step keeps quantile error an order of magnitude below the -// smallest change worth acting on, while staying fixed-size and bounded for -// multi-hour soaks. The old 20-bucket ladder (…610, 1000, 2000, 5000…) could -// not resolve anything finer than a doubling: two identical runs would report -// p95 as 1000ms and 2000ms purely from which side of a boundary they landed. -// Keep the release SLO as an exact boundary so values at or below it cannot -// round up into a failing bucket. -var latBoundsMs = buildLatencyBounds() - -func buildLatencyBounds() []float64 { - const ( - first = 0.5 - last = 30000.0 - ratio = 1.005 - ) - bounds := make([]float64, 0, 2300) - for value := first; value < last; value *= ratio { - bounds = append(bounds, value) - } - // An explicit boundary at 1.5s keeps query latency legible around the point - // where a dashboard stops feeling interactive. Histogram resolution, not a - // threshold — nothing passes or fails here. - const queryLegibilityBoundMs = 1500 - bounds = append(bounds, queryLegibilityBoundMs, last) - sort.Float64s(bounds) - - unique := bounds[:0] - for _, bound := range bounds { - if len(unique) == 0 || bound != unique[len(unique)-1] { - unique = append(unique, bound) - } - } - return unique -} - -type histogram struct { - counts []atomic.Int64 // counts[i] = samples in (bounds[i-1], bounds[i]] - over atomic.Int64 // samples > last bound - n atomic.Int64 - sumMicros atomic.Int64 -} - -func newHistogram() *histogram { return &histogram{counts: make([]atomic.Int64, len(latBoundsMs))} } - -func (h *histogram) record(d time.Duration) { - h.n.Add(1) - h.sumMicros.Add(d.Microseconds()) - ms := float64(d) / float64(time.Millisecond) - index := sort.SearchFloat64s(latBoundsMs, ms) - if index < len(latBoundsMs) { - h.counts[index].Add(1) - return - } - h.over.Add(1) -} - -func (h *histogram) quantile(q float64) float64 { - total := h.n.Load() - if total == 0 { - return 0 - } - target := int64(math.Ceil(q * float64(total))) - var cum int64 - for i := range latBoundsMs { - cum += h.counts[i].Load() - if cum >= target { - return latBoundsMs[i] - } - } - return latBoundsMs[len(latBoundsMs)-1] // in the overflow bucket: report the cap as a floor -} - -func (h *histogram) snapshot() latencyReport { - n := h.n.Load() - mean := 0.0 - if n > 0 { - mean = float64(h.sumMicros.Load()) / float64(n) / 1000.0 - } - return latencyReport{ - Count: n, - MeanMs: round2(mean), - P50Ms: h.quantile(0.50), - P95Ms: h.quantile(0.95), - P99Ms: h.quantile(0.99), - Over30s: h.over.Load(), - } -} - -// ── Report ───────────────────────────────────────────────────────────────── - -type report struct { - Manifest runManifest `json:"manifest"` - Passed bool `json:"passed"` - Failures []string `json:"failures,omitempty"` - Endpoint string `json:"endpoint"` - // Elapsed wall clock, as opposed to the requested duration recorded in - // Manifest.Workload. Worker/service counts live in the manifest only. - DurationSec float64 `json:"duration_sec"` - TargetRate float64 `json:"target_rate"` - TracesSent int64 `json:"traces_sent"` - SpansSent int64 `json:"spans_sent"` - LogsSent int64 `json:"logs_sent"` - MetricsSent int64 `json:"metrics_sent"` - SendErrors int64 `json:"send_errors"` - AvgTracesPerSec float64 `json:"avg_traces_per_sec"` - ExportLatencyMs latencyReport `json:"export_latency_ms"` - QueriesRun int64 `json:"queries_run,omitempty"` - QueryErrors int64 `json:"query_errors,omitempty"` - QueryLatencyMs *latencyReport `json:"query_latency_ms,omitempty"` - QueryLatencyByOperation map[string]latencyReport `json:"query_latency_by_operation,omitempty"` - Server *serverReport `json:"server,omitempty"` - Capacity *capacityReport `json:"capacity,omitempty"` -} - -// capacityReport is what an adaptive run discovered about the machine. Absent -// from a fixed -rate run, which asserts a rate rather than discovering one. -type capacityReport struct { - Steps []rampStep `json:"steps"` - // SustainableTracesPerSec is the rate to quote: the lower of what the ramp - // found and what the confirmation pass actually held. - // - // They diverge when capacity is not stationary. Rollup and merge cost grows - // with the stored dataset, so a ramp can measure a machine whose capacity is - // falling underneath it and hand back a rate that no longer holds minutes - // later. Quoting the confirmed figure keeps the headline number one that was - // demonstrated for a full pass at the largest dataset of the run. - SustainableTracesPerSec float64 `json:"sustainable_traces_per_sec"` - // RampEstimateTracesPerSec is what the ramp concluded before confirmation. - RampEstimateTracesPerSec float64 `json:"ramp_estimate_traces_per_sec"` - // ConfirmedTracesPerSec is what the confirmation pass held. - ConfirmedTracesPerSec float64 `json:"confirmed_traces_per_sec"` - // SaturationTracesPerSec is the most throughput seen at any offered rate, - // including rates the server could not keep up with. - SaturationTracesPerSec float64 `json:"saturation_traces_per_sec"` - StopReason string `json:"stop_reason"` - // Driver-side context: a ramp is only as trustworthy as the machine that - // generated it, so record what was doing the sending. - Workers int `json:"driver_workers"` - DriverLogicalCPUs int `json:"driver_logical_cpus"` -} - -type latencyReport struct { - Count int64 `json:"count"` - MeanMs float64 `json:"mean_ms"` - P50Ms float64 `json:"p50_ms"` - P95Ms float64 `json:"p95_ms"` - P99Ms float64 `json:"p99_ms"` - Over30s int64 `json:"over_30s"` -} - -type serverReport struct { - BaselineAvailable bool `json:"baseline_available"` - ProcessStartTime float64 `json:"process_start_time_seconds"` - ProcessRestarted bool `json:"process_restarted"` - IngestRowsStart float64 `json:"ingest_rows_start"` - IngestRowsEnd float64 `json:"ingest_rows_end"` - IngestRowsDelta float64 `json:"ingest_rows_delta"` - RowsDroppedStart float64 `json:"rows_dropped_start"` - RowsDroppedEnd float64 `json:"rows_dropped_end"` - RowsDroppedDelta float64 `json:"rows_dropped_delta"` - ParquetFilesStart float64 `json:"parquet_files_start"` - ParquetFiles float64 `json:"parquet_files"` - ParquetFilesDelta float64 `json:"parquet_files_delta"` - ParquetSizeBytesStart float64 `json:"parquet_size_bytes_start"` - ParquetSizeBytes float64 `json:"parquet_size_bytes"` - ParquetSizeBytesDelta float64 `json:"parquet_size_bytes_delta"` - ParquetGrowthBytesPerSec float64 `json:"parquet_growth_bytes_per_sec"` - IngestQueueDepth float64 `json:"ingest_queue_depth"` - AvgRollupMs float64 `json:"avg_rollup_ms"` - AvgFlushMs float64 `json:"avg_flush_ms"` - AvgQueryMs float64 `json:"avg_query_ms"` - CPUSecondsStart float64 `json:"cpu_seconds_start"` - CPUSecondsEnd float64 `json:"cpu_seconds_end"` - CPUSecondsDelta float64 `json:"cpu_seconds_delta"` - CPUCores float64 `json:"cpu_cores"` - RSSBytes float64 `json:"rss_bytes"` - HeapAllocBytes float64 `json:"heap_alloc_bytes"` - AllocBytesStart float64 `json:"alloc_bytes_start"` - AllocBytesEnd float64 `json:"alloc_bytes_end"` - AllocBytesDelta float64 `json:"alloc_bytes_delta"` - AllocBytesPerSec float64 `json:"alloc_bytes_per_sec"` - GCPauseSecondsStart float64 `json:"gc_pause_seconds_start"` - GCPauseSecondsEnd float64 `json:"gc_pause_seconds_end"` - GCPauseSecondsDelta float64 `json:"gc_pause_seconds_delta"` - WriteGateWaitMs map[string]distributionReport `json:"write_gate_wait_ms,omitempty"` - WriteGateHoldMs map[string]distributionReport `json:"write_gate_hold_ms,omitempty"` - TelemetryOperations map[string]backgroundOperationReport `json:"telemetry_operations,omitempty"` - Rollups map[string]rollupReport `json:"rollups,omitempty"` -} - -func printReport(r report) { - fmt.Printf("\n── performance report ──────────────────────────────────────\n") - fmt.Printf("duration %.1fs (target %.0f traces/s → actual %.0f)\n", r.DurationSec, r.TargetRate, r.AvgTracesPerSec) - fmt.Printf("sent traces=%d spans=%d logs=%d metrics=%d\n", r.TracesSent, r.SpansSent, r.LogsSent, r.MetricsSent) - fmt.Printf("send errors %d\n", r.SendErrors) - if c := r.Capacity; c != nil { - fmt.Printf("capacity sustainable=%.0f/s (ramp estimate=%.0f/s, confirmed=%.0f/s, peak=%.0f/s)\n", - c.SustainableTracesPerSec, c.RampEstimateTracesPerSec, c.ConfirmedTracesPerSec, c.SaturationTracesPerSec) - // A confirmation well under the ramp's estimate is the signature of - // capacity falling as the dataset grows, not of a bad measurement. - if c.RampEstimateTracesPerSec > 0 && c.ConfirmedTracesPerSec < c.RampEstimateTracesPerSec*0.9 { - fmt.Printf(" note: confirmation held %.0f%% of the ramp estimate — capacity fell as the dataset grew\n", - 100*c.ConfirmedTracesPerSec/c.RampEstimateTracesPerSec) - } - } - l := r.ExportLatencyMs - fmt.Printf("export latency mean=%.1fms p50≈%.0f p95≈%.0f p99≈%.0f >30s=%d (n=%d)\n", - l.MeanMs, l.P50Ms, l.P95Ms, l.P99Ms, l.Over30s, l.Count) - if r.QueryLatencyMs != nil { - q := r.QueryLatencyMs - fmt.Printf("query latency mean=%.0fms p50≈%.0f p95≈%.0f p99≈%.0f (n=%d, errors=%d)\n", - q.MeanMs, q.P50Ms, q.P95Ms, q.P99Ms, q.Count, r.QueryErrors) - for _, operation := range []string{"overview", "topology", "performance", "trace", "logs"} { - if item, ok := r.QueryLatencyByOperation[operation]; ok { - fmt.Printf(" %-11s p50≈%-5.0f p95≈%-5.0f p99≈%-5.0f (n=%d)\n", - operation, item.P50Ms, item.P95Ms, item.P99Ms, item.Count) - } - } - } - if r.Server != nil { - s := r.Server - fmt.Printf("server (Δ over run):\n") - fmt.Printf(" rows accepted=%.0f dropped=%.0f\n", s.IngestRowsDelta, s.RowsDroppedDelta) - fmt.Printf(" parquet_files=%.0f parquet_size=%.1fMB ingest_queue_depth=%.0f\n", - s.ParquetFiles, s.ParquetSizeBytes/(1<<20), s.IngestQueueDepth) - fmt.Printf(" avg rollup=%.1fms flush=%.1fms query=%.1fms\n", s.AvgRollupMs, s.AvgFlushMs, s.AvgQueryMs) - fmt.Printf(" cpu=%.2f core(s) rss=%.1fMB alloc=%.1fMB/s parquet_growth=%.1fMB\n", - s.CPUCores, s.RSSBytes/(1<<20), s.AllocBytesPerSec/(1<<20), s.ParquetSizeBytesDelta/(1<<20)) - } - if r.Passed { - fmt.Printf("verdict PASS\n") - } else { - fmt.Printf("verdict FAIL (%s)\n", strings.Join(r.Failures, "; ")) - } - fmt.Printf("────────────────────────────────────────────────────────────\n") -} - -func writeReport(path string, r report) error { - b, err := json.MarshalIndent(r, "", " ") - if err != nil { - return err - } - return os.WriteFile(path, append(b, '\n'), 0o644) -} - -func round2(f float64) float64 { return math.Round(f*100) / 100 } diff --git a/cmd/bench/main_test.go b/cmd/bench/main_test.go deleted file mode 100644 index f9924eab8..000000000 --- a/cmd/bench/main_test.go +++ /dev/null @@ -1,124 +0,0 @@ -package main - -import ( - "context" - "math/rand/v2" - "net/http" - "net/http/httptest" - "reflect" - "testing" - "time" -) - -func TestQueryTargetAtRotatesEveryOperationAcrossEveryWindow(t *testing.T) { - windowsByOperation := make(map[string]map[string]bool) - for i := 0; i < 25; i++ { - target := queryTargetAt(i) - windows := windowsByOperation[target.operation] - if windows == nil { - windows = make(map[string]bool) - windowsByOperation[target.operation] = windows - } - for _, window := range queryWindows { - if target.path == "/api/observability/"+target.operation+"?window="+window+"&limit=100" { - windows[window] = true - } - } - } - - for operation, windows := range windowsByOperation { - if len(windows) != len(queryWindows) { - t.Errorf("%s exercised %d windows, want %d: %#v", operation, len(windows), len(queryWindows), windows) - } - } -} - -func TestScrapeMetricsSendsBearerToken(t *testing.T) { - server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - if got := r.Header.Get("Authorization"); got != "Bearer metrics-secret" { - http.Error(w, "unauthorized", http.StatusUnauthorized) - return - } - _, _ = w.Write([]byte("fanout_rows_total 42\n")) - })) - defer server.Close() - - metrics, err := scrapeMetrics(server.URL, "metrics-secret") - if err != nil { - t.Fatal(err) - } - if metrics.total("fanout_rows_total") != 42 { - t.Fatalf("metrics = %#v", metrics) - } -} - -func TestQueryRequestUsesAuthenticatedSession(t *testing.T) { - g := generator{cfg: config{querySessionCookie: "fanout_session=session-token"}} - req, err := g.newQueryRequest(context.Background(), "https://fanout.example.test/api/observability/overview") - if err != nil { - t.Fatalf("newQueryRequest: %v", err) - } - if got := req.Header.Get("Cookie"); got != "fanout_session=session-token" { - t.Fatalf("Cookie = %q, want authenticated session", got) - } -} - -func TestHistogramHasExactBoundaryAtQueryLegibilityPoint(t *testing.T) { - h := newHistogram() - for range 100 { - h.record(1499 * time.Millisecond) - } - - // 1.5s is a bucket edge so latency around the interactive threshold reports - // exactly rather than landing mid-bucket. Not a pass/fail threshold. - if got := h.snapshot().P95Ms; got != 1500 { - t.Fatalf("p95 = %vms, want exact 1500ms bucket boundary", got) - } -} - -func TestHistogramBoundsCanEnforceFivePercentGuardrail(t *testing.T) { - for index := 1; index < len(latBoundsMs); index++ { - previous := latBoundsMs[index-1] - current := latBoundsMs[index] - if ratio := current / previous; ratio > 1.0050001 { - t.Fatalf("bounds %vms -> %vms have ratio %.4f, want <= 1.005", previous, current, ratio) - } - } -} - -func TestHistogramRecordsWithBinarySearchAtExactBoundary(t *testing.T) { - h := newHistogram() - h.record(500 * time.Microsecond) - h.record(501 * time.Microsecond) - if got := h.n.Load(); got != 2 { - t.Fatalf("samples = %d, want 2", got) - } - if got := h.counts[0].Load(); got != 1 { - t.Fatalf("first bucket = %d, want 1", got) - } -} - -func TestSyntheticSequenceIsDeterministicPerWorkerSeed(t *testing.T) { - type sequence struct { - TraceID []byte - Namespace string - Route string - Attrs any - Status any - } - generate := func() sequence { - rng := rand.New(rand.NewPCG(17, 3)) - generator := generator{cfg: config{namespaces: 4, cardinality: 100, errorRate: 0.2}} - return sequence{ - TraceID: randBytes(rng, 16), - Namespace: generator.namespace(rng), - Route: generator.route(rng), - Attrs: generator.spanAttrs(rng), - Status: generator.status(rng), - } - } - - if first, second := generate(), generate(); !reflect.DeepEqual(first, second) { - t.Fatalf("same worker seed produced different sequences:\nfirst: %#v\nsecond: %#v", first, second) - } -} diff --git a/cmd/bench/manifest.go b/cmd/bench/manifest.go deleted file mode 100644 index ad801c49c..000000000 --- a/cmd/bench/manifest.go +++ /dev/null @@ -1,155 +0,0 @@ -package main - -import ( - "crypto/sha256" - "encoding/hex" - "encoding/json" - "os" - "runtime" - "runtime/debug" - "strconv" - "strings" - "time" -) - -// runManifest records what produced a report so a stored run.json is still -// readable months later. Everything here is either derived from the running -// process or copied from the flags that shape the workload — nothing describes -// the server under test beyond the version string the caller passes, because -// bench cannot verify server configuration. -type runManifest struct { - StartedAt time.Time `json:"started_at"` - FinishedAt time.Time `json:"finished_at"` - Build buildManifest `json:"build"` - Host hostManifest `json:"host"` - Workload workloadManifest `json:"workload"` - // WorkloadHash identifies the requested workload: two runs with the same hash - // asked for the same synthetic telemetry. It does not certify that they - // produced the same data — the event count depends on the rate actually - // achieved, which varies with the host. - WorkloadHash string `json:"workload_sha256"` -} - -type buildManifest struct { - FanoutVersion string `json:"fanout_version"` - BenchVersion string `json:"bench_version"` - GoVersion string `json:"go_version"` - GitCommit string `json:"git_commit"` - GitDirty bool `json:"git_dirty"` -} - -type hostManifest struct { - OS string `json:"os"` - Architecture string `json:"architecture"` - LogicalCPUs int `json:"logical_cpus"` - GOMAXPROCS int `json:"gomaxprocs"` - MemoryLimitBytes int64 `json:"memory_limit_bytes,omitempty"` -} - -type workloadManifest struct { - Seed uint64 `json:"seed"` - // Requested, not achieved: report.duration_sec carries elapsed wall clock. - RequestedDurationSec float64 `json:"requested_duration_sec"` - TargetRate float64 `json:"target_traces_per_sec"` - Workers int `json:"workers"` - Services int `json:"services"` - Namespaces int `json:"namespaces"` - Cardinality int `json:"attribute_cardinality"` - ErrorRate float64 `json:"error_rate"` - MessagingRatio float64 `json:"messaging_ratio"` - SendLogs bool `json:"send_logs"` - SendMetrics bool `json:"send_metrics"` - BackfillHours float64 `json:"backfill_hours"` - QueryWorkers int `json:"query_workers"` - TargetQueryRate float64 `json:"target_queries_per_sec"` - QueryOperations []string `json:"query_operations"` - QueryWindows []string `json:"query_windows"` -} - -func newRunManifest(cfg config, startedAt, finishedAt time.Time) runManifest { - workload := workloadManifest{ - Seed: cfg.seed, - RequestedDurationSec: round2(cfg.duration.Seconds()), - TargetRate: cfg.rate, - Workers: cfg.workers, - Services: cfg.services, - Namespaces: cfg.namespaces, - Cardinality: cfg.cardinality, - ErrorRate: cfg.errorRate, - MessagingRatio: cfg.msgRatio, - SendLogs: cfg.sendLogs, - SendMetrics: cfg.sendMetrics, - BackfillHours: cfg.backfillHours, - QueryWorkers: cfg.queryWorkers, - TargetQueryRate: cfg.queryRate, - QueryOperations: append([]string(nil), queryOperations...), - QueryWindows: append([]string(nil), queryWindows...), - } - hashInput, _ := json.Marshal(workload) // fixed value-only struct cannot fail - hash := sha256.Sum256(hashInput) - - build := currentBuildManifest() - build.FanoutVersion = strings.TrimSpace(cfg.fanoutVersion) - build.BenchVersion = version - - return runManifest{ - StartedAt: startedAt.UTC(), - FinishedAt: finishedAt.UTC(), - Build: build, - Host: hostManifest{ - OS: runtime.GOOS, - Architecture: runtime.GOARCH, - LogicalCPUs: runtime.NumCPU(), - GOMAXPROCS: runtime.GOMAXPROCS(0), - MemoryLimitBytes: detectMemoryLimit(), - }, - Workload: workload, - WorkloadHash: hex.EncodeToString(hash[:]), - } -} - -func currentBuildManifest() buildManifest { - manifest := buildManifest{ - GoVersion: runtime.Version(), - GitCommit: "unknown", - } - info, ok := debug.ReadBuildInfo() - if !ok { - return manifest - } - if info.GoVersion != "" { - manifest.GoVersion = info.GoVersion - } - for _, setting := range info.Settings { - switch setting.Key { - case "vcs.revision": - manifest.GitCommit = setting.Value - case "vcs.modified": - manifest.GitDirty = setting.Value == "true" - } - } - return manifest -} - -// detectMemoryLimit reports the cgroup memory ceiling so a containerized run -// records what it was actually allowed to use rather than the host total. -// Returns 0 when unlimited or unavailable (including on non-Linux). -func detectMemoryLimit() int64 { - for _, path := range []string{ - "/sys/fs/cgroup/memory.max", - "/sys/fs/cgroup/memory/memory.limit_in_bytes", - } { - value, err := os.ReadFile(path) - if err != nil { - continue - } - raw := strings.TrimSpace(string(value)) - if raw == "" || raw == "max" { - continue - } - if limit, err := strconv.ParseInt(raw, 10, 64); err == nil && limit > 0 { - return limit - } - } - return 0 -} diff --git a/cmd/bench/manifest_test.go b/cmd/bench/manifest_test.go deleted file mode 100644 index 741e8d09f..000000000 --- a/cmd/bench/manifest_test.go +++ /dev/null @@ -1,100 +0,0 @@ -package main - -import ( - "bytes" - "encoding/json" - "os" - "path/filepath" - "reflect" - "testing" - "time" -) - -func TestRunManifestIsDeterministicAndSecretFree(t *testing.T) { - startedAt := time.Date(2026, time.August, 1, 12, 0, 0, 0, time.FixedZone("test", -7*60*60)) - finishedAt := startedAt.Add(30 * time.Minute) - cfg := config{ - endpoint: "secret-ingest-host:4317", - token: "ingest-secret-sentinel", - metricsURL: "https://metrics-secret-host/-/metrics", - metricsToken: "metrics-secret-sentinel", - queryURL: "https://query-secret-host", - rate: 2500, - duration: 30 * time.Minute, - workers: 12, - services: 50, - namespaces: 3, - cardinality: 200, - errorRate: 0.05, - msgRatio: 0.2, - sendLogs: true, - sendMetrics: true, - backfillHours: 6, - queryWorkers: 4, - queryRate: 20, - seed: 42, - fanoutVersion: "fanout-v1", - } - - first := newRunManifest(cfg, startedAt, finishedAt) - if second := newRunManifest(cfg, startedAt, finishedAt); !reflect.DeepEqual(first, second) { - t.Fatalf("same inputs produced different manifests:\nfirst: %#v\nsecond: %#v", first, second) - } - if len(first.WorkloadHash) != 64 { - t.Fatalf("workload hash = %q, want 64 hex characters", first.WorkloadHash) - } - if first.StartedAt.Location() != time.UTC || first.FinishedAt.Location() != time.UTC { - t.Fatalf("manifest times are not UTC: %v %v", first.StartedAt, first.FinishedAt) - } - - // The manifest travels with shared evidence, so it must never carry the - // endpoints or credentials the run was pointed at. - encoded, err := json.Marshal(first) - if err != nil { - t.Fatal(err) - } - for _, secret := range []string{cfg.endpoint, cfg.token, cfg.metricsURL, cfg.metricsToken, cfg.queryURL} { - if bytes.Contains(encoded, []byte(secret)) { - t.Fatalf("manifest leaked %q: %s", secret, encoded) - } - } - - changed := cfg - changed.seed++ - if got := newRunManifest(changed, startedAt, finishedAt).WorkloadHash; got == first.WorkloadHash { - t.Fatalf("workload hash did not change with seed: %s", got) - } -} - -func TestWriteReportProducesValidNewlineTerminatedJSON(t *testing.T) { - report := report{ - Manifest: runManifest{ - StartedAt: time.Date(2026, time.August, 1, 19, 0, 0, 0, time.UTC), - FinishedAt: time.Date(2026, time.August, 1, 19, 30, 0, 0, time.UTC), - }, - Endpoint: "localhost:4317", - DurationSec: 1800, - TargetRate: 1000, - TracesSent: 1, - AvgTracesPerSec: 1000, - Server: &serverReport{ - WriteGateWaitMs: map[string]distributionReport{ - "rollup_service": {Count: 2, P95Ms: 50}, - "ingest_spans": {Count: 5, P95Ms: 10}, - }, - }, - } - path := filepath.Join(t.TempDir(), "run.json") - if err := writeReport(path, report); err != nil { - t.Fatal(err) - } - written, err := os.ReadFile(path) - if err != nil { - t.Fatal(err) - } - // Trailing newline matters: the report is appended to evidence logs and read - // back line-wise by the harness scripts. - if !json.Valid(written) || len(written) == 0 || written[len(written)-1] != '\n' { - t.Fatalf("report is not valid newline-terminated JSON: %q", written) - } -} diff --git a/cmd/bench/metrics_report.go b/cmd/bench/metrics_report.go deleted file mode 100644 index 936ecae1b..000000000 --- a/cmd/bench/metrics_report.go +++ /dev/null @@ -1,515 +0,0 @@ -package main - -import ( - "bufio" - "context" - "fmt" - "io" - "math" - "net/http" - "sort" - "strconv" - "strings" - "time" -) - -type metricSample struct { - Name string - Labels map[string]string - Value float64 -} - -// metricSnapshot retains both aggregate totals for legacy report fields and -// individual bounded-label series for operation-level distributions. -type metricSnapshot struct { - Totals map[string]float64 - Samples []metricSample -} - -type distributionReport struct { - Count int64 `json:"count"` - MeanMs float64 `json:"mean_ms"` - P50Ms float64 `json:"p50_ms"` - P95Ms float64 `json:"p95_ms"` - P99Ms float64 `json:"p99_ms"` -} - -type backgroundOperationReport struct { - DurationMs distributionReport `json:"duration_ms"` - Outcomes map[string]float64 `json:"outcomes,omitempty"` -} - -type rollupReport struct { - Enabled bool `json:"enabled"` - WatermarkTimestampSeconds float64 `json:"watermark_timestamp_seconds"` - SourceTimestampSeconds float64 `json:"source_timestamp_seconds"` - LagSeconds float64 `json:"lag_seconds"` - BacklogChunks float64 `json:"backlog_chunks"` - RowsDelta float64 `json:"rows_delta"` - DurationMs distributionReport `json:"duration_ms"` - Outcomes map[string]float64 `json:"outcomes,omitempty"` -} - -// scrapeTimeout bounds both the baseline and final scrape. An endpoint that -// accepts the connection and then stops responding would otherwise hang the -// benchmark forever — on a remote runner that surfaces only as the harness -// deadline expiring with no report at all. -const scrapeTimeout = 30 * time.Second - -// scrapeMetrics fetches a Prometheus text endpoint without retaining the URL -// or bearer token in benchmark evidence. -func scrapeMetrics(url, token string) (*metricSnapshot, error) { - ctx, cancel := context.WithTimeout(context.Background(), scrapeTimeout) - defer cancel() - req, err := http.NewRequestWithContext(ctx, http.MethodGet, url, nil) - if err != nil { - return nil, err - } - if token != "" { - req.Header.Set("Authorization", "Bearer "+token) - } - resp, err := http.DefaultClient.Do(req) - if err != nil { - return nil, err - } - defer resp.Body.Close() - if resp.StatusCode != http.StatusOK { - return nil, fmt.Errorf("status %d", resp.StatusCode) - } - return parseMetricSnapshot(resp.Body) -} - -func parseMetricSnapshot(r io.Reader) (*metricSnapshot, error) { - snapshot := &metricSnapshot{Totals: make(map[string]float64)} - scanner := bufio.NewScanner(r) - scanner.Buffer(make([]byte, 0, 64*1024), 1<<20) - for lineNumber := 1; scanner.Scan(); lineNumber++ { - line := strings.TrimSpace(scanner.Text()) - if line == "" || strings.HasPrefix(line, "#") { - continue - } - head, rawValue, ok := splitMetricLine(line) - if !ok { - return nil, fmt.Errorf("metrics line %d: missing sample value", lineNumber) - } - name, labels, err := parseMetricHead(head) - if err != nil { - return nil, fmt.Errorf("metrics line %d: %w", lineNumber, err) - } - value, err := strconv.ParseFloat(rawValue, 64) - if err != nil { - return nil, fmt.Errorf("metrics line %d: parse value: %w", lineNumber, err) - } - snapshot.Totals[name] += value - snapshot.Samples = append(snapshot.Samples, metricSample{Name: name, Labels: labels, Value: value}) - } - if err := scanner.Err(); err != nil { - return nil, err - } - return snapshot, nil -} - -func splitMetricLine(line string) (head, value string, ok bool) { - depth := 0 - quoted := false - escaped := false - for i := 0; i < len(line); i++ { - ch := line[i] - if quoted { - if escaped { - escaped = false - continue - } - if ch == '\\' { - escaped = true - continue - } - if ch == '"' { - quoted = false - } - continue - } - switch ch { - case '"': - quoted = true - case '{': - depth++ - case '}': - depth-- - case ' ', '\t': - if depth == 0 { - fields := strings.Fields(line[i:]) - if len(fields) == 0 { - return "", "", false - } - return line[:i], fields[0], true - } - } - } - return "", "", false -} - -func parseMetricHead(head string) (string, map[string]string, error) { - open := strings.IndexByte(head, '{') - if open < 0 { - if head == "" { - return "", nil, fmt.Errorf("empty metric name") - } - return head, nil, nil - } - if open == 0 || !strings.HasSuffix(head, "}") { - return "", nil, fmt.Errorf("malformed metric labels") - } - labels, err := parsePrometheusLabels(head[open+1 : len(head)-1]) - if err != nil { - return "", nil, err - } - return head[:open], labels, nil -} - -func parsePrometheusLabels(raw string) (map[string]string, error) { - labels := make(map[string]string) - for i := 0; i < len(raw); { - for i < len(raw) && (raw[i] == ' ' || raw[i] == '\t' || raw[i] == ',') { - i++ - } - if i == len(raw) { - break - } - keyStart := i - for i < len(raw) && raw[i] != '=' { - i++ - } - if i == len(raw) { - return nil, fmt.Errorf("malformed label assignment") - } - key := strings.TrimSpace(raw[keyStart:i]) - i++ - for i < len(raw) && (raw[i] == ' ' || raw[i] == '\t') { - i++ - } - if key == "" || i == len(raw) || raw[i] != '"' { - return nil, fmt.Errorf("malformed label value") - } - valueStart := i - i++ - escaped := false - for i < len(raw) { - if escaped { - escaped = false - i++ - continue - } - if raw[i] == '\\' { - escaped = true - i++ - continue - } - if raw[i] == '"' { - break - } - i++ - } - if i == len(raw) { - return nil, fmt.Errorf("unterminated label value") - } - value, err := strconv.Unquote(raw[valueStart : i+1]) - if err != nil { - return nil, fmt.Errorf("parse label %q: %w", key, err) - } - labels[key] = value - i++ - for i < len(raw) && (raw[i] == ' ' || raw[i] == '\t') { - i++ - } - if i < len(raw) && raw[i] != ',' { - return nil, fmt.Errorf("malformed separator after label %q", key) - } - } - return labels, nil -} - -func (s *metricSnapshot) total(name string) float64 { - if s == nil { - return 0 - } - return s.Totals[name] -} - -func (s *metricSnapshot) filteredTotal(name string, labels map[string]string) float64 { - if s == nil { - return 0 - } - if len(labels) == 0 { - return s.total(name) - } - var total float64 - for _, sample := range s.Samples { - if sample.Name == name && hasLabels(sample.Labels, labels) { - total += sample.Value - } - } - return total -} - -func (s *metricSnapshot) labelValues(name, label string, filters map[string]string) []string { - if s == nil { - return nil - } - values := make(map[string]struct{}) - for _, sample := range s.Samples { - if sample.Name != name || !hasLabels(sample.Labels, filters) { - continue - } - if value, ok := sample.Labels[label]; ok { - values[value] = struct{}{} - } - } - return sortedMapKeys(values) -} - -func hasLabels(actual, required map[string]string) bool { - for key, value := range required { - if actual[key] != value { - return false - } - } - return true -} - -func serverDelta(base, final *metricSnapshot, durationSeconds float64) *serverReport { - if final == nil { - return nil - } - baselineAvailable := base != nil - if base == nil { - base = &metricSnapshot{Totals: make(map[string]float64)} - } - delta := func(name string) float64 { return final.total(name) - base.total(name) } - rate := func(value float64) float64 { - if durationSeconds <= 0 { - return 0 - } - return round2(value / durationSeconds) - } - parquetFilesStart := base.total("fanout_parquet_files") - parquetSizeStart := base.total("fanout_parquet_size_bytes") - cpuSeconds := delta("process_cpu_seconds_total") - allocBytes := delta("go_memstats_alloc_bytes_total") - // process_start_time_seconds is constant for the life of a process, so a - // change between the two scrapes is a restart — detected even when the new - // process out-counts the old baseline and every delta stays positive. - startTimeBefore := base.total("process_start_time_seconds") - startTimeAfter := final.total("process_start_time_seconds") - return &serverReport{ - BaselineAvailable: baselineAvailable, - ProcessStartTime: startTimeAfter, - ProcessRestarted: baselineAvailable && startTimeBefore > 0 && startTimeAfter != startTimeBefore, - IngestRowsStart: base.total("fanout_ingest_rows_total"), - IngestRowsEnd: final.total("fanout_ingest_rows_total"), - IngestRowsDelta: delta("fanout_ingest_rows_total"), - RowsDroppedStart: base.total("fanout_rows_dropped_total"), - RowsDroppedEnd: final.total("fanout_rows_dropped_total"), - RowsDroppedDelta: delta("fanout_rows_dropped_total"), - ParquetFilesStart: parquetFilesStart, - ParquetFiles: final.total("fanout_parquet_files"), - ParquetFilesDelta: final.total("fanout_parquet_files") - parquetFilesStart, - ParquetSizeBytesStart: parquetSizeStart, - ParquetSizeBytes: final.total("fanout_parquet_size_bytes"), - ParquetSizeBytesDelta: final.total("fanout_parquet_size_bytes") - parquetSizeStart, - ParquetGrowthBytesPerSec: rate(final.total("fanout_parquet_size_bytes") - parquetSizeStart), - IngestQueueDepth: final.total("fanout_ingest_queue_depth"), - AvgRollupMs: averageDurationMs(base, final, "fanout_rollup_duration_seconds"), - AvgFlushMs: averageDurationMs(base, final, "fanout_flush_duration_seconds"), - AvgQueryMs: averageDurationMs(base, final, "fanout_query_duration_seconds"), - CPUSecondsStart: round4(base.total("process_cpu_seconds_total")), - CPUSecondsEnd: round4(final.total("process_cpu_seconds_total")), - CPUSecondsDelta: round4(cpuSeconds), - CPUCores: perSecond(cpuSeconds, durationSeconds), - RSSBytes: final.total("process_resident_memory_bytes"), - HeapAllocBytes: final.total("go_memstats_heap_alloc_bytes"), - AllocBytesStart: base.total("go_memstats_alloc_bytes_total"), - AllocBytesEnd: final.total("go_memstats_alloc_bytes_total"), - AllocBytesDelta: allocBytes, - AllocBytesPerSec: rate(allocBytes), - GCPauseSecondsStart: round4(base.total("go_gc_duration_seconds_sum")), - GCPauseSecondsEnd: round4(final.total("go_gc_duration_seconds_sum")), - GCPauseSecondsDelta: round4(delta("go_gc_duration_seconds_sum")), - WriteGateWaitMs: histogramReports(base, final, "fanout_write_gate_wait_seconds", "operation"), - WriteGateHoldMs: histogramReports(base, final, "fanout_write_gate_hold_seconds", "operation"), - TelemetryOperations: backgroundReports(base, final), - Rollups: rollupReports(base, final), - } -} - -func averageDurationMs(base, final *metricSnapshot, name string) float64 { - count := final.total(name+"_count") - base.total(name+"_count") - if count <= 0 { - return 0 - } - sum := final.total(name+"_sum") - base.total(name+"_sum") - return round2(sum / count * 1000) -} - -func perSecond(value, durationSeconds float64) float64 { - if durationSeconds <= 0 { - return 0 - } - return round4(value / durationSeconds) -} - -func histogramReports(base, final *metricSnapshot, name, groupLabel string) map[string]distributionReport { - groups := unionStrings( - base.labelValues(name+"_count", groupLabel, nil), - final.labelValues(name+"_count", groupLabel, nil), - ) - if len(groups) == 0 { - return nil - } - reports := make(map[string]distributionReport, len(groups)) - for _, group := range groups { - reports[group] = histogramDelta(base, final, name, map[string]string{groupLabel: group}) - } - return reports -} - -func histogramDelta(base, final *metricSnapshot, name string, filters map[string]string) distributionReport { - countValue := final.filteredTotal(name+"_count", filters) - base.filteredTotal(name+"_count", filters) - if countValue <= 0 { - return distributionReport{} - } - count := int64(math.Round(countValue)) - sum := final.filteredTotal(name+"_sum", filters) - base.filteredTotal(name+"_sum", filters) - bounds := unionStrings( - base.labelValues(name+"_bucket", "le", filters), - final.labelValues(name+"_bucket", "le", filters), - ) - type histogramBound struct { - raw string - value float64 - } - finiteBounds := make([]histogramBound, 0, len(bounds)) - for _, raw := range bounds { - if raw == "+Inf" || raw == "Inf" { - continue - } - bound, err := strconv.ParseFloat(raw, 64) - if err == nil { - finiteBounds = append(finiteBounds, histogramBound{raw: raw, value: bound}) - } - } - sort.Slice(finiteBounds, func(i, j int) bool { return finiteBounds[i].value < finiteBounds[j].value }) - quantile := func(q float64) float64 { - if len(finiteBounds) == 0 { - return 0 - } - target := math.Ceil(q * countValue) - for _, bound := range finiteBounds { - bucketFilters := copyLabels(filters) - bucketFilters["le"] = bound.raw - cumulative := final.filteredTotal(name+"_bucket", bucketFilters) - base.filteredTotal(name+"_bucket", bucketFilters) - if cumulative >= target { - return round2(bound.value * 1000) - } - } - return round2(finiteBounds[len(finiteBounds)-1].value * 1000) - } - return distributionReport{ - Count: count, - MeanMs: round2(sum / countValue * 1000), - P50Ms: quantile(0.50), - P95Ms: quantile(0.95), - P99Ms: quantile(0.99), - } -} - -func backgroundReports(base, final *metricSnapshot) map[string]backgroundOperationReport { - operations := unionStrings( - base.labelValues("fanout_telemetry_operation_total", "operation", nil), - final.labelValues("fanout_telemetry_operation_total", "operation", nil), - base.labelValues("fanout_telemetry_operation_duration_seconds_count", "operation", nil), - final.labelValues("fanout_telemetry_operation_duration_seconds_count", "operation", nil), - ) - if len(operations) == 0 { - return nil - } - reports := make(map[string]backgroundOperationReport, len(operations)) - for _, operation := range operations { - filters := map[string]string{"operation": operation} - reports[operation] = backgroundOperationReport{ - DurationMs: histogramDelta(base, final, "fanout_telemetry_operation_duration_seconds", filters), - Outcomes: counterOutcomes(base, final, "fanout_telemetry_operation_total", "result", filters), - } - } - return reports -} - -func rollupReports(base, final *metricSnapshot) map[string]rollupReport { - rollups := unionStrings( - base.labelValues("fanout_rollup_enabled", "rollup", nil), - final.labelValues("fanout_rollup_enabled", "rollup", nil), - base.labelValues("fanout_rollup_component_total", "rollup", nil), - final.labelValues("fanout_rollup_component_total", "rollup", nil), - ) - if len(rollups) == 0 { - return nil - } - reports := make(map[string]rollupReport, len(rollups)) - for _, rollup := range rollups { - filters := map[string]string{"rollup": rollup} - reports[rollup] = rollupReport{ - Enabled: final.filteredTotal("fanout_rollup_enabled", filters) > 0, - WatermarkTimestampSeconds: final.filteredTotal("fanout_rollup_watermark_timestamp_seconds", filters), - SourceTimestampSeconds: final.filteredTotal("fanout_rollup_source_timestamp_seconds", filters), - LagSeconds: final.filteredTotal("fanout_rollup_lag_seconds", filters), - BacklogChunks: final.filteredTotal("fanout_rollup_backlog_chunks", filters), - RowsDelta: final.filteredTotal("fanout_rollup_component_rows_total", filters) - base.filteredTotal("fanout_rollup_component_rows_total", filters), - DurationMs: histogramDelta(base, final, "fanout_rollup_component_duration_seconds", filters), - Outcomes: counterOutcomes(base, final, "fanout_rollup_component_total", "result", filters), - } - } - return reports -} - -func counterOutcomes(base, final *metricSnapshot, name, outcomeLabel string, filters map[string]string) map[string]float64 { - outcomes := unionStrings( - base.labelValues(name, outcomeLabel, filters), - final.labelValues(name, outcomeLabel, filters), - ) - if len(outcomes) == 0 { - return nil - } - deltas := make(map[string]float64, len(outcomes)) - for _, outcome := range outcomes { - outcomeFilters := copyLabels(filters) - outcomeFilters[outcomeLabel] = outcome - deltas[outcome] = final.filteredTotal(name, outcomeFilters) - base.filteredTotal(name, outcomeFilters) - } - return deltas -} - -func copyLabels(labels map[string]string) map[string]string { - copy := make(map[string]string, len(labels)+1) - for key, value := range labels { - copy[key] = value - } - return copy -} - -func unionStrings(groups ...[]string) []string { - values := make(map[string]struct{}) - for _, group := range groups { - for _, value := range group { - values[value] = struct{}{} - } - } - return sortedMapKeys(values) -} - -func round4(value float64) float64 { - value = math.Round(value*10000) / 10000 - if value == 0 { - return 0 - } - return value -} diff --git a/cmd/bench/metrics_report_test.go b/cmd/bench/metrics_report_test.go deleted file mode 100644 index 3074dd946..000000000 --- a/cmd/bench/metrics_report_test.go +++ /dev/null @@ -1,219 +0,0 @@ -package main - -import ( - "math" - "strings" - "testing" -) - -func TestParseMetricSnapshotPreservesLabelsAndTotals(t *testing.T) { - snapshot := mustMetricSnapshot(t, ` -# HELP fanout_test_total test metric -fanout_test_total{operation="compaction",detail="quoted\" value\\path"} 2 -fanout_test_total{operation="maintenance",detail="plain"} 3 -`) - - if got := snapshot.total("fanout_test_total"); got != 5 { - t.Fatalf("total = %v, want 5", got) - } - if got := snapshot.filteredTotal("fanout_test_total", map[string]string{"operation": "compaction"}); got != 2 { - t.Fatalf("merge total = %v, want 2", got) - } - if got := snapshot.Samples[0].Labels["detail"]; got != "quoted\" value\\path" { - t.Fatalf("decoded label = %q", got) - } -} - -func TestServerDeltaReportsOperationAndRuntimeDistributions(t *testing.T) { - base := mustMetricSnapshot(t, ` -fanout_ingest_rows_total{signal="spans"} 10 -fanout_ingest_rows_total{signal="logs"} 5 -fanout_rows_dropped_total{signal="spans"} 0 -fanout_parquet_files{signal="spans"} 2 -fanout_parquet_files{signal="logs"} 1 -fanout_parquet_size_bytes{signal="spans"} 60 -fanout_parquet_size_bytes{signal="logs"} 40 -fanout_ingest_queue_depth{signal="spans"} 2 -fanout_ingest_queue_depth{signal="logs"} 1 -process_cpu_seconds_total 10 -process_resident_memory_bytes 400 -go_memstats_heap_alloc_bytes 100 -go_memstats_alloc_bytes_total 1000 -go_gc_duration_seconds_sum 0.1 -fanout_rollup_duration_seconds_sum 1 -fanout_rollup_duration_seconds_count 2 -fanout_flush_duration_seconds_sum{signal="spans"} 0.5 -fanout_flush_duration_seconds_count{signal="spans"} 5 -fanout_query_duration_seconds_sum{endpoint="overview"} 2 -fanout_query_duration_seconds_count{endpoint="overview"} 4 -fanout_write_gate_wait_seconds_bucket{operation="ingest_spans",le="0.001"} 1 -fanout_write_gate_wait_seconds_bucket{operation="ingest_spans",le="0.010"} 2 -fanout_write_gate_wait_seconds_bucket{operation="ingest_spans",le="0.100"} 2 -fanout_write_gate_wait_seconds_bucket{operation="ingest_spans",le="+Inf"} 2 -fanout_write_gate_wait_seconds_sum{operation="ingest_spans"} 0.012 -fanout_write_gate_wait_seconds_count{operation="ingest_spans"} 2 -fanout_write_gate_hold_seconds_bucket{operation="ingest_spans",le="0.010"} 1 -fanout_write_gate_hold_seconds_bucket{operation="ingest_spans",le="0.100"} 1 -fanout_write_gate_hold_seconds_bucket{operation="ingest_spans",le="+Inf"} 1 -fanout_write_gate_hold_seconds_sum{operation="ingest_spans"} 0.005 -fanout_write_gate_hold_seconds_count{operation="ingest_spans"} 1 -fanout_telemetry_operation_duration_seconds_bucket{operation="compaction",le="0.010"} 1 -fanout_telemetry_operation_duration_seconds_bucket{operation="compaction",le="0.100"} 1 -fanout_telemetry_operation_duration_seconds_bucket{operation="compaction",le="1.000"} 1 -fanout_telemetry_operation_duration_seconds_bucket{operation="compaction",le="+Inf"} 1 -fanout_telemetry_operation_duration_seconds_sum{operation="compaction"} 0.005 -fanout_telemetry_operation_duration_seconds_count{operation="compaction"} 1 -fanout_telemetry_operation_total{operation="compaction",result="success"} 1 -fanout_telemetry_operation_total{operation="compaction",result="throttled"} 2 -fanout_rollup_enabled{rollup="service"} 1 -fanout_rollup_watermark_timestamp_seconds{rollup="service"} 100 -fanout_rollup_source_timestamp_seconds{rollup="service"} 105 -fanout_rollup_lag_seconds{rollup="service"} 5 -fanout_rollup_backlog_chunks{rollup="service"} 1 -fanout_rollup_component_rows_total{rollup="service"} 10 -fanout_rollup_component_duration_seconds_bucket{rollup="service",le="0.010"} 1 -fanout_rollup_component_duration_seconds_bucket{rollup="service",le="0.100"} 1 -fanout_rollup_component_duration_seconds_bucket{rollup="service",le="+Inf"} 1 -fanout_rollup_component_duration_seconds_sum{rollup="service"} 0.008 -fanout_rollup_component_duration_seconds_count{rollup="service"} 1 -fanout_rollup_component_total{rollup="service",result="success"} 1 -`) - final := mustMetricSnapshot(t, ` -fanout_ingest_rows_total{signal="spans"} 30 -fanout_ingest_rows_total{signal="logs"} 15 -fanout_rows_dropped_total{signal="spans"} 0 -fanout_parquet_files{signal="spans"} 3 -fanout_parquet_files{signal="logs"} 2 -fanout_parquet_size_bytes{signal="spans"} 100 -fanout_parquet_size_bytes{signal="logs"} 80 -fanout_ingest_queue_depth{signal="spans"} 0 -fanout_ingest_queue_depth{signal="logs"} 1 -process_cpu_seconds_total 14 -process_resident_memory_bytes 512 -go_memstats_heap_alloc_bytes 256 -go_memstats_alloc_bytes_total 2600 -go_gc_duration_seconds_sum 0.3 -fanout_rollup_duration_seconds_sum 2.5 -fanout_rollup_duration_seconds_count 4 -fanout_flush_duration_seconds_sum{signal="spans"} 1.1 -fanout_flush_duration_seconds_count{signal="spans"} 8 -fanout_query_duration_seconds_sum{endpoint="overview"} 3.5 -fanout_query_duration_seconds_count{endpoint="overview"} 7 -fanout_write_gate_wait_seconds_bucket{operation="ingest_spans",le="0.001"} 2 -fanout_write_gate_wait_seconds_bucket{operation="ingest_spans",le="0.010"} 4 -fanout_write_gate_wait_seconds_bucket{operation="ingest_spans",le="0.100"} 5 -fanout_write_gate_wait_seconds_bucket{operation="ingest_spans",le="+Inf"} 5 -fanout_write_gate_wait_seconds_sum{operation="ingest_spans"} 0.043 -fanout_write_gate_wait_seconds_count{operation="ingest_spans"} 5 -fanout_write_gate_hold_seconds_bucket{operation="ingest_spans",le="0.010"} 2 -fanout_write_gate_hold_seconds_bucket{operation="ingest_spans",le="0.100"} 3 -fanout_write_gate_hold_seconds_bucket{operation="ingest_spans",le="+Inf"} 3 -fanout_write_gate_hold_seconds_sum{operation="ingest_spans"} 0.035 -fanout_write_gate_hold_seconds_count{operation="ingest_spans"} 3 -fanout_telemetry_operation_duration_seconds_bucket{operation="compaction",le="0.010"} 1 -fanout_telemetry_operation_duration_seconds_bucket{operation="compaction",le="0.100"} 2 -fanout_telemetry_operation_duration_seconds_bucket{operation="compaction",le="1.000"} 3 -fanout_telemetry_operation_duration_seconds_bucket{operation="compaction",le="+Inf"} 3 -fanout_telemetry_operation_duration_seconds_sum{operation="compaction"} 0.125 -fanout_telemetry_operation_duration_seconds_count{operation="compaction"} 3 -fanout_telemetry_operation_total{operation="compaction",result="success"} 3 -fanout_telemetry_operation_total{operation="compaction",result="throttled"} 5 -fanout_telemetry_operation_total{operation="compaction",result="error"} 1 -fanout_rollup_enabled{rollup="service"} 1 -fanout_rollup_watermark_timestamp_seconds{rollup="service"} 200 -fanout_rollup_source_timestamp_seconds{rollup="service"} 212 -fanout_rollup_lag_seconds{rollup="service"} 12 -fanout_rollup_backlog_chunks{rollup="service"} 2 -fanout_rollup_component_rows_total{rollup="service"} 25 -fanout_rollup_component_duration_seconds_bucket{rollup="service",le="0.010"} 1 -fanout_rollup_component_duration_seconds_bucket{rollup="service",le="0.100"} 3 -fanout_rollup_component_duration_seconds_bucket{rollup="service",le="+Inf"} 3 -fanout_rollup_component_duration_seconds_sum{rollup="service"} 0.108 -fanout_rollup_component_duration_seconds_count{rollup="service"} 3 -fanout_rollup_component_total{rollup="service",result="success"} 2 -fanout_rollup_component_total{rollup="service",result="noop"} 2 -`) - - report := serverDelta(base, final, 8) - if report == nil || !report.BaselineAvailable { - t.Fatalf("server report = %#v", report) - } - assertFloat(t, "ingest rows start", report.IngestRowsStart, 15) - assertFloat(t, "ingest rows end", report.IngestRowsEnd, 45) - assertFloat(t, "ingest rows", report.IngestRowsDelta, 30) - assertFloat(t, "Parquet files delta", report.ParquetFilesDelta, 2) - assertFloat(t, "Parquet size delta", report.ParquetSizeBytesDelta, 80) - assertFloat(t, "Parquet growth rate", report.ParquetGrowthBytesPerSec, 10) - assertFloat(t, "average rollup", report.AvgRollupMs, 750) - assertFloat(t, "average flush", report.AvgFlushMs, 200) - assertFloat(t, "average query", report.AvgQueryMs, 500) - assertFloat(t, "cpu seconds start", report.CPUSecondsStart, 10) - assertFloat(t, "cpu seconds end", report.CPUSecondsEnd, 14) - assertFloat(t, "cpu seconds", report.CPUSecondsDelta, 4) - assertFloat(t, "cpu cores", report.CPUCores, 0.5) - assertFloat(t, "rss", report.RSSBytes, 512) - assertFloat(t, "heap", report.HeapAllocBytes, 256) - assertFloat(t, "allocation start", report.AllocBytesStart, 1000) - assertFloat(t, "allocation end", report.AllocBytesEnd, 2600) - assertFloat(t, "allocation delta", report.AllocBytesDelta, 1600) - assertFloat(t, "allocation rate", report.AllocBytesPerSec, 200) - assertFloat(t, "gc pause start", report.GCPauseSecondsStart, 0.1) - assertFloat(t, "gc pause end", report.GCPauseSecondsEnd, 0.3) - assertFloat(t, "gc pause", report.GCPauseSecondsDelta, 0.2) - - wait := report.WriteGateWaitMs["ingest_spans"] - if wait.Count != 3 { - t.Fatalf("wait count = %d, want 3", wait.Count) - } - assertFloat(t, "wait mean", wait.MeanMs, 10.33) - assertFloat(t, "wait p50", wait.P50Ms, 10) - assertFloat(t, "wait p95", wait.P95Ms, 100) - assertFloat(t, "hold mean", report.WriteGateHoldMs["ingest_spans"].MeanMs, 15) - - merge := report.TelemetryOperations["compaction"] - if merge.DurationMs.Count != 2 { - t.Fatalf("merge duration count = %d, want 2", merge.DurationMs.Count) - } - assertFloat(t, "merge duration mean", merge.DurationMs.MeanMs, 60) - assertFloat(t, "merge p95", merge.DurationMs.P95Ms, 1000) - assertFloat(t, "merge successes", merge.Outcomes["success"], 2) - assertFloat(t, "merge throttles", merge.Outcomes["throttled"], 3) - assertFloat(t, "merge errors", merge.Outcomes["error"], 1) - - service := report.Rollups["service"] - if !service.Enabled || service.DurationMs.Count != 2 { - t.Fatalf("service rollup = %#v", service) - } - assertFloat(t, "rollup watermark", service.WatermarkTimestampSeconds, 200) - assertFloat(t, "rollup source", service.SourceTimestampSeconds, 212) - assertFloat(t, "rollup lag", service.LagSeconds, 12) - assertFloat(t, "rollup backlog", service.BacklogChunks, 2) - assertFloat(t, "rollup rows", service.RowsDelta, 15) - assertFloat(t, "rollup duration", service.DurationMs.MeanMs, 50) - assertFloat(t, "rollup success", service.Outcomes["success"], 1) - assertFloat(t, "rollup noop", service.Outcomes["noop"], 2) -} - -func TestServerDeltaWithoutBaselineIsExplicit(t *testing.T) { - final := mustMetricSnapshot(t, "process_cpu_seconds_total 2\n") - report := serverDelta(nil, final, 10) - if report == nil || report.BaselineAvailable { - t.Fatalf("server report = %#v", report) - } -} - -func mustMetricSnapshot(t *testing.T, text string) *metricSnapshot { - t.Helper() - snapshot, err := parseMetricSnapshot(strings.NewReader(text)) - if err != nil { - t.Fatal(err) - } - return snapshot -} - -func assertFloat(t *testing.T, name string, got, want float64) { - t.Helper() - if math.Abs(got-want) > 0.0001 { - t.Fatalf("%s = %v, want %v", name, got, want) - } -} diff --git a/cmd/bench/verdict.go b/cmd/bench/verdict.go deleted file mode 100644 index 0f069f27f..000000000 --- a/cmd/bench/verdict.go +++ /dev/null @@ -1,104 +0,0 @@ -package main - -import ( - "fmt" - "sort" -) - -func sortedMapKeys[V any](values map[string]V) []string { - keys := make([]string, 0, len(values)) - for key := range values { - keys = append(keys, key) - } - sort.Strings(keys) - return keys -} - -// Latency is judged against -max-export-p95-ms / -max-query-p95-ms when the -// caller sets them, and not otherwise. -// -// There is deliberately no built-in latency SLO. A fixed millisecond threshold -// encodes the machine it was calibrated on: the same binary that clears it on -// four cores fails it on two, and the run reports a defect where there is only -// a smaller computer. Capacity is reported instead, and a gate that wants a -// threshold passes one in. -// -// Everything below this line stays a hard failure, because it makes the numbers -// themselves untrustworthy rather than merely unflattering: a run that ingested -// nothing, dropped rows, lost its baseline, or straddled a server restart reads -// as a clean p95-of-survivors if it is allowed to pass. - -// evaluateReport lists everything that makes a run untrustworthy or failing. -// A run that ingested nothing, dropped rows, lost its metrics baseline, or saw -// the server restart mid-run must FAIL loudly: otherwise p95-of-survivors reads -// a healthy number and a broken run looks like a clean pass. -func evaluateReport(cfg config, report report, infrastructureFailures []string) []string { - fails := append([]string(nil), infrastructureFailures...) - - if report.ExportLatencyMs.Count == 0 { - fails = append(fails, "no OTLP exports succeeded") - } - if attempted := report.ExportLatencyMs.Count + report.SendErrors; report.SendErrors > 0 && attempted > 0 { - if rate := float64(report.SendErrors) / float64(attempted); rate > maxSendErrorRate { - fails = append(fails, fmt.Sprintf("send error rate %.3f%% (%d/%d) > %.1f%%", - rate*100, report.SendErrors, attempted, maxSendErrorRate*100)) - } - } - if cfg.queryURL != "" && cfg.queryWorkers > 0 && report.QueriesRun == 0 { - fails = append(fails, "no mixed queries succeeded") - } - if report.QueryErrors > 0 { - fails = append(fails, fmt.Sprintf("query errors=%d", report.QueryErrors)) - } - - // Without a server-side baseline every delta below is meaningless, and a - // negative delta means a counter reset — i.e. fanout restarted mid-run, so - // the whole report describes two different processes. - if cfg.metricsURL != "" { - switch { - case report.Server == nil: - fails = append(fails, "server metrics evidence unavailable") - case !report.Server.BaselineAvailable: - fails = append(fails, "server metrics baseline unavailable") - default: - if report.Server.RowsDroppedDelta > 0 { - fails = append(fails, fmt.Sprintf("rows dropped=%.0f", report.Server.RowsDroppedDelta)) - } - // A restart is detected from the process start time, not from a - // negative delta: if fanout restarts early enough, the new process - // can out-count the old baseline and every delta stays positive - // while the report silently describes two different processes. - if report.Server.ProcessRestarted { - fails = append(fails, "fanout restarted mid-run (process start time changed)") - } else if report.Server.IngestRowsDelta < 0 || report.Server.RowsDroppedDelta < 0 || - report.Server.CPUSecondsDelta < 0 || report.Server.AllocBytesDelta < 0 || - report.Server.GCPauseSecondsDelta < 0 { - fails = append(fails, "server counter reset mid-run (fanout restarted?)") - } - // Rollups, merge, and maintenance are what this harness exists to - // measure. Left unchecked they can fail every cycle while queries - // stay fast on raw fallback and the run reports PASS. - // Sorted so the failure list — which is written to the report — does - // not vary with Go's map iteration order between runs. - for _, name := range sortedMapKeys(report.Server.Rollups) { - if errors := report.Server.Rollups[name].Outcomes["error"]; errors > 0 { - fails = append(fails, fmt.Sprintf("%s rollup errors=%.0f", name, errors)) - } - } - for _, name := range sortedMapKeys(report.Server.TelemetryOperations) { - if errors := report.Server.TelemetryOperations[name].Outcomes["error"]; errors > 0 { - fails = append(fails, fmt.Sprintf("telemetry %s errors=%.0f", name, errors)) - } - } - } - } - - if cfg.maxExportP95 > 0 && report.ExportLatencyMs.P95Ms > cfg.maxExportP95 { - fails = append(fails, fmt.Sprintf("export p95 %.0fms > %.0fms", report.ExportLatencyMs.P95Ms, cfg.maxExportP95)) - } - if cfg.maxQueryP95 > 0 && - report.QueryLatencyMs != nil && report.QueryLatencyMs.P95Ms > cfg.maxQueryP95 { - fails = append(fails, fmt.Sprintf("query p95 %.0fms > %.0fms", report.QueryLatencyMs.P95Ms, cfg.maxQueryP95)) - } - return fails -} diff --git a/cmd/bench/verdict_test.go b/cmd/bench/verdict_test.go deleted file mode 100644 index d67ef9eda..000000000 --- a/cmd/bench/verdict_test.go +++ /dev/null @@ -1,192 +0,0 @@ -package main - -import ( - "strings" - "testing" -) - -func hasFailure(failures []string, want string) bool { - for _, failure := range failures { - if strings.Contains(failure, want) { - return true - } - } - return false -} - -func TestEvaluateReportCollectsEveryFailure(t *testing.T) { - queryLatency := latencyReport{Count: 10, P95Ms: 1600} - cfg := config{ - metricsURL: "https://metrics.example.test", - queryURL: "https://query.example.test", - queryWorkers: 4, - maxExportP95: 100, - maxQueryP95: 1000, - } - report := report{ - // 5 send errors against 100 successes is a 4.76% drop rate, far above - // the 0.1% guardrail: a run that silently loses that much telemetry - // cannot support a throughput comparison. - ExportLatencyMs: latencyReport{Count: 100, P95Ms: 150}, - SendErrors: 5, - QueriesRun: 10, - QueryErrors: 2, - QueryLatencyMs: &queryLatency, - Server: &serverReport{ - BaselineAvailable: true, - RowsDroppedDelta: 1, - }, - } - - failures := evaluateReport(cfg, report, []string{"final metrics scrape failed"}) - for _, expected := range []string{ - "final metrics scrape failed", - "query errors=2", - "rows dropped=1", - "export p95 150ms > 100ms", - "query p95 1600ms > 1000ms", - "send error rate 4.762% (5/105) > 0.1%", - } { - if !hasFailure(failures, expected) { - t.Fatalf("failures %q do not contain %q", failures, expected) - } - } -} - -// Latency alone must not fail a run. A built-in millisecond threshold encodes -// the machine it was calibrated on, so the same binary would "fail" purely for -// running on a smaller computer. -func TestEvaluateReportDoesNotJudgeLatencyWithoutACallerThreshold(t *testing.T) { - queryLatency := latencyReport{Count: 100, P95Ms: 9000} - cfg := config{queryURL: "https://query.example.test", queryWorkers: 4} - report := report{ - ExportLatencyMs: latencyReport{Count: 100, P95Ms: 8000}, - QueriesRun: 100, - QueryLatencyMs: &queryLatency, - } - - if failures := evaluateReport(cfg, report, nil); len(failures) != 0 { - t.Fatalf("slow but healthy run failed: %q", failures) - } -} - -// A run that produced no usable evidence must fail rather than report -// percentiles computed over whatever survived. -func TestEvaluateReportRejectsUnusableRuns(t *testing.T) { - metricsCfg := config{metricsURL: "https://metrics.example.test"} - tests := []struct { - name string - cfg config - rep report - want string - }{ - { - name: "no exports", - rep: report{}, - want: "no OTLP exports succeeded", - }, - { - name: "query load configured but no queries ran", - cfg: config{queryURL: "http://query", queryWorkers: 1}, - rep: report{ExportLatencyMs: latencyReport{Count: 10}}, - want: "no mixed queries succeeded", - }, - { - name: "metrics requested but absent", - cfg: metricsCfg, - rep: report{ExportLatencyMs: latencyReport{Count: 10}}, - want: "server metrics evidence unavailable", - }, - { - name: "baseline scrape missing", - cfg: metricsCfg, - rep: report{ - ExportLatencyMs: latencyReport{Count: 10}, - Server: &serverReport{BaselineAvailable: false}, - }, - want: "server metrics baseline unavailable", - }, - { - name: "counter went backwards", - cfg: metricsCfg, - rep: report{ - ExportLatencyMs: latencyReport{Count: 10}, - Server: &serverReport{BaselineAvailable: true, IngestRowsDelta: -1}, - }, - want: "server counter reset mid-run", - }, - } - for _, test := range tests { - t.Run(test.name, func(t *testing.T) { - if failures := evaluateReport(test.cfg, test.rep, nil); !hasFailure(failures, test.want) { - t.Fatalf("failures %q do not contain %q", failures, test.want) - } - }) - } -} - -func TestEvaluateReportAcceptsHealthyRun(t *testing.T) { - queryLatency := latencyReport{Count: 10, P95Ms: 100} - failures := evaluateReport(config{queryURL: "http://query", queryWorkers: 1}, report{ - ExportLatencyMs: latencyReport{Count: 100, P95Ms: 20}, - QueriesRun: 10, - QueryLatencyMs: &queryLatency, - }, nil) - if len(failures) != 0 { - t.Fatalf("healthy run failed: %s", strings.Join(failures, "; ")) - } -} - -// Rollup, merge, and maintenance are what this harness exists to measure. If -// they fail every cycle the run must fail, even though raw-fallback queries stay -// fast and every other signal looks healthy. -func TestEvaluateReportFailsOnBackgroundWorkErrors(t *testing.T) { - queryLatency := latencyReport{Count: 10, P95Ms: 100} - failures := evaluateReport(config{metricsURL: "https://metrics.example.test"}, report{ - ExportLatencyMs: latencyReport{Count: 100, P95Ms: 20}, - QueriesRun: 10, - QueryLatencyMs: &queryLatency, - Server: &serverReport{ - BaselineAvailable: true, - Rollups: map[string]rollupReport{ - "service": {Outcomes: map[string]float64{"success": 30}}, - "edge": {Outcomes: map[string]float64{"error": 12}}, - }, - TelemetryOperations: map[string]backgroundOperationReport{ - "maintenance": {Outcomes: map[string]float64{"error": 3}}, - "compaction": {Outcomes: map[string]float64{"success": 9}}, - }, - }, - }, nil) - for _, want := range []string{"edge rollup errors=12", "telemetry maintenance errors=3"} { - if !hasFailure(failures, want) { - t.Fatalf("failures %q do not contain %q", failures, want) - } - } - if hasFailure(failures, "service rollup") || hasFailure(failures, "telemetry merge") { - t.Errorf("healthy components reported as failures: %q", failures) - } -} - -// A restart is caught from the process start time. Deltas alone miss it: a -// server that restarts early can out-count its own baseline, leaving every -// delta positive while the report describes two different processes. -func TestEvaluateReportDetectsRestartWithPositiveDeltas(t *testing.T) { - queryLatency := latencyReport{Count: 10, P95Ms: 100} - failures := evaluateReport(config{metricsURL: "https://metrics.example.test"}, report{ - ExportLatencyMs: latencyReport{Count: 100, P95Ms: 20}, - QueriesRun: 10, - QueryLatencyMs: &queryLatency, - Server: &serverReport{ - BaselineAvailable: true, - ProcessRestarted: true, - // Every delta is positive — the old sign check sees nothing wrong. - IngestRowsDelta: 5000, - CPUSecondsDelta: 42, - AllocBytesDelta: 1 << 30, - }, - }, nil) - if !hasFailure(failures, "fanout restarted mid-run") { - t.Fatalf("failures %q do not report the restart", failures) - } -} diff --git a/docs/benchmarking.md b/docs/benchmarking.md deleted file mode 100644 index 924bb0f6e..000000000 --- a/docs/benchmarking.md +++ /dev/null @@ -1,33 +0,0 @@ -# Benchmarking Fanout - -`cmd/bench` is the supported load generator. It sends traces, metrics, and logs, -ramps to find the ingest boundary, and confirms the result at the selected rate. -It can also add authenticated dashboard reads. - -Fanout does not currently publish a throughput headline. The earlier two-vCPU -report was withdrawn before the first public release because its raw JSON reports -and exact driver commit were not retained, and its ingest run predated mandatory -ingest authentication. The prose was detailed, but detail is not provenance. - -A future published result must include: - -- the Fanout image digest and benchmark-driver commit; -- complete machine, network, container, and Fanout configuration; -- raw `ingest.json` and `mixed.json` reports from the same run; -- an authenticated OTLP path matching the current release; -- repeated runs or an explicit single-run limitation; -- separate ingest-only and mixed read/write results. - -For local capacity testing, build the driver with `just build`, create an ingest -token through first-admin setup, and run: - -```sh -./bin/bench \ - -endpoint localhost:4317 \ - -token "$INGEST_TOKEN" \ - -report ingest.json -``` - -Use `./bin/bench -help` for fixed-rate, query-load, and metrics options. Results -describe the tested hardware and dataset, not an equivalent vCPU count on every -provider. diff --git a/internal/config/config.go b/internal/config/config.go index 78f27664f..c854a6067 100644 --- a/internal/config/config.go +++ b/internal/config/config.go @@ -51,7 +51,8 @@ type Config struct { // values validated on the reference deployment target, a small shared VM // (Hetzner CPX32: 4 vCPU, 8 GB RAM, 160 GB disk). There the self-sizing // resolves to a ~6.4 GB memory cap and 4 query threads (deterministic from - // 8 GB / 4 vCPU). Measure the target host with cmd/bench before setting + // 8 GB / 4 vCPU). Measure the target host with the external fanout-bench + // harness before setting // production limits; throughput and memory use vary with CPU, disk, and the // telemetry mix. // diff --git a/internal/observability/performance_benchmark_test.go b/internal/observability/performance_benchmark_test.go deleted file mode 100644 index b1ab6b26e..000000000 --- a/internal/observability/performance_benchmark_test.go +++ /dev/null @@ -1,118 +0,0 @@ -package observability - -import ( - "database/sql" - "testing" - "time" - - _ "github.com/duckdb/duckdb-go/v2" - - "github.com/labstack/fanout/internal/query" -) - -func BenchmarkEndpointQueries24Hours(b *testing.B) { - db, err := sql.Open("duckdb", "") - if err != nil { - b.Fatal(err) - } - defer db.Close() - if _, err := db.Exec(`SET TimeZone='UTC'; SET threads=4`); err != nil { - b.Fatal(err) - } - if err := query.CreateTables(db); err != nil { - b.Fatal(err) - } - if err := query.CreateViews(db); err != nil { - b.Fatal(err) - } - start := time.Date(2026, 7, 20, 0, 0, 30, 0, time.UTC) - if _, err := db.Exec(`INSERT INTO telemetry.spans ( - namespace, service, start_time, duration_ms, status, http_method, http_route, operation -) -SELECT - 'prod'::VARCHAR AS namespace, - 'svc-' || lpad((i % 50)::VARCHAR, 2, '0') AS service, - ?::TIMESTAMP + (i % 86400) * INTERVAL 1 SECOND AS start_time, - ((i % 300000) + 1)::DOUBLE / 100.0 AS duration_ms, - CASE WHEN i % 20 = 0 THEN 'STATUS_CODE_ERROR' ELSE 'STATUS_CODE_OK' END::VARCHAR AS status, - CASE WHEN i % 2 = 0 THEN 'GET' ELSE 'POST' END::VARCHAR AS http_method, - '/route-' || (i % 20)::VARCHAR AS http_route, - 'operation-' || (i % 20)::VARCHAR AS operation -FROM range(5000000) t(i)`, start); err != nil { - b.Fatal(err) - } - if _, err := db.Exec(`INSERT INTO endpoint_rollup -SELECT - namespace, - date_trunc('minute', start_time), - COALESCE(service, ''), - COALESCE(NULLIF(http_method, ''), 'CALL'), - COALESCE(NULLIF(http_route, ''), NULLIF(operation, ''), 'unknown'), - COUNT(*), - COUNT(*) FILTER (WHERE upper(status) IN ('ERROR', 'STATUS_CODE_ERROR')), - COUNT(duration_ms), - struct_pack( - le_0_1 := COUNT(*) FILTER (WHERE duration_ms <= 0.1), - le_0_5 := COUNT(*) FILTER (WHERE duration_ms <= 0.5), - le_1 := COUNT(*) FILTER (WHERE duration_ms <= 1), - le_2_5 := COUNT(*) FILTER (WHERE duration_ms <= 2.5), - le_5 := COUNT(*) FILTER (WHERE duration_ms <= 5), - le_10 := COUNT(*) FILTER (WHERE duration_ms <= 10), - le_25 := COUNT(*) FILTER (WHERE duration_ms <= 25), - le_50 := COUNT(*) FILTER (WHERE duration_ms <= 50), - le_100 := COUNT(*) FILTER (WHERE duration_ms <= 100), - le_250 := COUNT(*) FILTER (WHERE duration_ms <= 250), - le_500 := COUNT(*) FILTER (WHERE duration_ms <= 500), - le_750 := COUNT(*) FILTER (WHERE duration_ms <= 750), - le_1000 := COUNT(*) FILTER (WHERE duration_ms <= 1000), - le_2000 := COUNT(*) FILTER (WHERE duration_ms <= 2000), - le_5000 := COUNT(*) FILTER (WHERE duration_ms <= 5000), - le_30000 := COUNT(*) FILTER (WHERE duration_ms <= 30000), - le_300000 := COUNT(*) FILTER (WHERE duration_ms <= 300000) - ) -FROM spans -GROUP BY - namespace, - date_trunc('minute', start_time), - service, - COALESCE(NULLIF(http_method, ''), 'CALL'), - COALESCE(NULLIF(http_route, ''), NULLIF(operation, ''), 'unknown')`); err != nil { - b.Fatal(err) - } - end := start.Add(24*time.Hour - time.Second) - - bench := func(b *testing.B, query string, args ...any) { - b.Helper() - for i := 0; i < b.N; i++ { - rows, err := db.Query(query, args...) - if err != nil { - b.Fatal(err) - } - count := 0 - for rows.Next() { - var method, path string - var calls int64 - var p50, p95, p99, errorRate float64 - if err := rows.Scan(&method, &path, &calls, &p50, &p95, &p99, &errorRate); err != nil { - rows.Close() - b.Fatal(err) - } - count++ - } - if err := rows.Close(); err != nil { - b.Fatal(err) - } - if count == 0 { - b.Fatal("query returned no endpoints") - } - } - } - b.Run("raw-spans", func(b *testing.B) { - b.ReportMetric(5000000, "dataset_spans") - bench(b, rawEndpointsQuery, start, end, "prod", "prod", "", "", 100) - }) - b.Run("endpoint-rollup", func(b *testing.B) { - b.ReportMetric(5000000, "dataset_spans") - bench(b, endpointRollupQuery, start, end, end, "prod", "", 100) - }) -} diff --git a/justfile b/justfile index c7734b4af..9c7405ca6 100644 --- a/justfile +++ b/justfile @@ -53,10 +53,9 @@ ui-host: # Every embedded browser asset. ui: ui-apps ui-host -# Browser assets, then the binaries that embed them. +# Browser assets, then the binary that embeds them. build VERSION=`git describe --tags --always --dirty 2>/dev/null || echo dev`: ui go build -ldflags "-s -w -X main.version={{VERSION}}" -o bin/fanout ./cmd/fanout - go build -ldflags "-s -w" -o bin/bench ./cmd/bench # CI publishes ghcr.io/labstack/fanout; this is for trying the image locally # without pushing anything. diff --git a/site/src/content/docs/explanation/performance.mdx b/site/src/content/docs/explanation/performance.mdx index a78da39f2..06a817df2 100644 --- a/site/src/content/docs/explanation/performance.mdx +++ b/site/src/content/docs/explanation/performance.mdx @@ -38,25 +38,31 @@ Before any figure goes back on this page, it has to ship with: ## Measure your own instead -`cmd/bench` is the supported load generator. It sends traces, metrics and logs, -ramps to find the ingest boundary, and confirms the result at the rate it found. -It can add authenticated dashboard reads on top, which is the part that matters. +The independent +[Fanout Bench](https://github.com/labstack/fanout-bench) project drives the +official OpenTelemetry `telemetrygen` client, counts rows accepted by Fanout, +and emits raw JSON plus a publication-ready report. It can add authenticated +dashboard reads on top, which is the part that matters. Parquet ingest and DuckDB reads no longer serialize through one write gate, but they still compete for the same CPU, memory bandwidth, filesystem cache, and disk. Maintenance adds native Parquet merge work and short reader-exclusive namespace changes. An ingest-only number therefore still overstates what an instance does while anyone is looking at it. -Build it, create an ingest token through first-admin setup, then: +Create a short-lived ingest token through first-admin setup, then follow the +standalone harness instructions. A local run starts with: ```sh -./bin/bench \ - -endpoint localhost:4317 \ - -token "$INGEST_TOKEN" \ - -report ingest.json +export FANOUT_BENCH_INGEST_TOKEN='…' +fanout-bench run \ + --endpoint localhost:4317 \ + --metrics-url http://localhost:7520/-/metrics \ + --insecure \ + --same-host ``` -`./bin/bench -help` covers fixed-rate, query-load and metrics options. +The harness repository documents signal-specific saturation, sustained mixed +load, read-under-write latency, machine identity, and publication rules. :::caution[A result describes the machine it ran on] Benchmark output describes the hardware and dataset you tested, not an From c6129ad52516001e79a3c4dc299893273c3b6994 Mon Sep 17 00:00:00 2001 From: Vishal Rana Date: Fri, 28 Aug 2026 15:54:40 -0700 Subject: [PATCH 2/2] docs: document benchmark metrics token --- site/src/content/docs/explanation/performance.mdx | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/site/src/content/docs/explanation/performance.mdx b/site/src/content/docs/explanation/performance.mdx index 06a817df2..9dbffb77e 100644 --- a/site/src/content/docs/explanation/performance.mdx +++ b/site/src/content/docs/explanation/performance.mdx @@ -49,11 +49,14 @@ disk. Maintenance adds native Parquet merge work and short reader-exclusive namespace changes. An ingest-only number therefore still overstates what an instance does while anyone is looking at it. -Create a short-lived ingest token through first-admin setup, then follow the -standalone harness instructions. A local run starts with: +Create a short-lived ingest token through first-admin setup, configure Fanout +with a dedicated `FANOUT_METRICS_TOKEN`, then expose that same metrics token to +the harness as `FANOUT_BENCH_METRICS_TOKEN`. Metrics are private by default. A +local run starts with: ```sh export FANOUT_BENCH_INGEST_TOKEN='…' +export FANOUT_BENCH_METRICS_TOKEN='…' fanout-bench run \ --endpoint localhost:4317 \ --metrics-url http://localhost:7520/-/metrics \