Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
10 changes: 8 additions & 2 deletions README.md
Original file line number Diff line number Diff line change
Expand Up @@ -45,9 +45,12 @@ The opt-in `integration` build tag starts MongoDB, Elasticsearch, two Store
owners and a discovery node. Tagged tests require a prepared `WEIR_TEST_BINARY`.
Coverage includes discovery through every node, direct owner connections, typed
CRUD and Lua transformations, independent persistence checks, same-Store batches,
whole-request validation, finite Scan continuation, Native response evidence,
slice preflight and per-frame validation, finite Scan continuation, Native response evidence,
cancellation, owner restart, directory convergence and persistent client recovery.
Separate multi-process tests verify independent single-record callers.
One-RPC reads deliver 64 MiB and 256 MiB across 1024 distinct documents while
checking bounded result credits and sampled owner RSS. Interrupted streams retain
confirmed record outcomes; unconfirmed mutations remain indeterminate.

Services bind to loopback and own exclusive namespaces. Cleanup verifies exact
process and container ownership. Fixture logs, manifests and cleanup receipts
Expand Down Expand Up @@ -98,7 +101,7 @@ distributions and the RPC/adapter ratio show whether aggregation occurred.

Reports retain per-round throughput, pooled latency histograms, client process
identities, start lag, CPU/RSS, database CPU/memory/I/O/network samples and raw
Weir metrics. Metrics report unary completions, terminal records, queue wait,
Weir metrics. Metrics report stream completions, terminal records, queue wait,
adapter time, rejections and operations per adapter invocation. MongoDB also
records physical command deltas. Adapter invocations can split by namespace,
action and byte limits; their counts are distinct from database wire commands.
Expand All @@ -112,6 +115,9 @@ concurrency alone cannot prove a plateau. A plateau with low database CPU leaves
maximum-capacity comparison unavailable. Reports still show observed throughput
and latency at each matched concurrency. No performance ratio is a CI threshold.

The lab records `weir_max_read_size_bytes`: the payload plus 4 KiB metadata
headroom, rounded up to 4 KiB and capped at 2 MiB. This prevents reserving 2 MiB
for every 1 KiB benchmark read; large-document integration still uses 2 MiB.
Weir working-memory and process-admission budgets scale with backend concurrency
and ingress sessions. Reports record declared budgets and observed usage.
`-store-working-memory-mib` allows explicit workspace experiments.
Expand Down
8 changes: 5 additions & 3 deletions cmd/weir-lab/main.go
Original file line number Diff line number Diff line change
Expand Up @@ -245,8 +245,10 @@ func runLab(ctx context.Context, opts labOptions) (resultErr error) {
for backend, memory := range workspace {
provenance["weir_"+backend+"_working_memory_mib"] = fmt.Sprint(memory)
}
maxReadBytes := min(2<<20, ((opts.payload+4096+4095)/4096)*4096)
provenance["weir_max_read_size_bytes"] = fmt.Sprint(maxReadBytes)
provenance["weir_memory_budget"] = fmt.Sprintf("%dMiB (declared process admission budget; not an OS reservation)", memoryMiB)
start := fixture.Options{WeirBinary: binary, MongoBinary: opts.mongoBinary, Backends: backends, OwnerCount: 1, StoreConcurrency: storeWorkers, BatchSize: opts.backendBatchLimit, BackendTimeout: businessTimeout, IngressSessions: max(64, opts.concurrency), DatabaseCPUs: opts.databaseCPUs, ProcessMemoryMiB: memoryMiB, WorkingMemoryMiB: workspace}
start := fixture.Options{WeirBinary: binary, MongoBinary: opts.mongoBinary, Backends: backends, OwnerCount: 1, StoreConcurrency: storeWorkers, BatchSize: opts.backendBatchLimit, MaxReadSizeBytes: maxReadBytes, BackendTimeout: businessTimeout, IngressSessions: max(64, opts.concurrency), DatabaseCPUs: opts.databaseCPUs, ProcessMemoryMiB: memoryMiB, WorkingMemoryMiB: workspace}
cluster, err := fixture.Start(ctx, start)
if err != nil {
return err
Expand Down Expand Up @@ -303,8 +305,8 @@ func fixtureMemoryBudget(opts labOptions, backends []string, workers int) (map[s
workspace := make(map[string]int, len(backends))
// The pinned adapters bound one 2MiB-read batch by at most 41MiB for
// MongoDB (native reply, guard and scratch) and 96MiB for Search.
// Add ingress envelopes and 1GiB for Store input/results and other framing.
memoryMiB := max(64, opts.concurrency)*96 + 1024
// Streaming ingress uses 32MiB per admitted stream. Add 1GiB for Store input/results and other framing.
memoryMiB := max(64, opts.concurrency)*32 + 1024
for _, backend := range backends {
perBatchMiB := 41
if opts.luaMutations {
Expand Down
8 changes: 4 additions & 4 deletions cmd/weir-lab/memory_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -12,19 +12,19 @@ func TestFixtureMemoryCoversActualBackendConcurrency(t *testing.T) {
if workspace["mongo"]<<20 < 32*((32<<20)+(128<<10)+(8<<20)) || workspace["search"] < 32*96 {
t.Fatal("workspace silently throttles the configured backend concurrency", workspace)
}
reservedMiB := 64 + 64/4 + 64*96 + workspace["mongo"] + workspace["search"] + 2*(64+32*2)
if processMiB < reservedMiB || processMiB != 12288 {
reservedMiB := 64 + 64/4 + 64*32 + workspace["mongo"] + workspace["search"] + 2*(64+32*2)
if processMiB < reservedMiB || processMiB != 8192 {
t.Fatal("ingress and both backend envelopes do not fit the process", processMiB, reservedMiB)
}
opts.workingMemoryMiB = 384
workspace, processMiB, err = fixtureMemoryBudget(opts, backends, 32)
if err != nil || workspace["mongo"] != 384 || workspace["search"] != 384 || processMiB != 8192 {
if err != nil || workspace["mongo"] != 384 || workspace["search"] != 384 || processMiB != 4096 {
t.Fatal("explicit workspace budget was not honored", workspace, processMiB, err)
}
opts.workingMemoryMiB = 0
opts.concurrency = 512
workspace, processMiB, err = fixtureMemoryBudget(opts, backends, 32)
if err != nil || workspace["mongo"] != 1312 || workspace["search"] != 3072 || processMiB != 55296 {
if err != nil || workspace["mongo"] != 1312 || workspace["search"] != 3072 || processMiB != 22528 {
t.Fatal("512 independent requests do not fit the explicitly declared ingress envelope", workspace, processMiB, err)
}
if _, _, err := fixtureMemoryBudget(opts, backends, int(^uint(0)>>1)); err == nil {
Expand Down
4 changes: 2 additions & 2 deletions go.mod
Original file line number Diff line number Diff line change
Expand Up @@ -3,8 +3,8 @@ module github.com/batchstream/weir-tests
go 1.27.1

require (
github.com/batchstream/weir-go v0.5.0
github.com/batchstream/weir-protocol v0.3.0
github.com/batchstream/weir-go v0.6.0
github.com/batchstream/weir-protocol v0.4.0
go.mongodb.org/mongo-driver/v2 v2.9.1
google.golang.org/grpc v1.83.2
)
Expand Down
8 changes: 4 additions & 4 deletions go.sum
Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@
github.com/batchstream/weir-go v0.5.0 h1:mcKSCcckl+jU4FkhLhwvIumm33YkOm/aomMwIlAElOw=
github.com/batchstream/weir-go v0.5.0/go.mod h1:poDKCB+xMXpRvpgUewEwB25vIAM9HeGpm3ub0pVEbow=
github.com/batchstream/weir-protocol v0.3.0 h1:6YDOl9yskTP27q4ighNq1+i9y4AGy+/kQTKJvCSQ+Y4=
github.com/batchstream/weir-protocol v0.3.0/go.mod h1:hHmwPoE9yHYAz7Auc3a14Ijo/uxAef4sQ9nBtyXNcGc=
github.com/batchstream/weir-go v0.6.0 h1:S9/+7diNNNbVEH68lg6GbepzXAiCJpF0K74R5pTNFmo=
github.com/batchstream/weir-go v0.6.0/go.mod h1:YsokqwtYw2sN7zVbt4yIgqJUnuoHsDyiA1vQlXPE2VU=
github.com/batchstream/weir-protocol v0.4.0 h1:t4dDLAKeOOGz/wajfsfrYZgbILDvVceoPvJYKxTQuiM=
github.com/batchstream/weir-protocol v0.4.0/go.mod h1:hHmwPoE9yHYAz7Auc3a14Ijo/uxAef4sQ9nBtyXNcGc=
github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs=
github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs=
github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
Expand Down
8 changes: 4 additions & 4 deletions internal/benchmark/saturation_report.go
Original file line number Diff line number Diff line change
Expand Up @@ -42,16 +42,16 @@ func (r *SaturationReport) Markdown() string {
fmt.Fprintf(&text, "| %d | %d | %d | %s | %d | %d | %d | %d | %.1f | %.3f | %.1f | %.1f | %.1f | %t |\n", pair.BatchSize, pair.Concurrency, pair.Round, result.Path, result.Reads, result.Writes, result.Succeeded, result.Errors+result.Indeterminate, result.OperationsPerSec, float64(result.Latency.P95NS)/1e6, result.Resources.MeanCPUPercent, result.Resources.FullCPUFraction*100, result.Resources.Coverage*100, result.Verified)
}
}
text.WriteString("\nMeasured Weir adapter and unary RPC evidence (warmup, reset and postflight excluded):\n\n| Batch | Workers | Round | Adapter invocations | Mean adapter batch | Read/Mutate RPCs | Queue mean ms | Adapter mean ms | Rejections | Backend concurrency limit |\n| ---: | ---: | ---: | ---: | ---: | --- | ---: | ---: | ---: | ---: |\n")
text.WriteString("\nMeasured Weir adapter and streaming RPC evidence (warmup, reset and postflight excluded):\n\n| Batch | Workers | Round | Adapter invocations | Mean adapter batch | Execute RPCs | Queue mean ms | Adapter mean ms | Rejections | Backend concurrency limit |\n| ---: | ---: | ---: | ---: | ---: | --- | ---: | ---: | ---: | ---: |\n")
var metricWarnings []string
for _, pair := range r.Pairs {
metrics := pair.Weir.ServerMetrics
if metrics == nil {
fmt.Fprintf(&text, "| %d | %d | %d | unavailable | unavailable | unavailable | unavailable | unavailable | unavailable | unavailable |\n", pair.BatchSize, pair.Concurrency, pair.Round)
continue
}
fmt.Fprintf(&text, "| %d | %d | %d | %s | %s | %s/%s | %s | %s | %s | %s |\n", pair.BatchSize, pair.Concurrency, pair.Round,
metricCounter(metrics, "weir_store_executions_total"), metricMean(metrics.AdapterBatchAverage, 1), metricCounter(metrics, "weir_rpc_completions_total:read"), metricCounter(metrics, "weir_rpc_completions_total:mutate"),
fmt.Fprintf(&text, "| %d | %d | %d | %s | %s | %s | %s | %s | %s | %s |\n", pair.BatchSize, pair.Concurrency, pair.Round,
metricCounter(metrics, "weir_store_executions_total"), metricMean(metrics.AdapterBatchAverage, 1), metricCounter(metrics, "weir_rpc_completions_total:execute"),
metricMean(metrics.QueueWaitMeanSeconds, 1000), metricMean(metrics.ExecutionMeanSeconds, 1000), metricCounter(metrics, "weir_store_rejections_total"), metricMean(metrics.ConcurrencyLimit, 1))
if metrics.Unavailable != "" {
warning := fmt.Sprintf("Metrics unavailable (batch %d, workers %d, round %d): %s.", pair.BatchSize, pair.Concurrency, pair.Round, strings.ReplaceAll(metrics.Unavailable, "\n", " "))
Expand Down Expand Up @@ -81,7 +81,7 @@ func (r *SaturationReport) Markdown() string {
text.WriteString("\nThese QPS and latency observations apply to this hardware and measured concurrency range. A business-throughput improvement can exist without demonstrating the maximum database capacity. An underfull database or missing next-level plateau keeps the database-capacity comparison unavailable.\n")
}
if r.Parameters.ClientProcesses > 0 {
text.WriteString("\nAggregation evidence: the JSON records one business request per record, Read/Mutate RPC counts, adapter invocation counts, average adapter batch and its cumulative histogram, and RPCs per adapter invocation. An adapter invocation is not necessarily one physical database wire command. MongoDB physical find/update/bulkWrite/getMore/killCursors/commitTransaction/abortTransaction deltas come from owned serverStatus at timed boundaries; missing counters remain unavailable. Elasticsearch has no equivalent physical HTTP-command counter. Every child PID, configured native pool limit, final revision ranges, exit result and sampled CPU/RSS is retained; coordinator sampling CPU is excluded from business-client CPU.\n")
text.WriteString("\nAggregation evidence: the JSON records one business request per record, Execute RPC counts, adapter invocation counts, average adapter batch and its cumulative histogram, and RPCs per adapter invocation. An adapter invocation is not necessarily one physical database wire command. MongoDB physical find/update/bulkWrite/getMore/killCursors/commitTransaction/abortTransaction deltas come from owned serverStatus at timed boundaries; missing counters remain unavailable. Elasticsearch has no equivalent physical HTTP-command counter. Every child PID, configured native pool limit, final revision ranges, exit result and sampled CPU/RSS is retained; coordinator sampling CPU is excluded from business-client CPU.\n")
}
text.WriteString("\nThe JSON retains before/after raw public Weir metrics and their observed deltas, every resource sample, monitor failures, raw cumulative database/container I/O and network counters, CPU quota/host core denominator, and client/Weir cumulative CPU. Docker CPU uses cumulative Engine usage and read timestamps across each entire sampling interval, never CLI instantaneous percentages. Both interval endpoints must succeed; missing intervals reduce full-run coverage. Samples include monitoring traffic and background database work. Native ps CPU has platform-dependent clock resolution. Native databases share the host CPU budget with the client and Weir. Disk/network utilization or capacity is not inferred from byte counters. A plateau without sustained database CPU saturation is not a database-full-load result. Startup/discovery/reset/postflight and warmup are excluded from throughput; all in-flight completion and validation is included.\n")
if r.Incomplete != "" {
Expand Down
34 changes: 12 additions & 22 deletions internal/benchmark/server_metrics.go
Original file line number Diff line number Diff line change
Expand Up @@ -49,7 +49,7 @@ func serverMetricDelta(before, after observe.Snapshot, store string) *ServerMetr
} else {
metrics.ConcurrencyLimit = &afterLimit
}
for _, method := range []string{"read", "mutate"} {
for _, method := range []string{"execute"} {
methodLabels := map[string]string{"method": method}
value, deltaErr := observe.Delta(before, after, "weir_rpc_completions_total", methodLabels)
if deltaErr == nil {
Expand All @@ -65,10 +65,8 @@ func serverMetricDelta(before, after observe.Snapshot, store string) *ServerMetr
if count := metrics.Deltas["weir_store_batch_operations_count"]; count > 0 {
average := metrics.Deltas["weir_store_batch_operations_sum"] / count
metrics.AdapterBatchAverage = &average
read, readFound := metrics.Deltas["weir_rpc_completions_total:read"]
mutate, mutateFound := metrics.Deltas["weir_rpc_completions_total:mutate"]
if readFound && mutateFound {
ratio := (read + mutate) / count
if streams, found := metrics.Deltas["weir_rpc_completions_total:execute"]; found {
ratio := streams / count
metrics.RPCsPerAdapter = &ratio
}
} else {
Expand Down Expand Up @@ -98,29 +96,21 @@ func serverMetricDelta(before, after observe.Snapshot, store string) *ServerMetr
return metrics
}

// Single-request aggregation evidence must match the measured business calls.
// An unused RPC method need not have an exported series; an active one must.
// Single-request aggregation evidence must match every measured business stream.
func (m *ServerMetrics) qualifySingleRequests(reads, writes uint64) {
m.RPCsPerAdapter = nil
if m.Unavailable != "" || m.AdapterBatchAverage == nil {
return
}
count := m.Deltas["weir_store_batch_operations_count"]
var observed float64
for method, calls := range map[string]uint64{"read": reads, "mutate": writes} {
value, found := m.Deltas["weir_rpc_completions_total:"+method]
if calls > 0 && !found {
return
}
if found && value != float64(calls) {
m.RPCUnavailable[method] = "timed unary completion count differs from measured single-request calls"
return
}
if found {
observed += value
}
observed, found := m.Deltas["weir_rpc_completions_total:execute"]
if !found {
return
}
if observed != float64(reads+writes) {
m.RPCUnavailable["execute"] = "timed stream completion count differs from measured single-request calls"
return
}
if count > 0 {
if count := m.Deltas["weir_store_batch_operations_count"]; count > 0 {
ratio := observed / count
m.RPCsPerAdapter = &ratio
}
Expand Down
23 changes: 15 additions & 8 deletions internal/benchmark/server_metrics_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -29,7 +29,7 @@ func metricsSnapshot(t *testing.T, measured bool) observe.Snapshot {
if measured {
rpc += 100
}
fmt.Fprintf(&raw, "weir_rpc_completions_total{method=\"read\",status=\"ok\"} %d\n", rpc)
fmt.Fprintf(&raw, "weir_rpc_completions_total{method=\"execute\",status=\"ok\"} %d\n", rpc)
metrics, err := observe.Parse(raw.String())
if err != nil {
t.Fatal(err)
Expand All @@ -42,7 +42,7 @@ func TestServerMetricsUseMeasuredDeltaAndReportMissingObservations(t *testing.T)
before := metricsSnapshot(t, false)
after := metricsSnapshot(t, true)
metrics := serverMetricDelta(before, after, "mongo")
if metrics.Unavailable != "" || metrics.AdapterBatchAverage == nil || *metrics.AdapterBatchAverage != 32 || metrics.Deltas["weir_store_executions_total"] != 100 || metrics.Deltas["weir_rpc_completions_total:read"] != 100 {
if metrics.Unavailable != "" || metrics.AdapterBatchAverage == nil || *metrics.AdapterBatchAverage != 32 || metrics.Deltas["weir_store_executions_total"] != 100 || metrics.Deltas["weir_rpc_completions_total:execute"] != 100 {
t.Fatalf("measured batch did not exclude warmup/other store: %+v", metrics)
}
if metrics.QueueWaitMeanSeconds == nil || *metrics.QueueWaitMeanSeconds != .001 || metrics.ExecutionMeanSeconds == nil || *metrics.ExecutionMeanSeconds != .005 {
Expand All @@ -51,8 +51,15 @@ func TestServerMetricsUseMeasuredDeltaAndReportMissingObservations(t *testing.T)
if metrics.ConcurrencyLimit == nil || *metrics.ConcurrencyLimit != 32 {
t.Fatal("configured limit became a delta or came from another store", metrics)
}
if metrics.RPCUnavailable["mutate"] == "" {
t.Fatal("missing optional RPC observation silently disappeared")
if len(metrics.RPCUnavailable) != 0 {
t.Fatal("observed stream RPC unexpectedly unavailable")
}
pair := SaturationPair{BatchSize: 1, Concurrency: 32, Round: 1}
pair.Weir.ServerMetrics = metrics
report := &SaturationReport{Pairs: []SaturationPair{pair}}
markdown := report.Markdown()
if !strings.Contains(markdown, "Execute RPCs") || !strings.Contains(markdown, "| 1 | 32 | 1 | 100 | 32.000 | 100 |") || strings.Contains(markdown, "Read/Mutate RPC counts") {
t.Fatal("human-readable report lost observed Execute counts", markdown)
}
after.Error = "scrape failed"
metrics = serverMetricDelta(before, after, "mongo")
Expand Down Expand Up @@ -83,14 +90,14 @@ func TestSingleRequestRPCProofRequiresActiveMethodCounts(t *testing.T) {
metrics := serverMetricDelta(before, after, "mongo")
metrics.qualifySingleRequests(100, 0)
if metrics.RPCsPerAdapter == nil || *metrics.RPCsPerAdapter != 1 {
t.Fatal("unused mutation series hid verified read-only aggregation evidence")
t.Fatal("read-only stream aggregation evidence lost")
}
metrics.qualifySingleRequests(100, 1)
if metrics.RPCsPerAdapter != nil {
t.Fatal("missing active mutation count became zero")
t.Fatal("mismatched total stream count became valid")
}
metrics.qualifySingleRequests(99, 0)
if metrics.RPCsPerAdapter != nil || metrics.RPCUnavailable["read"] == "" {
t.Fatal("extra timed unary completions were treated as single-request proof")
if metrics.RPCsPerAdapter != nil || metrics.RPCUnavailable["execute"] == "" {
t.Fatal("extra timed stream completions were treated as single-request proof")
}
}
Loading
Loading