diff --git a/.github/workflows/saturation.yml b/.github/workflows/saturation.yml index 5eabd5b1..7362343c 100644 --- a/.github/workflows/saturation.yml +++ b/.github/workflows/saturation.yml @@ -26,7 +26,7 @@ on: default: '32' required: true type: choice - options: ['32', '1', '1,32'] + options: ['32', '128', '256', '1', '1,32', '32,128,256'] lua-mutations: description: Compare native single-record RMW with server-batched Lua mutations default: false @@ -99,15 +99,15 @@ jobs: python3 - <<'PY' import hashlib, json, os, pathlib, subprocess binary = pathlib.Path('.tools/weir-lab') - command = [str(binary), '-mode', 'saturation', '-weir', '.tools/weir', '-backend', 'all', '-database-cpus', os.environ['DATABASE_CPUS'], '-store-concurrency', '32', '-backend-batch-limit', os.environ['BACKEND_BATCH_LIMIT'], '-client-processes', '4', '-concurrency-levels', os.environ['CLIENT_LEVELS'], '-batch-sizes', '1', '-records', '2048', '-payload-bytes', '1024', '-write-percent', os.environ['WRITE_PERCENT'], '-rounds', '3', '-warmup-duration', '10s', '-duration', '20s', '-output', os.environ['MEASUREMENT_OUTPUT']] + command = [str(binary), '-mode', 'saturation', '-weir', '.tools/weir', '-backend', 'all', '-database-cpus', os.environ['DATABASE_CPUS'], '-backend-batch-limit', os.environ['BACKEND_BATCH_LIMIT'], '-client-processes', '4', '-concurrency-levels', os.environ['CLIENT_LEVELS'], '-batch-sizes', '1', '-records', '2048', '-payload-bytes', '1024', '-write-percent', os.environ['WRITE_PERCENT'], '-rounds', '3', '-warmup-duration', '10s', '-duration', '20s', '-output', os.environ['MEASUREMENT_OUTPUT']] if os.environ['LUA_MUTATIONS'] == 'true': command.append('-lua-mutations') - receipt = {'client_operation_timeout': '10s', 'weir_backend_timeout': '10s', 'host_memory_bytes': int(next(line for line in pathlib.Path('/proc/meminfo').read_text().splitlines() if line.startswith('MemTotal:')).split()[1]) * 1024, 'source': subprocess.check_output(['git', 'rev-parse', 'HEAD'], text=True).strip(), 'source_dirty': bool(subprocess.check_output(['git', 'status', '--porcelain'])), 'client_binary_sha256': hashlib.sha256(binary.read_bytes()).hexdigest(), 'client_build': subprocess.check_output(['go', 'version', '-m', str(binary)], text=True), 'versions': json.loads(pathlib.Path('versions.json').read_text()), 'command': command} + receipt = {'client_operation_timeout': '10s', 'host_memory_bytes': int(next(line for line in pathlib.Path('/proc/meminfo').read_text().splitlines() if line.startswith('MemTotal:')).split()[1]) * 1024, 'source': subprocess.check_output(['git', 'rev-parse', 'HEAD'], text=True).strip(), 'source_dirty': bool(subprocess.check_output(['git', 'status', '--porcelain'])), 'client_binary_sha256': hashlib.sha256(binary.read_bytes()).hexdigest(), 'client_build': subprocess.check_output(['go', 'version', '-m', str(binary)], text=True), 'versions': json.loads(pathlib.Path('versions.json').read_text()), 'command': command} if receipt['source_dirty']: raise SystemExit('measurement requires a clean source checkout') pathlib.Path(os.environ['MEASUREMENT_RECEIPT']).write_text(json.dumps(receipt, indent=2) + '\n') PY - .tools/weir-lab -mode saturation -weir .tools/weir -backend all -database-cpus "$DATABASE_CPUS" -store-concurrency 32 -backend-batch-limit "$BACKEND_BATCH_LIMIT" -client-processes 4 -concurrency-levels "$CLIENT_LEVELS" -batch-sizes 1 -records 2048 -payload-bytes 1024 -write-percent "$WRITE_PERCENT" -rounds 3 -warmup-duration 10s -duration 20s -output "$MEASUREMENT_OUTPUT" "${lua_flags[@]}" 2>&1 | tee "results/local/saturation-batch-$backend_limit.log" + .tools/weir-lab -mode saturation -weir .tools/weir -backend all -database-cpus "$DATABASE_CPUS" -backend-batch-limit "$BACKEND_BATCH_LIMIT" -client-processes 4 -concurrency-levels "$CLIENT_LEVELS" -batch-sizes 1 -records 2048 -payload-bytes 1024 -write-percent "$WRITE_PERCENT" -rounds 3 -warmup-duration 10s -duration 20s -output "$MEASUREMENT_OUTPUT" "${lua_flags[@]}" 2>&1 | tee "results/local/saturation-batch-$backend_limit.log" done git diff --exit-code - name: Preserve raw resource samples, workload validation and cleanup diff --git a/Makefile b/Makefile index 61ba3cd2..304bddc3 100644 --- a/Makefile +++ b/Makefile @@ -13,7 +13,7 @@ integration: WEIR_TEST_BINARY="$(CURDIR)/.tools/weir" GOWORK=off go test -race -tags=integration -count=1 -timeout=10m -v ./tests/integration benchmark: - GOWORK=off go run ./cmd/weir-lab -mode saturation -weir "$(CURDIR)/.tools/weir" -backend all -database-cpus 1 -store-concurrency 32 -backend-batch-limit 32 -client-processes 4 -concurrency-levels 8,32,128 -batch-sizes 1 -records 2048 -write-percent 0 -rounds 3 -warmup-duration 10s -duration 20s -output results/local/benchmark + GOWORK=off go run ./cmd/weir-lab -mode saturation -weir "$(CURDIR)/.tools/weir" -backend all -database-cpus 1 -client-processes 4 -concurrency-levels 8,32,128 -batch-sizes 1 -records 2048 -write-percent 0 -rounds 3 -warmup-duration 10s -duration 20s -output results/local/benchmark diagnostic: GOWORK=off go run ./cmd/weir-lab -mode fixed -weir "$(CURDIR)/.tools/weir" -output results/local/diagnostic diff --git a/README.md b/README.md index 453a2c5b..32b07926 100644 --- a/README.md +++ b/README.md @@ -6,8 +6,10 @@ Go SDK and public protocol. It imports no server packages. The SDK, protocol, server revision, Go toolchain and database artifacts are locked in [versions.json](versions.json). Server binaries are built separately -from the verified immutable source. Module replacements and floating revisions -are rejected by the dependency check. +from verified immutable module source. Explicit local builds for before/after +experiments are frozen with a source snapshot and checksummed receipt; they are +reported as local source. Module replacements and floating revisions are rejected +by the dependency check. ## Preparation and offline tests @@ -72,7 +74,8 @@ are retained in the printed output directory, including startup failures. The primary benchmark measures Weir's aggregation of independent single-record business requests. Four client OS processes issue one record per database call -or SDK RPC. Every process owns its driver or SDK connection pool and a disjoint +or SDK RPC. Concurrency levels count total workers across all processes; with +four processes, levels 8 and 32 mean 2 and 8 workers per process. Every process owns its driver or SDK connection pool and a disjoint key partition. There is no client batching. Run on Linux with Docker so database CPU quotas can be enforced and inspected: @@ -83,7 +86,7 @@ make benchmark # Equivalent command; choose a fresh output directory for each run. go run ./cmd/weir-lab -mode saturation \ -weir .tools/weir -backend all -database-cpus 1 \ - -store-concurrency 32 -backend-batch-limit 32 -client-processes 4 \ + -client-processes 4 \ -concurrency-levels 8,32,128 -batch-sizes 1 \ -records 2048 -payload-bytes 1024 -write-percent 0 \ -rounds 3 -warmup-duration 10s -duration 20s \ @@ -132,12 +135,82 @@ concurrency alone cannot prove a plateau. A plateau with low database CPU leaves maximum-capacity comparison unavailable. Reports still show observed throughput and latency at each matched concurrency. No performance ratio is a CI threshold. -The lab records `weir_max_read_size_bytes`: the payload plus 4 KiB metadata -headroom, rounded up to 4 KiB and capped at 2 MiB. This prevents reserving 2 MiB -for every 1 KiB benchmark read; large-document integration still uses 2 MiB. -Weir working-memory and process-admission budgets scale with backend concurrency -and ingress sessions. Reports record declared budgets and observed usage. -`-store-working-memory-mib` allows explicit workspace experiments. +The lab leaves server configuration at its defaults unless an experiment explicitly +sets `-backend-batch-limit` (`batching.max_operations`), `-pending-records` +(`transport.max_pending_records`) or `-exchange-bytes` +(`backend.max_exchange_bytes`). Zero means omitted; the generated fixture +configuration and raw metrics preserve the actual settings. There is no Store +concurrency, workspace memory or per-backend timeout override in the current +server contract. The SDK and native callers both use the complete 10s request +budget. + +For an unpublished local server refactor, freeze its Go source and build a new +binary. Existing output artifacts are preserved: + +```sh +python3 scripts/build_local_server.py --source ../weir \ + --output .tools/performance-after-server +go build -trimpath -buildvcs=false -o .tools/performance-lab ./cmd/weir-lab +.tools/performance-lab -mode saturation \ + -weir .tools/performance-after-server \ + -server-receipt .tools/performance-after-server.receipt.json \ + -backend mongo -mongod .tools/mongod -client-processes 4 \ + -concurrency-levels 8,32 -batch-sizes 1 -records 1024 \ + -payload-bytes 1024 -write-percent 10 -rounds 3 \ + -warmup-duration 2s -duration 5s -output results/local/after-mixed +``` + +The receipt records the binary SHA256, source HEAD, dirty state, Go source tree +SHA256 and every frozen input file. The retained source snapshot includes local +untracked Go files and a tracked Go diff, so the binary can be rebuilt independently +of later edits. Only Go build inputs are copied; secret files are excluded. +Local macOS measurements describe the observed workload on that host and do not +prove Linux database saturation or a production capacity limit. + +`scripts/run_performance_matrix.py` serializes read, mixed (10%), write and Lua +cases at total concurrency 8/32, with three alternating native/Weir rounds, +2s warmup and 5s measurement for native MongoDB, or 20s warmup and 20s +measurement for Elasticsearch per stage. Longer Search stages gave better +resource sampling coverage and avoided a short-run regression that did not +reproduce in the adjacent longer comparison. These are measurement defaults; +the server batching and transport defaults remain 32 records. +`--warmup-duration` and `--duration` allow explicit overrides. It retains exact commands, client source +snapshots, binary hashes, JSON/Markdown reports and copied fixture cleanup/log +evidence. The runner uses an explicit `--server-receipt` or automatically loads +the adjacent `.receipt.json`, passing frozen local receipts to the lab while +retaining locked module receipts as provenance. It checks the server SHA256 +before and after every case, and the summarizer requires each report's actual +binary, source and pinned SDK/protocol/backend images to match its receipt. +Use a fresh output root for each server or batching candidate: + +```sh +python3 scripts/run_performance_matrix.py --backend mongo \ + --weir .tools/performance-before-server --mongod .tools/mongod \ + --output results/local/mongo-before --bulk +python3 scripts/run_performance_matrix.py --backend mongo \ + --weir .tools/performance-after-server \ + --server-receipt .tools/performance-after-server.receipt.json \ + --mongod .tools/mongod --output results/local/mongo-after --bulk +python3 scripts/summarize_performance.py --backend mongo \ + --before results/local/mongo-before --after results/local/mongo-after \ + --output results/local/mongo-comparison +``` + +`--bulk` appends read and write cases using 32 records per call. Bulk cases use +one client process with 8/32 workers; ordinary and Lua cases use four independent +client processes. Batching limits 128/256 need the bulk cases or higher single +request concurrency to exercise batches above 32 records. + +The summary recomputes throughput from all successful records and measured time, +preserves each round's throughput and p95, and checks that every compared case +has matching client source/binary hashes, workload parameters and paired round +order. Missing core cases are marked partial; a one-sided bulk case is recorded +as missing. Server CPU time and Go allocation costs use Prometheus counter +differences around each timed stage, including small metrics collection overhead. +RSS is a sampled maximum. Physical database commands are reported only when the +harness captured counters around the full stage; an unavailable bulk count is +not inferred from adapter calls. Histograms without separate 31 and 32 boundaries +cannot determine the exact 32-record full-batch fraction. ## Other workload modes @@ -167,6 +240,8 @@ concurrency and grouping controls on dispatch. Workflow artifacts preserve raw reports and owned-resource cleanup receipts. Generated measurements belong in ignored output directories under `results/`. +The minimal `results/go.mod` isolates historical generated Go evidence from root +unit tests and dependency checks without moving or deleting old measurements. See [results/README.md](results/README.md) for the evidence required when comparing runs. The current checkout documents and tests the current contract; previous source revisions and measurements remain available in Git history. diff --git a/cmd/weir-lab/main.go b/cmd/weir-lab/main.go index e56aa1f0..45640a1e 100644 --- a/cmd/weir-lab/main.go +++ b/cmd/weir-lab/main.go @@ -46,8 +46,9 @@ type labOptions struct { warmupDuration time.Duration duration time.Duration databaseCPUs float64 - storeConcurrency int - workingMemoryMiB int + pendingRecords int + exchangeBytes int + serverReceipt string clientProcesses int backendBatchLimit int luaMutations bool @@ -72,7 +73,7 @@ func main() { flag.IntVar(&opts.warmup, "warmup", 1000, "untimed operations per path") flag.IntVar(&opts.records, "records", 2048, "fixed working-set document count") flag.IntVar(&opts.payload, "payload-bytes", 1024, "deterministic document payload size") - flag.IntVar(&opts.concurrency, "concurrency", 8, "matched client workers and Weir Store concurrency") + flag.IntVar(&opts.concurrency, "concurrency", 8, "client workers for finite diagnostics") flag.IntVar(&opts.rounds, "rounds", 3, "alternating paired rounds") flag.IntVar(&opts.writePercent, "write-percent", 10, "write fraction from 0 through 100") flag.BoolVar(&opts.luaMutations, "lua-mutations", false, "single-record read-modify-write: native transaction/OCC versus Weir Lua; no client batching") @@ -80,12 +81,13 @@ func main() { flag.StringVar(&opts.levels, "concurrency-levels", "8,32,128", "ascending total client concurrency, at most 512 in single-request mode") flag.StringVar(&opts.batches, "batch-sizes", "1", "records per call: saturation requires 1; bulk-saturation accepts 1 through 64") flag.IntVar(&opts.clientProcesses, "client-processes", 4, "independent client OS processes; each level is evenly divided into concurrent workers and its own connection pool") - flag.IntVar(&opts.backendBatchLimit, "backend-batch-limit", 32, "server adapter aggregation limit, independent of one-record client requests; 1 disables cross-request aggregation") + flag.IntVar(&opts.backendBatchLimit, "backend-batch-limit", 0, "batching.max_operations override; 0 uses server default, 1 disables aggregation") flag.DurationVar(&opts.warmupDuration, "warmup-duration", 10*time.Second, "untimed warmup per saturation stage") flag.DurationVar(&opts.duration, "duration", 20*time.Second, "timed duration per saturation stage, including completion joins") flag.Float64Var(&opts.databaseCPUs, "database-cpus", 1, "enforced CPU quota for each owned Docker database; native Mongo uses all host CPUs") - flag.IntVar(&opts.storeConcurrency, "store-concurrency", 32, "fixed Weir backend concurrency in saturation mode, independently of client workers") - flag.IntVar(&opts.workingMemoryMiB, "store-working-memory-mib", 0, "backend working memory override; zero sizes each backend for the configured concurrency") + flag.IntVar(&opts.pendingRecords, "pending-records", 0, "transport.max_pending_records override; 0 uses server default") + flag.IntVar(&opts.exchangeBytes, "exchange-bytes", 0, "backend.max_exchange_bytes override; 0 uses server default") + flag.StringVar(&opts.serverReceipt, "server-receipt", "", "verified build receipt for an explicit local source server") flag.Parse() ctx, cancel := signal.NotifyContext(context.Background(), os.Interrupt, syscall.SIGTERM) defer cancel() @@ -118,7 +120,7 @@ func runLab(ctx context.Context, opts labOptions) (resultErr error) { if err != nil { return err } - if len(levels) < 2 || len(levels) > 12 || len(batches) > 8 || opts.duration < 5*time.Second || opts.duration > 10*time.Minute || opts.warmupDuration < time.Second || opts.warmupDuration > time.Minute || opts.storeConcurrency < 1 || opts.storeConcurrency > 32 { + if len(levels) < 2 || len(levels) > 12 || len(batches) > 8 || opts.duration < 5*time.Second || opts.duration > 10*time.Minute || opts.warmupDuration < time.Second || opts.warmupDuration > time.Minute { return errors.New("invalid saturation matrix or durations") } previous := 0 @@ -152,8 +154,8 @@ func runLab(ctx context.Context, opts labOptions) (resultErr error) { if opts.databaseCPUs <= 0 || opts.databaseCPUs > float64(runtime.NumCPU()) { return errors.New("database CPU quota exceeds host budget") } - if opts.backendBatchLimit < 1 || opts.backendBatchLimit > 1024 { - return errors.New("backend-batch-limit must be 1 through 1024") + if opts.backendBatchLimit < 0 || opts.backendBatchLimit > 1024 || opts.pendingRecords < 0 || opts.pendingRecords > 4096 || opts.exchangeBytes < 0 || opts.exchangeBytes > 1<<30 || opts.exchangeBytes > 0 && opts.exchangeBytes < 4<<20 { + return errors.New("invalid batching, pending-records or exchange-bytes override") } maxWorkers := 32 if opts.mode != "fixed" { @@ -181,13 +183,13 @@ func runLab(ctx context.Context, opts labOptions) (resultErr error) { } digest := sha256.Sum256(data) provenance := map[string]string{ - "weir_binary_sha256": hex.EncodeToString(digest[:]), - "topology": "native client and Weir on one host; dedicated database Docker containers with loopback ports", - "client_operation_timeout": businessTimeout.String(), - "weir_backend_timeout": businessTimeout.String(), - "weir_max_batch_operations": fmt.Sprint(opts.backendBatchLimit), - "weir_ingress_max_connections": "64", - "host_cpus": fmt.Sprint(runtime.NumCPU()), + "weir_binary_sha256": hex.EncodeToString(digest[:]), + "topology": "native client and Weir on one host; dedicated database Docker containers with loopback ports", + "client_operation_timeout": businessTimeout.String(), + "weir_batching_max_operations_override": fmt.Sprint(opts.backendBatchLimit), + "weir_transport_max_pending_records_override": fmt.Sprint(opts.pendingRecords), + "weir_backend_max_exchange_bytes_override": fmt.Sprint(opts.exchangeBytes), + "host_cpus": fmt.Sprint(runtime.NumCPU()), } identityRaw, err := exec.CommandContext(ctx, binary, "version").Output() if err != nil { @@ -201,14 +203,14 @@ func runLab(ctx context.Context, opts labOptions) (resultErr error) { } provenance["weir_source"] = identity.Revision if opts.mongoBinary != "" && slices.Contains(backends, "mongo") { - provenance["topology"] = "native client, Weir and isolated MongoDB on one host; Elasticsearch in a dedicated loopback Docker container" + provenance["topology"] = "native client, Weir and isolated MongoDB on one host; native MongoDB has no enforced CPU quota" } if pinsRaw, err := os.ReadFile("versions.json"); err == nil { var pins map[string]string if err := json.Unmarshal(pinsRaw, &pins); err != nil { return err } - if identity.Revision != pins["weir_source"] { + if opts.serverReceipt == "" && identity.Revision != pins["weir_source"] { return errors.New("Weir binary does not match versions.json") } for _, key := range []string{"sdk", "protocol", "mongodb_image", "elasticsearch_image"} { @@ -217,6 +219,36 @@ func runLab(ctx context.Context, opts labOptions) (resultErr error) { } else { return errors.New("run the lab from the repository root with its versions.json lock") } + if opts.serverReceipt != "" { + receiptRaw, err := os.ReadFile(opts.serverReceipt) + if err != nil { + return err + } + var receipt struct { + Source string `json:"source"` + SHA256 string `json:"sha256"` + SourceDiffSHA256 string `json:"source_diff_sha256"` + SourceTreeSHA256 string `json:"source_tree_sha256"` + SourceDirty bool `json:"source_dirty"` + Identity struct { + Revision string `json:"revision"` + } `json:"identity"` + } + if err := json.Unmarshal(receiptRaw, &receipt); err != nil { + return err + } + if receipt.Identity.Revision != identity.Revision || receipt.SHA256 != provenance["weir_binary_sha256"] || len(receipt.Source) != 40 || len(receipt.SourceTreeSHA256) != 64 || len(receipt.SourceDiffSHA256) != 64 { + return errors.New("local server build receipt does not match binary identity, checksum or source evidence") + } + provenance["weir_binary_revision"] = identity.Revision + provenance["weir_source"] = receipt.Source + provenance["weir_source_diff_sha256"] = receipt.SourceDiffSHA256 + provenance["weir_source_tree_sha256"] = receipt.SourceTreeSHA256 + provenance["weir_source_dirty"] = fmt.Sprint(receipt.SourceDirty) + provenance["weir_source_kind"] = "explicit local source build; unpublished changes may be present" + } else { + provenance["weir_source_kind"] = "locked immutable module source" + } if revision, err := exec.CommandContext(ctx, "git", "rev-parse", "HEAD").Output(); err == nil { provenance["test_source"] = strings.TrimSpace(string(revision)) } @@ -229,24 +261,10 @@ func runLab(ctx context.Context, opts labOptions) (resultErr error) { if err := os.Mkdir(opts.output, 0o755); err != nil { return fmt.Errorf("requires a new output directory; preserving existing reports: %w", err) } - storeWorkers := opts.concurrency - if opts.mode != "fixed" { - storeWorkers = opts.storeConcurrency - } - provenance["weir_store_concurrency"] = fmt.Sprint(storeWorkers) - provenance["weir_ingress_max_sessions"] = fmt.Sprint(max(64, opts.concurrency)) - provenance["database_docker_cpu_quota"] = fmt.Sprint(opts.databaseCPUs) - workspace, memoryMiB, err := fixtureMemoryBudget(opts, backends, storeWorkers) - if err != nil { - return err - } - for backend, memory := range workspace { - provenance["weir_"+backend+"_working_memory_mib"] = fmt.Sprint(memory) + if opts.mongoBinary == "" || slices.Contains(backends, "search") { + provenance["database_docker_cpu_quota"] = fmt.Sprint(opts.databaseCPUs) } - maxReadBytes := min(2<<20, ((opts.payload+4096+4095)/4096)*4096) - provenance["weir_max_read_size_bytes"] = fmt.Sprint(maxReadBytes) - provenance["weir_memory_budget"] = fmt.Sprintf("%dMiB (declared process admission budget; not an OS reservation)", memoryMiB) - start := fixture.Options{WeirBinary: binary, MongoBinary: opts.mongoBinary, Backends: backends, OwnerCount: 1, StoreConcurrency: storeWorkers, BatchSize: opts.backendBatchLimit, MaxReadSizeBytes: maxReadBytes, BackendTimeout: businessTimeout, IngressSessions: max(64, opts.concurrency), DatabaseCPUs: opts.databaseCPUs, ProcessMemoryMiB: memoryMiB, WorkingMemoryMiB: workspace} + start := fixture.Options{WeirBinary: binary, MongoBinary: opts.mongoBinary, Backends: backends, OwnerCount: 1, BatchSize: opts.backendBatchLimit, PendingRecords: opts.pendingRecords, ExchangeBytes: opts.exchangeBytes, DatabaseCPUs: opts.databaseCPUs} cluster, err := fixture.Start(ctx, start) if err != nil { return err @@ -296,40 +314,6 @@ func runLab(ctx context.Context, opts labOptions) (resultErr error) { return nil } -func fixtureMemoryBudget(opts labOptions, backends []string, workers int) (map[string]int, int, error) { - if workers < 1 || workers > 65536/2 || opts.workingMemoryMiB < 0 || opts.workingMemoryMiB > 65536 { - return nil, 0, errors.New("invalid backend memory parameters") - } - workspace := make(map[string]int, len(backends)) - // The pinned adapters bound one 2MiB-read batch by at most 41MiB for - // MongoDB (native reply, guard and scratch) and 96MiB for Search. - // Streaming ingress uses 32MiB per admitted stream. Add 1GiB for Store input/results and other framing. - memoryMiB := max(64, opts.concurrency)*32 + 1024 - for _, backend := range backends { - perBatchMiB := 41 - if opts.luaMutations { - perBatchMiB = 64 - } - if backend == "search" { - perBatchMiB = 96 - } - memory := opts.workingMemoryMiB - if memory == 0 { - if workers > 65536/perBatchMiB { - return nil, 0, errors.New("backend concurrency cannot fit the process memory envelope") - } - memory = max(384, workers*perBatchMiB) - } - workspace[backend] = memory - memoryMiB += memory + workers*2 - } - memoryMiB = (memoryMiB + 1023) / 1024 * 1024 - if memoryMiB > 65536 { - return nil, 0, errors.New("combined ingress and backend memory cannot fit the process envelope") - } - return workspace, memoryMiB, nil -} - func parsePositiveList(raw string) ([]int, error) { values := strings.Split(raw, ",") result := make([]int, 0, len(values)) diff --git a/cmd/weir-lab/memory_test.go b/cmd/weir-lab/memory_test.go deleted file mode 100644 index 97cd72bf..00000000 --- a/cmd/weir-lab/memory_test.go +++ /dev/null @@ -1,38 +0,0 @@ -package main - -import "testing" - -func TestFixtureMemoryCoversActualBackendConcurrency(t *testing.T) { - opts := labOptions{concurrency: 64} - backends := []string{"mongo", "search"} - workspace, processMiB, err := fixtureMemoryBudget(opts, backends, 32) - if err != nil { - t.Fatal(err) - } - if workspace["mongo"]<<20 < 32*((32<<20)+(128<<10)+(8<<20)) || workspace["search"] < 32*96 { - t.Fatal("workspace silently throttles the configured backend concurrency", workspace) - } - reservedMiB := 64 + 64/4 + 64*32 + workspace["mongo"] + workspace["search"] + 2*(64+32*2) - if processMiB < reservedMiB || processMiB != 8192 { - t.Fatal("ingress and both backend envelopes do not fit the process", processMiB, reservedMiB) - } - opts.workingMemoryMiB = 384 - workspace, processMiB, err = fixtureMemoryBudget(opts, backends, 32) - if err != nil || workspace["mongo"] != 384 || workspace["search"] != 384 || processMiB != 4096 { - t.Fatal("explicit workspace budget was not honored", workspace, processMiB, err) - } - opts.workingMemoryMiB = 0 - opts.concurrency = 512 - workspace, processMiB, err = fixtureMemoryBudget(opts, backends, 32) - if err != nil || workspace["mongo"] != 1312 || workspace["search"] != 3072 || processMiB != 22528 { - t.Fatal("512 independent requests do not fit the explicitly declared ingress envelope", workspace, processMiB, err) - } - if _, _, err := fixtureMemoryBudget(opts, backends, int(^uint(0)>>1)); err == nil { - t.Fatal("overflowing execution envelope accepted") - } - opts.luaMutations = true - workspace, _, err = fixtureMemoryBudget(opts, backends, 32) - if err != nil || workspace["mongo"] != 32*64 || workspace["search"] != 32*96 { - t.Fatal("Lua transaction scratch silently reduced requested backend capacity", workspace, err) - } -} diff --git a/go.mod b/go.mod index 4e4c0f0f..11c6d51b 100644 --- a/go.mod +++ b/go.mod @@ -3,8 +3,8 @@ module github.com/batchstream/weir-tests go 1.27.1 require ( - github.com/batchstream/weir-go v0.9.0 - github.com/batchstream/weir-protocol v0.7.0 + github.com/batchstream/weir-go v0.10.0 + github.com/batchstream/weir-protocol v0.8.0 go.mongodb.org/mongo-driver/v2 v2.9.1 google.golang.org/grpc v1.83.2 ) diff --git a/go.sum b/go.sum index 35f74460..67c1b9e3 100644 --- a/go.sum +++ b/go.sum @@ -1,7 +1,11 @@ github.com/batchstream/weir-go v0.9.0 h1:a4pGdTJd9/5Y8QMTz37ipHpfSXztpPB84Wm4Wl0fc7A= github.com/batchstream/weir-go v0.9.0/go.mod h1:lAlK3m/CBl+WZUhMIKwhGdqoeb4wnoteKWppyCfgZLo= +github.com/batchstream/weir-go v0.10.0 h1:D4sv8YoXqUP44W+jd/DsbsW6NkZqK5FTbNFWqhDaABA= +github.com/batchstream/weir-go v0.10.0/go.mod h1:ud7wNpCOrnNuIfwu7CPxkveku9qNi9NrBxuNv44NBfk= github.com/batchstream/weir-protocol v0.7.0 h1:+LSSLPCWv8GL8RgkH06LYNT8wDN5tFRAs0rle1Ym4sY= github.com/batchstream/weir-protocol v0.7.0/go.mod h1:hHmwPoE9yHYAz7Auc3a14Ijo/uxAef4sQ9nBtyXNcGc= +github.com/batchstream/weir-protocol v0.8.0 h1:A+diMm2VPP0YoAsnVQJTW71rf1vV+IVMyUQ02zta2Oo= +github.com/batchstream/weir-protocol v0.8.0/go.mod h1:hHmwPoE9yHYAz7Auc3a14Ijo/uxAef4sQ9nBtyXNcGc= github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs= github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= diff --git a/internal/benchmark/saturation_report.go b/internal/benchmark/saturation_report.go index 6f4cc1d9..e5fca770 100644 --- a/internal/benchmark/saturation_report.go +++ b/internal/benchmark/saturation_report.go @@ -28,9 +28,7 @@ func (r *SaturationReport) Markdown() string { if r.Parameters.OperationTimeout != "" { fmt.Fprintf(&text, "Complete business-request budget for both paths: %s.\n\n", r.Parameters.OperationTimeout) } - if timeout := r.Provenance["weir_backend_timeout"]; timeout != "" { - fmt.Fprintf(&text, "Configured Weir Store backend budget: %s; the active caller deadline also applies.\n\n", timeout) - } + if r.Parameters.ClientProcesses > 0 { fmt.Fprintf(&text, "%d independent OS client processes. Each worker sends one business request containing one record, waits for its acknowledged result, and validates it. p50/p95/p99 describe individual requests; histograms are pooled rather than averaging process quantiles.\n\n", r.Parameters.ClientProcesses) } else { @@ -42,17 +40,17 @@ func (r *SaturationReport) Markdown() string { fmt.Fprintf(&text, "| %d | %d | %d | %s | %d | %d | %d | %d | %.1f | %.3f | %.1f | %.1f | %.1f | %t |\n", pair.BatchSize, pair.Concurrency, pair.Round, result.Path, result.Reads, result.Writes, result.Succeeded, result.Errors+result.Indeterminate, result.OperationsPerSec, float64(result.Latency.P95NS)/1e6, result.Resources.MeanCPUPercent, result.Resources.FullCPUFraction*100, result.Resources.Coverage*100, result.Verified) } } - text.WriteString("\nMeasured Weir adapter and streaming RPC evidence (warmup, reset and postflight excluded):\n\n| Batch | Workers | Round | Adapter invocations | Mean adapter batch | Execute RPCs | Queue mean ms | Adapter mean ms | Rejections | Backend concurrency limit |\n| ---: | ---: | ---: | ---: | ---: | --- | ---: | ---: | ---: | ---: |\n") + text.WriteString("\nMeasured Weir adapter and streaming RPC evidence (warmup, reset and postflight excluded):\n\n| Batch | Workers | Round | Adapter invocations | Mean adapter batch | Execute RPCs | Queue mean ms | Adapter mean ms | Rejections |\n| ---: | ---: | ---: | ---: | ---: | --- | ---: | ---: | ---: |\n") var metricWarnings []string for _, pair := range r.Pairs { metrics := pair.Weir.ServerMetrics if metrics == nil { - fmt.Fprintf(&text, "| %d | %d | %d | unavailable | unavailable | unavailable | unavailable | unavailable | unavailable | unavailable |\n", pair.BatchSize, pair.Concurrency, pair.Round) + fmt.Fprintf(&text, "| %d | %d | %d | unavailable | unavailable | unavailable | unavailable | unavailable | unavailable |\n", pair.BatchSize, pair.Concurrency, pair.Round) continue } - fmt.Fprintf(&text, "| %d | %d | %d | %s | %s | %s | %s | %s | %s | %s |\n", pair.BatchSize, pair.Concurrency, pair.Round, + fmt.Fprintf(&text, "| %d | %d | %d | %s | %s | %s | %s | %s | %s |\n", pair.BatchSize, pair.Concurrency, pair.Round, metricCounter(metrics, "weir_store_executions_total"), metricMean(metrics.AdapterBatchAverage, 1), metricCounter(metrics, "weir_rpc_completions_total:execute"), - metricMean(metrics.QueueWaitMeanSeconds, 1000), metricMean(metrics.ExecutionMeanSeconds, 1000), metricCounter(metrics, "weir_store_rejections_total"), metricMean(metrics.ConcurrencyLimit, 1)) + metricMean(metrics.QueueWaitMeanSeconds, 1000), metricMean(metrics.ExecutionMeanSeconds, 1000), metricCounter(metrics, "weir_store_rejections_total")) if metrics.Unavailable != "" { warning := fmt.Sprintf("Metrics unavailable (batch %d, workers %d, round %d): %s.", pair.BatchSize, pair.Concurrency, pair.Round, strings.ReplaceAll(metrics.Unavailable, "\n", " ")) metricWarnings = append(metricWarnings, warning) diff --git a/internal/benchmark/saturation_test.go b/internal/benchmark/saturation_test.go index 07054f04..1c391684 100644 --- a/internal/benchmark/saturation_test.go +++ b/internal/benchmark/saturation_test.go @@ -210,9 +210,9 @@ func TestDurationMeasurementJoinsInFlightBatchesAndCountsLogicalWork(t *testing. } } -func TestSaturationReportRecordsClientAndBackendRequestBudgets(t *testing.T) { +func TestSaturationReportRecordsCompleteRequestBudget(t *testing.T) { parameters := SaturationParameters{OperationTimeout: "10s", ClientProcesses: 4} - report := &SaturationReport{Parameters: parameters, Provenance: map[string]string{"weir_backend_timeout": "10s", "client_operation_timeout": "10s"}} + report := &SaturationReport{Parameters: parameters, Provenance: map[string]string{"client_operation_timeout": "10s"}} raw, err := json.Marshal(report) if err != nil { t.Fatal(err) @@ -226,11 +226,11 @@ func TestSaturationReportRecordsClientAndBackendRequestBudgets(t *testing.T) { if err := json.Unmarshal(raw, &decoded); err != nil { t.Fatal(err) } - if decoded.Parameters.OperationTimeout != "10s" || decoded.Provenance["client_operation_timeout"] != "10s" || decoded.Provenance["weir_backend_timeout"] != "10s" { + if decoded.Parameters.OperationTimeout != "10s" || decoded.Provenance["client_operation_timeout"] != "10s" { t.Fatal("raw report lost the matched complete-request policy", string(raw)) } markdown := report.Markdown() - if !strings.Contains(markdown, "Complete business-request budget for both paths: 10s") || !strings.Contains(markdown, "Configured Weir Store backend budget: 10s") { + if !strings.Contains(markdown, "Complete business-request budget for both paths: 10s") { t.Fatal("human-readable report hid the request budgets", markdown) } } diff --git a/internal/benchmark/server_metrics.go b/internal/benchmark/server_metrics.go index e2f65b04..0cb3c92e 100644 --- a/internal/benchmark/server_metrics.go +++ b/internal/benchmark/server_metrics.go @@ -16,7 +16,6 @@ type ServerMetrics struct { After observe.Snapshot `json:"after"` Deltas map[string]float64 `json:"timed_counter_deltas,omitempty"` AdapterBatchAverage *float64 `json:"timed_adapter_batch_average,omitempty"` - ConcurrencyLimit *float64 `json:"configured_backend_concurrency_limit,omitempty"` QueueWaitMeanSeconds *float64 `json:"timed_queue_wait_mean_seconds,omitempty"` ExecutionMeanSeconds *float64 `json:"timed_adapter_execution_mean_seconds,omitempty"` RPCUnavailable map[string]string `json:"rpc_observation_unavailable,omitempty"` @@ -42,13 +41,7 @@ func serverMetricDelta(before, after observe.Snapshot, store string) *ServerMetr } metrics.Deltas[name] = value } - beforeLimit, foundBefore := before.Sum("weir_store_concurrency_limit", labels) - afterLimit, foundAfter := after.Sum("weir_store_concurrency_limit", labels) - if !foundBefore || !foundAfter || beforeLimit < 1 || afterLimit != beforeLimit { - err = errors.Join(err, errors.New("configured backend concurrency limit unavailable or changed during measurement")) - } else { - metrics.ConcurrencyLimit = &afterLimit - } + for _, method := range []string{"execute"} { methodLabels := map[string]string{"method": method} value, deltaErr := observe.Delta(before, after, "weir_rpc_completions_total", methodLabels) diff --git a/internal/benchmark/server_metrics_test.go b/internal/benchmark/server_metrics_test.go index d4852a59..cbc00aa9 100644 --- a/internal/benchmark/server_metrics_test.go +++ b/internal/benchmark/server_metrics_test.go @@ -24,7 +24,6 @@ func metricsSnapshot(t *testing.T, measured bool) observe.Snapshot { } fmt.Fprintf(&raw, "%s{store=\"mongo\"} %g\n%s{store=\"search\"} 999999\n", name, value, name) } - raw.WriteString("weir_store_concurrency_limit{store=\"mongo\"} 32\nweir_store_concurrency_limit{store=\"search\"} 16\n") rpc := 10 if measured { rpc += 100 @@ -48,9 +47,7 @@ func TestServerMetricsUseMeasuredDeltaAndReportMissingObservations(t *testing.T) if metrics.QueueWaitMeanSeconds == nil || *metrics.QueueWaitMeanSeconds != .001 || metrics.ExecutionMeanSeconds == nil || *metrics.ExecutionMeanSeconds != .005 { t.Fatal("queue/adapter means lost", metrics) } - if metrics.ConcurrencyLimit == nil || *metrics.ConcurrencyLimit != 32 { - t.Fatal("configured limit became a delta or came from another store", metrics) - } + if len(metrics.RPCUnavailable) != 0 { t.Fatal("observed stream RPC unexpectedly unavailable") } @@ -72,17 +69,7 @@ func TestServerMetricsUseMeasuredDeltaAndReportMissingObservations(t *testing.T) if metrics.Unavailable == "" || metrics.AdapterBatchAverage != nil { t.Fatal("missing metrics became zero batching", metrics) } - after = metricsSnapshot(t, true) - for index := range after.Metrics { - metric := &after.Metrics[index] - if metric.Name == "weir_store_concurrency_limit" && metric.Labels["store"] == "mongo" { - metric.Value = 16 - } - } - metrics = serverMetricDelta(before, after, "mongo") - if metrics.Unavailable == "" || metrics.ConcurrencyLimit != nil || metrics.AdapterBatchAverage != nil { - t.Fatal("changing concurrency configuration produced comparable evidence", metrics) - } + } func TestSingleRequestRPCProofRequiresActiveMethodCounts(t *testing.T) { diff --git a/internal/fixture/backends.go b/internal/fixture/backends.go index 7d5b5a87..c75aeff0 100644 --- a/internal/fixture/backends.go +++ b/internal/fixture/backends.go @@ -170,7 +170,7 @@ func (c *Cluster) readySearch(ctx context.Context) error { var acknowledged struct { Acknowledged bool `json:"acknowledged"` } - template := `{"index_patterns":["*"],"priority":1000,"template":{"settings":{"number_of_shards":1,"number_of_replicas":0}}}` + template := `{"index_patterns":["weirtest_*","weir_*"],"priority":1000,"template":{"settings":{"number_of_shards":1,"number_of_replicas":0}}}` templateRequest := searchRequest{Method: http.MethodPut, URL: c.SearchURL + "/_index_template/weir-tests", Body: template, Result: &acknowledged} // Cluster-state publication can exceed a readiness probe at small CPU quotas. // The complete startup context and this request still bound the wait. diff --git a/internal/fixture/fixture.go b/internal/fixture/fixture.go index 9ef65fb8..226c4df1 100644 --- a/internal/fixture/fixture.go +++ b/internal/fixture/fixture.go @@ -32,18 +32,14 @@ const ownerLabel = "io.batchstream.weir-tests.owner" type Options struct { WeirBinary string // MongoBinary explicitly selects a native pinned mongod; empty uses Docker. - MongoBinary string - Backends []string - OwnerCount int - DiscoveryOnly bool - StoreConcurrency int - BatchSize int - MaxReadSizeBytes int - BackendTimeout time.Duration - IngressSessions int - DatabaseCPUs float64 - ProcessMemoryMiB int - WorkingMemoryMiB map[string]int + MongoBinary string + Backends []string + OwnerCount int + DiscoveryOnly bool + BatchSize int + PendingRecords int + ExchangeBytes int + DatabaseCPUs float64 } type Node struct { @@ -113,20 +109,11 @@ func Start(ctx context.Context, options Options) (*Cluster, error) { } func validateOptions(options Options) (Options, error) { - if options.BackendTimeout == 0 { - options.BackendTimeout = 2 * time.Second - } - if options.BackendTimeout < 0 { - return options, errors.New("backend timeout must be positive") - } - if options.IngressSessions == 0 { - options.IngressSessions = 64 - } if options.DatabaseCPUs == 0 { options.DatabaseCPUs = 2 } - if options.IngressSessions < 1 || options.IngressSessions > 512 || options.DatabaseCPUs <= 0 || options.DatabaseCPUs > float64(runtime.NumCPU()) { - return options, errors.New("invalid ingress sessions or database CPU budget") + if options.DatabaseCPUs <= 0 || options.DatabaseCPUs > float64(runtime.NumCPU()) { + return options, errors.New("invalid database CPU budget") } if runtime.GOOS != "linux" && runtime.GOOS != "darwin" { return options, errors.New("fixture requires Linux or macOS") @@ -172,35 +159,9 @@ func validateOptions(options Options) (Options, error) { if options.OwnerCount < 1 || options.OwnerCount > 8 { return options, errors.New("OwnerCount must be between 1 and 8") } - if options.StoreConcurrency == 0 { - options.StoreConcurrency = 2 - } - if options.MaxReadSizeBytes == 0 { - options.MaxReadSizeBytes = 2 << 20 - } - if options.MaxReadSizeBytes < 1024 || options.MaxReadSizeBytes > 2<<20 { - return options, errors.New("invalid maximum read document size") - } - if options.BatchSize == 0 { - options.BatchSize = 32 - } - if options.StoreConcurrency < 1 || options.BatchSize < 1 { - return options, errors.New("StoreConcurrency or BatchSize exceeds Weir bounds") + if options.BatchSize < 0 || options.BatchSize > 1024 || options.PendingRecords < 0 || options.PendingRecords > 4096 || options.ExchangeBytes < 0 || options.ExchangeBytes > 1<<30 || options.ExchangeBytes > 0 && options.ExchangeBytes < 4<<20 { + return options, errors.New("invalid batching, transport or exchange limit") } - if options.ProcessMemoryMiB == 0 { - options.ProcessMemoryMiB = 8192 - } - if options.ProcessMemoryMiB < 64 || options.ProcessMemoryMiB > 65536 { - return options, errors.New("invalid process memory envelope") - } - workspace := make(map[string]int, len(options.WorkingMemoryMiB)) - for backend, memory := range options.WorkingMemoryMiB { - if !slices.Contains(options.Backends, backend) || memory < 24 || memory > options.ProcessMemoryMiB { - return options, errors.New("invalid backend working memory envelope") - } - workspace[backend] = memory - } - options.WorkingMemoryMiB = workspace return options, nil } @@ -285,22 +246,29 @@ func (c *Cluster) writeConfiguration(index int) error { if !node.Owner { discovery["group"] = "directory-" + c.owner } - transport := map[string]any{"max_connections": 64, "max_sessions": c.options.IngressSessions} - basic := map[string]any{"listeners": listeners, "diagnostics": diagnostics, "discovery": discovery, "transport": transport, "memory": fmt.Sprintf("%dMiB", c.options.ProcessMemoryMiB)} + transport := map[string]any{} + if c.options.PendingRecords > 0 { + transport["max_pending_records"] = c.options.PendingRecords + } + basic := map[string]any{"listeners": listeners, "diagnostics": diagnostics, "discovery": discovery, "transport": transport} if err := writeJSON(c.nodeFile(index, "config.json"), basic); err != nil { return err } stores := make([]map[string]any, 0, len(c.options.Backends)) if node.Owner { for _, backend := range c.options.Backends { - store := map[string]any{"name": backend, "max_concurrency": c.options.StoreConcurrency, "max_batch_operations": c.options.BatchSize, "max_read_size": fmt.Sprintf("%dB", c.options.MaxReadSizeBytes), "backend_timeout": c.options.BackendTimeout.String()} - if memory, configured := c.options.WorkingMemoryMiB[backend]; configured { - store["working_memory"] = fmt.Sprintf("%dMiB", memory) + adapter := map[string]any{} + if c.options.ExchangeBytes > 0 { + adapter["max_exchange_bytes"] = fmt.Sprintf("%dB", c.options.ExchangeBytes) } if backend == "mongo" { - store["mongodb"] = map[string]any{"uri": c.MongoURI} + adapter["mongodb"] = map[string]any{"uri": c.MongoURI} } else { - store["search"] = map[string]any{"url": c.SearchURL} + adapter["search"] = map[string]any{"url": c.SearchURL} + } + store := map[string]any{"name": backend, "backend": adapter} + if c.options.BatchSize > 0 { + store["batching"] = map[string]any{"max_operations": c.options.BatchSize} } stores = append(stores, store) } diff --git a/internal/fixture/fixture_test.go b/internal/fixture/fixture_test.go index 8a12bbb9..137bfed5 100644 --- a/internal/fixture/fixture_test.go +++ b/internal/fixture/fixture_test.go @@ -34,12 +34,11 @@ func TestValidateOptionsRejectsInvalidBeforeExternalWork(t *testing.T) { {WeirBinary: binary, Backends: []string{"redis"}}, {WeirBinary: binary, OwnerCount: -1}, {WeirBinary: binary, OwnerCount: 9}, - {WeirBinary: binary, StoreConcurrency: -1}, {WeirBinary: binary, BatchSize: -1}, - {WeirBinary: binary, MaxReadSizeBytes: 1023}, - {WeirBinary: binary, MaxReadSizeBytes: (2 << 20) + 1}, - {WeirBinary: binary, BackendTimeout: -time.Second}, - {WeirBinary: binary, ProcessMemoryMiB: -1}, + {WeirBinary: binary, BatchSize: 1025}, + {WeirBinary: binary, PendingRecords: -1}, + {WeirBinary: binary, PendingRecords: 4097}, + {WeirBinary: binary, ExchangeBytes: (4 << 20) - 1}, {WeirBinary: binary, MongoBinary: filepath.Join(t.TempDir(), "missing-mongod")}, } for _, options := range cases { @@ -54,23 +53,14 @@ func TestValidateOptionsRejectsInvalidBeforeExternalWork(t *testing.T) { t.Fatal(err) } backends[0] = "search" - if validated.Backends[0] != "mongo" || validated.OwnerCount != 1 || validated.StoreConcurrency != 2 || validated.BatchSize != 32 || validated.BackendTimeout != 2*time.Second { + if validated.Backends[0] != "mongo" || validated.OwnerCount != 1 || validated.BatchSize != 0 || validated.PendingRecords != 0 || validated.ExchangeBytes != 0 { t.Fatalf("invalid defaults or retained caller slice: %+v", validated) } large := Options{WeirBinary: binary, BatchSize: 513} if _, err := validateOptions(large); err != nil { t.Fatal("valid physical batch greater than 128 rejected:", err) } - workspace := map[string]int{"mongo": 384} - parallel := Options{WeirBinary: binary, StoreConcurrency: 33, WorkingMemoryMiB: workspace} - prepared, err := validateOptions(parallel) - if err != nil { - t.Fatal("backend concurrency retained an arbitrary cap", err) - } - workspace["mongo"] = 1024 - if prepared.WorkingMemoryMiB["mongo"] != 384 { - t.Fatal("caller changed the admitted fixture memory envelope") - } + ctx := context.Background() missing := Options{} if _, err := Start(ctx, missing); err == nil { @@ -154,7 +144,7 @@ func TestPublishedPortMustBeOneLoopbackBinding(t *testing.T) { } func TestConfigurationSharesStoresWithoutRetainingDeadEndpoints(t *testing.T) { - options := Options{Backends: []string{"mongo", "search"}, OwnerCount: 2, DiscoveryOnly: true, StoreConcurrency: 4, BatchSize: 16, BackendTimeout: 10 * time.Second} + options := Options{Backends: []string{"mongo", "search"}, OwnerCount: 2, DiscoveryOnly: true, BatchSize: 16, PendingRecords: 128, ExchangeBytes: 8 << 20} c := &Cluster{Directory: t.TempDir(), options: options, owner: "test", MongoURI: "mongodb://127.0.0.1:50001/?directConnection=true", SearchURL: "http://127.0.0.1:50002"} c.Nodes = []Node{ {Application: "127.0.0.1:50100", Peer: "127.0.0.1:50101", Diagnostics: "127.0.0.1:50102", Owner: true}, @@ -190,8 +180,13 @@ func TestConfigurationSharesStoresWithoutRetainingDeadEndpoints(t *testing.T) { } var routing struct { Stores []struct { - Name string `json:"name"` - BackendTimeout string `json:"backend_timeout"` + Name string `json:"name"` + Batching struct { + MaxOperations int `json:"max_operations"` + } `json:"batching"` + Backend struct { + MaxExchangeBytes string `json:"max_exchange_bytes"` + } `json:"backend"` } } if err := json.Unmarshal(routes, &routing); err != nil { @@ -201,8 +196,8 @@ func TestConfigurationSharesStoresWithoutRetainingDeadEndpoints(t *testing.T) { t.Fatal("wrong Store ownership", string(routes)) } for position, store := range routing.Stores { - if store.Name != options.Backends[position] || store.BackendTimeout != "10s" { - t.Fatal("owner backend did not receive the explicit business timeout", string(routes)) + if store.Name != options.Backends[position] || store.Batching.MaxOperations != 16 || store.Backend.MaxExchangeBytes != "8388608B" { + t.Fatal("owner backend did not receive the current batching/exchange overrides", string(routes)) } } } @@ -381,7 +376,7 @@ func TestStoppedProcessFailureIsPreservedInCloseAndReceipt(t *testing.T) { } func TestSingleOwnerConfigurationUsesAnEmptySeedList(t *testing.T) { - input := Options{WeirBinary: executableFile(t), Backends: []string{"mongo"}, OwnerCount: 1, StoreConcurrency: 2, BatchSize: 32} + input := Options{WeirBinary: executableFile(t), Backends: []string{"mongo"}, OwnerCount: 1} options, err := validateOptions(input) if err != nil { t.Fatal(err) @@ -397,8 +392,8 @@ func TestSingleOwnerConfigurationUsesAnEmptySeedList(t *testing.T) { t.Fatal("single-owner config emits unsupported null seeds", string(data), err) } routes, err := os.ReadFile(c.nodeFile(0, "routes.json")) - if err != nil || !bytes.Contains(routes, []byte(`"backend_timeout": "2s"`)) { - t.Fatal("ordinary fixture lost the production backend timeout default", string(routes), err) + if err != nil || bytes.Contains(routes, []byte(`"batching"`)) || bytes.Contains(routes, []byte(`"max_exchange_bytes"`)) { + t.Fatal("ordinary fixture must leave server defaults unchanged", string(routes), err) } } diff --git a/results/README.md b/results/README.md index 674fbba0..b261cd74 100644 --- a/results/README.md +++ b/results/README.md @@ -2,6 +2,9 @@ Use a fresh directory such as `results/local/read` for each run. Generated output is ignored by Git. CI uploads reports and fixture receipts as workflow artifacts. +The minimal `go.mod` in this directory gives evidence data a module boundary, so +historical generated Go files are excluded from root unit tests and dependency +checks. It has no code or dependencies; retain existing measurements in place. A reproducible comparison needs the immutable test/server/SDK/protocol versions, exact command, client and server binary identities, database resource limits, @@ -11,3 +14,9 @@ owned-resource cleanup receipts. Compare runs with the same workload and limits. Observed throughput at a fixed concurrency is distinct from a qualified maximum-capacity comparison. Both paths must meet the database-saturation and next-concurrency plateau criteria described in the repository README. + +Local refactor runs must include the checksummed binary receipt and retained +`*.source` snapshot produced by `scripts/build_local_server.py`. The tree +manifest covers tracked and untracked Go inputs. A source HEAD alone does not +identify an unpublished build. Preserve the exact command and source snapshot +for the benchmark client as well when its checkout is dirty. diff --git a/results/go.mod b/results/go.mod new file mode 100644 index 00000000..8206bdd3 --- /dev/null +++ b/results/go.mod @@ -0,0 +1,3 @@ +module github.com/batchstream/weir-tests/results + +go 1.27.1 diff --git a/scripts/build_local_server.py b/scripts/build_local_server.py new file mode 100644 index 00000000..7d0c79a4 --- /dev/null +++ b/scripts/build_local_server.py @@ -0,0 +1,83 @@ +#!/usr/bin/env python3 +"""Freeze a local Weir Go source tree and build a checksummed benchmark binary.""" +import argparse +import hashlib +import json +import os +from pathlib import Path +import subprocess + + +def digest(data): + return hashlib.sha256(data).hexdigest() + + +def freeze_source(source, output): + source = source.resolve() + files = [] + for directory, directories, names in os.walk(source, followlinks=False): + current = Path(directory) + directories[:] = [name for name in directories + if not name.startswith('.') and name not in ('vendor', 'credentials') + and not (current / name).is_symlink() + and (current != source or name in ('cmd', 'internal', 'pkg', 'api', 'tests'))] + for name in names: + if name.startswith('.') or name == 'credentials': + continue + path = current / name + relative = path.relative_to(source) + if path.is_symlink() or not path.is_file(): + continue + if path.suffix != '.go' and relative.as_posix() not in ('go.mod', 'go.sum'): + continue + files.append((relative, path.read_bytes())) + if not any(path.as_posix() == 'go.mod' for path, _ in files): + raise ValueError('local source must contain go.mod') + tree = hashlib.sha256() + manifest = [] + for relative, data in sorted(files): + name = relative.as_posix() + tree.update(name.encode() + b'\0' + data + b'\0') + entry = dict(path=name, sha256=digest(data)) + manifest.append(entry) + target = output / relative + target.parent.mkdir(parents=True, exist_ok=True) + target.write_bytes(data) + return tree.hexdigest(), manifest + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument('--source', required=True, type=Path) + parser.add_argument('--output', required=True, type=Path, help='new binary path; existing artifacts are preserved') + args = parser.parse_args() + source, output = args.source.resolve(), args.output.resolve() + receipt_path = output.with_suffix('.receipt.json') + frozen = output.with_suffix('.source') + if output.exists() or receipt_path.exists() or frozen.exists(): + raise ValueError('output binary, receipt and source snapshot must all be new') + revision = subprocess.check_output(['git', 'rev-parse', 'HEAD'], cwd=source, text=True).strip() + status = subprocess.check_output(['git', 'status', '--porcelain'], cwd=source, text=True) + frozen.mkdir(parents=True) + tree_sha256, manifest = freeze_source(source, frozen) + # Limit Git reads to the same explicit Go input allowlist used by the snapshot. + diff_paths = [item['path'] for item in manifest] + diff = subprocess.check_output(['git', 'diff', 'HEAD', '--', *diff_paths], cwd=source) + (frozen / 'source.diff').write_bytes(diff) + env = dict(os.environ, GOENV='off', GOWORK='off', GOFLAGS='-mod=readonly', CGO_ENABLED='0') + # Ordinary local builds remain dev identities. The receipt identifies the frozen inputs. + subprocess.run(['go', 'build', '-trimpath', '-buildvcs=false', + '-o', str(output), './cmd/weir'], cwd=frozen, env=env, check=True, timeout=600) + identity = json.loads(subprocess.check_output([str(output), 'version'], text=True)) + receipt = dict(source=revision, source_dirty=bool(status.strip()), source_tree_sha256=tree_sha256, + source_diff_sha256=digest(diff), source_files=manifest, + source_snapshot=str(frozen), source_kind='frozen local Go source tree', + sha256=digest(output.read_bytes()), identity=identity) + receipt_path.write_text(json.dumps(receipt, indent=2) + '\n') + result = dict(binary=str(output), receipt=str(receipt_path), + sha256=receipt['sha256'], source_tree_sha256=tree_sha256) + print(json.dumps(result, indent=2)) + + +if __name__ == '__main__': + main() diff --git a/scripts/build_local_server_test.py b/scripts/build_local_server_test.py new file mode 100644 index 00000000..c459ff6d --- /dev/null +++ b/scripts/build_local_server_test.py @@ -0,0 +1,67 @@ +import importlib.util +import io +from pathlib import Path +import tempfile +import unittest +from unittest import mock + + +spec = importlib.util.spec_from_file_location('build_local_server', Path(__file__).with_name('build_local_server.py')) +builder = importlib.util.module_from_spec(spec) +spec.loader.exec_module(builder) + + +class FrozenSourceTests(unittest.TestCase): + def test_build_overrides_inherited_overlay_flags(self): + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + source, output = root / 'source', root / 'server' + source.mkdir() + (source / 'go.mod').write_text('module local.invalid/source\n') + identity = dict(revision='unknown') + + def run_build(command, **kwargs): + self.assertEqual(kwargs['env']['GOFLAGS'], '-mod=readonly') + self.assertEqual(kwargs['env']['GOENV'], 'off') + self.assertEqual(kwargs['env']['GOWORK'], 'off') + self.assertEqual(kwargs['env']['CGO_ENABLED'], '0') + self.assertEqual(kwargs['cwd'], output.resolve().with_suffix('.source')) + output.write_bytes(b'built from frozen inputs') + + arguments = ['build_local_server.py', '--source', str(source), '--output', str(output)] + responses = ['a' * 40, '', b'', builder.json.dumps(identity)] + environment = dict(GOFLAGS='-overlay=/unrecorded-input.json -mod=mod') + with mock.patch('sys.argv', arguments), mock.patch.dict(builder.os.environ, environment), \ + mock.patch('sys.stdout', new_callable=io.StringIO), \ + mock.patch.object(builder.subprocess, 'check_output', side_effect=responses), \ + mock.patch.object(builder.subprocess, 'run', side_effect=run_build) as build: + builder.main() + self.assertEqual(build.call_count, 1) + receipt = builder.json.loads(output.with_suffix('.receipt.json').read_text()) + self.assertEqual(receipt['sha256'], builder.digest(output.read_bytes())) + + def test_untracked_go_inputs_are_frozen_without_secret_or_symlink_paths(self): + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + source, frozen = root / 'source', root / 'frozen' + source.mkdir() + (source / 'go.mod').write_text('module local.invalid/source\n') + (source / 'internal').mkdir() + ordinary = source / 'internal' / 'new.go' + ordinary.write_text('package internal\n') + (source / 'internal' / '.env.go').write_text('excluded fixture\n') + secrets = source / 'credentials' + secrets.mkdir() + (secrets / 'excluded.go').write_text('excluded fixture\n') + (source / 'internal' / 'linked.go').symlink_to(secrets / 'excluded.go') + tree, manifest = builder.freeze_source(source, frozen) + self.assertEqual([item['path'] for item in manifest], ['go.mod', 'internal/new.go']) + self.assertEqual((frozen / 'internal' / 'new.go').read_text(), 'package internal\n') + ordinary.write_text('package changed\n') + changed, _ = builder.freeze_source(source, root / 'changed') + self.assertNotEqual(tree, changed) + self.assertEqual((frozen / 'internal' / 'new.go').read_text(), 'package internal\n') + + +if __name__ == '__main__': + unittest.main() diff --git a/scripts/performance_evidence.py b/scripts/performance_evidence.py new file mode 100644 index 00000000..347843e4 --- /dev/null +++ b/scripts/performance_evidence.py @@ -0,0 +1,62 @@ +"""Validate the binary and source provenance shared by benchmark runs and summaries.""" +import re + + +def require_digest(value, field, length=64): + if not isinstance(value, str) or re.fullmatch('[0-9a-f]{' + str(length) + '}', value) is None: + raise ValueError(field + ': missing or invalid immutable digest') + + +def validate_server_receipt(receipt, binary_sha256, versions): + require_digest(binary_sha256, 'server_binary_sha256') + require_digest(receipt.get('source'), 'server_receipt.source', 40) + if receipt.get('sha256') != binary_sha256: + raise ValueError('server receipt SHA does not match the server binary') + empty = {} + identity = receipt.get('identity', empty) + if not isinstance(identity, dict) or not isinstance(identity.get('revision'), str) or not identity['revision']: + raise ValueError('server receipt must record the binary identity revision') + if 'source_tree_sha256' in receipt or receipt.get('source_kind') == 'frozen local Go source tree': + require_digest(receipt.get('source_tree_sha256'), 'server_receipt.source_tree_sha256') + require_digest(receipt.get('source_diff_sha256'), 'server_receipt.source_diff_sha256') + if not isinstance(receipt.get('source_dirty'), bool): + raise ValueError('local server receipt must record source_dirty as a boolean') + return 'local' + if receipt.get('version') != versions.get('weir_version') or not receipt.get('version'): + raise ValueError('locked server receipt version does not match the pinned manifest') + if receipt['source'] != versions.get('weir_source') or identity['revision'] != receipt['source']: + raise ValueError('locked server receipt source or binary revision does not match the pinned manifest') + return 'locked' + + +def validate_report_provenance(report, receipt): + for field in ('client_source_tree_sha256', 'client_binary_sha256', 'server_binary_sha256'): + require_digest(receipt.get(field), field) + if receipt.get('validation_error'): + raise ValueError('benchmark receipt records a validation failure: ' + receipt['validation_error']) + for field in ('server_binary_before_sha256', 'server_binary_after_sha256'): + if field in receipt and receipt[field] != receipt['server_binary_sha256']: + raise ValueError(field + ': server binary changed during the matrix') + empty = {} + versions = receipt.get('versions', empty) + server_receipt = receipt.get('server_receipt') + if not isinstance(server_receipt, dict): + raise ValueError('missing server receipt') + kind = validate_server_receipt(server_receipt, receipt['server_binary_sha256'], versions) + expected = dict(weir_binary_sha256=receipt['server_binary_sha256'], weir_source=server_receipt['source']) + for field in ('sdk', 'protocol', 'mongodb_image', 'elasticsearch_image'): + if not isinstance(versions.get(field), str) or not versions[field]: + raise ValueError('missing pinned ' + field) + expected[field] = versions[field] + if kind == 'local': + expected.update(weir_source_tree_sha256=server_receipt['source_tree_sha256'], + weir_source_diff_sha256=server_receipt['source_diff_sha256'], + weir_source_dirty=str(server_receipt['source_dirty']).lower(), + weir_binary_revision=server_receipt['identity']['revision'], + weir_source_kind='explicit local source build; unpublished changes may be present') + else: + expected['weir_source_kind'] = 'locked immutable module source' + actual = report.get('provenance', empty) + for field, value in expected.items(): + if actual.get(field) != value: + raise ValueError('report provenance ' + field + ' does not match the recorded server receipt or pins') diff --git a/scripts/performance_tools_test.py b/scripts/performance_tools_test.py new file mode 100644 index 00000000..b318870d --- /dev/null +++ b/scripts/performance_tools_test.py @@ -0,0 +1,250 @@ +import copy +import hashlib +import io +import json +from pathlib import Path +import subprocess +import tempfile +import unittest +from unittest import mock + +import performance_evidence as evidence +import run_performance_matrix as runner +import summarize_performance as summary + + +def make_receipt(kind, versions): + identity = dict(revision='unknown' if kind == 'local' else versions['weir_source']) + server = dict(source=versions['weir_source'], sha256='a' * 64, identity=identity) + if kind == 'local': + server.update(source_tree_sha256='b' * 64, source_diff_sha256='c' * 64, + source_dirty=True, source_kind='frozen local Go source tree') + else: + server['version'] = versions['weir_version'] + receipt = dict(client_source_tree_sha256='d' * 64, client_binary_sha256='e' * 64, + server_binary_sha256=server['sha256'], server_receipt=server, + versions=versions, exit_code=0, command=['performance-lab', '-backend', 'mongo']) + return receipt + + +def make_report(receipt): + server, versions = receipt['server_receipt'], receipt['versions'] + provenance = dict(weir_binary_sha256=server['sha256'], weir_source=server['source']) + for field in ('sdk', 'protocol', 'mongodb_image', 'elasticsearch_image'): + provenance[field] = versions[field] + if 'source_tree_sha256' in server: + provenance.update(weir_binary_revision=server['identity']['revision'], + weir_source_tree_sha256=server['source_tree_sha256'], + weir_source_diff_sha256=server['source_diff_sha256'], + weir_source_dirty=str(server['source_dirty']).lower(), + weir_source_kind='explicit local source build; unpublished changes may be present') + else: + provenance['weir_source_kind'] = 'locked immutable module source' + commands = dict(find=100, update=0) + latency = dict(p95_upper_bound_ns=5_000_000) + resources = dict(samples=[]) + stage = dict(succeeded=100, elapsed_ns=1_000_000_000, successful_operations_per_second=100, + reads=100, writes=0, errors=0, indeterminate=0, verified=True, + latency=latency, resources=resources, timed_physical_database_command_deltas=commands) + pair = dict(concurrency=8, batch_size=1, round=1, order='direct-first', direct=stage, weir=copy.deepcopy(stage)) + dataset = dict(backend='mongo', concurrency=8, namespace='owned', weir_seed='local') + parameters = dict(dataset=dataset, concurrency_levels=[8], batch_sizes=[1], paired_rounds=1) + report = dict(provenance=provenance, parameters=parameters, pairs=[pair]) + return report + + +class ProvenanceTests(unittest.TestCase): + def setUp(self): + self.versions = dict(weir_source='f' * 40, weir_version='v0.1.1-0.pinned', sdk='v0.10.0', + protocol='v0.8.0', mongodb_image='mongo@sha256:pinned', + elasticsearch_image='elasticsearch@sha256:pinned') + + def test_local_and_locked_receipts_match_actual_report(self): + for kind in ('local', 'locked'): + with self.subTest(kind=kind): + receipt = make_receipt(kind, self.versions) + report = make_report(receipt) + evidence.validate_report_provenance(report, receipt) + + def test_each_actual_provenance_field_is_required_and_bound(self): + for kind in ('local', 'locked'): + receipt = make_receipt(kind, self.versions) + original = make_report(receipt) + for field in original['provenance']: + for replacement in ('missing', 'different'): + with self.subTest(kind=kind, field=field, replacement=replacement): + report = copy.deepcopy(original) + if replacement == 'missing': + del report['provenance'][field] + else: + report['provenance'][field] = 'different' + with self.assertRaisesRegex(ValueError, field): + evidence.validate_report_provenance(report, receipt) + + def test_incomplete_or_inconsistent_receipts_cannot_certify_reports(self): + original = make_receipt('local', self.versions) + report = make_report(original) + failures = [('client_binary_sha256', None), ('client_source_tree_sha256', ''), + ('server_receipt', None), ('validation_error', 'binary replaced'), + ('server_binary_before_sha256', '0' * 64), ('server_binary_after_sha256', '0' * 64)] + for field, value in failures: + with self.subTest(field=field): + receipt = copy.deepcopy(original) + receipt[field] = value + with self.assertRaises(ValueError): + evidence.validate_report_provenance(report, receipt) + for field in ('source', 'source_tree_sha256', 'source_diff_sha256', 'source_dirty', 'identity', 'sha256'): + with self.subTest(server_field=field): + receipt = copy.deepcopy(original) + del receipt['server_receipt'][field] + with self.assertRaises(ValueError): + evidence.validate_report_provenance(report, receipt) + + def test_locked_receipt_cannot_claim_an_unpinned_source(self): + receipt = make_receipt('locked', self.versions) + receipt['server_receipt']['source'] = '0' * 40 + with self.assertRaisesRegex(ValueError, 'pinned manifest'): + evidence.validate_report_provenance(make_report(receipt), receipt) + + def test_summary_cli_rejects_one_replaced_actual_report(self): + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + before, after = root / 'before', root / 'after' + for phase in (before, after): + phase.mkdir() + for case in ('read', 'mixed', 'write', 'lua'): + receipt = make_receipt('local', self.versions) + report = make_report(receipt) + (phase / case).mkdir() + (phase / case / 'mongo.json').write_text(json.dumps(report)) + (phase / (case + '.receipt.json')).write_text(json.dumps(receipt)) + output = root / 'comparison' + arguments = ['summarize_performance.py', '--before', str(before), '--after', str(after), + '--backend', 'mongo', '--output', str(output)] + with mock.patch('sys.argv', arguments), mock.patch('sys.stdout', new_callable=io.StringIO): + summary.main() + self.assertEqual(json.loads(output.with_suffix('.json').read_text())['status'], 'complete') + changed = after / 'mixed' / 'mongo.json' + report = json.loads(changed.read_text()) + report['provenance']['weir_binary_sha256'] = '0' * 64 + changed.write_text(json.dumps(report)) + with mock.patch('sys.argv', arguments): + with self.assertRaisesRegex(SystemExit, 'mixed after:.*weir_binary_sha256'): + summary.main() + + +class MatrixRunnerTests(unittest.TestCase): + def test_automatic_and_explicit_receipts_select_local_or_locked_lab_flags(self): + versions = json.loads((Path(runner.__file__).resolve().parent.parent / 'versions.json').read_text()) + for kind in ('local', 'locked'): + for explicit in (False, True): + with self.subTest(kind=kind, explicit=explicit), tempfile.TemporaryDirectory() as directory: + root = Path(directory) + weir, output = root / 'server', root / 'matrix' + weir.write_bytes(b'frozen server') + receipt = make_receipt(kind, versions) + receipt['server_receipt']['sha256'] = hashlib.sha256(weir.read_bytes()).hexdigest() + receipt['server_binary_sha256'] = receipt['server_receipt']['sha256'] + server_receipt_path = weir.with_suffix('.receipt.json') + server_receipt_path.write_text(json.dumps(receipt['server_receipt'])) + arguments = ['run_performance_matrix.py', '--backend', 'mongo', '--weir', str(weir), '--output', str(output)] + if explicit: + arguments += ['--server-receipt', str(server_receipt_path)] + commands = [] + fixture = root / 'fixture' + fixture.mkdir() + + def run_command(command, **kwargs): + if command[0] == 'go': + Path(command[command.index('-o') + 1]).write_bytes(b'frozen client') + else: + commands.append(command) + report = make_report(receipt) + report['provenance']['fixture_logs'] = str(fixture) + target = Path(command[command.index('-output') + 1]) + target.mkdir() + (target / 'mongo.json').write_text(json.dumps(report)) + result = subprocess.CompletedProcess(command, 0) + return result + + with mock.patch('sys.argv', arguments), mock.patch('sys.stdout', new_callable=io.StringIO), \ + mock.patch.object(runner.subprocess, 'run', side_effect=run_command), \ + mock.patch.object(runner.subprocess, 'check_output', side_effect=['f' * 40, b'']): + runner.main() + self.assertEqual(len(commands), 4) + for command in commands: + self.assertEqual('-server-receipt' in command, kind == 'local') + if kind == 'local': + self.assertEqual(command[command.index('-server-receipt') + 1], str(output.resolve() / 'server.receipt.json')) + saved = json.loads((output / 'read.receipt.json').read_text()) + self.assertEqual(saved['server_receipt'], receipt['server_receipt']) + self.assertEqual(saved['server_receipt_kind'], kind) + self.assertEqual(saved['server_binary_before_sha256'], saved['server_binary_after_sha256']) + + def test_binary_replacement_before_or_during_case_aborts_matrix_and_retains_evidence(self): + versions = json.loads((Path(runner.__file__).resolve().parent.parent / 'versions.json').read_text()) + for when in ('before', 'during'): + with self.subTest(when=when), tempfile.TemporaryDirectory() as directory: + root = Path(directory) + weir, output = root / 'server', root / 'matrix' + weir.write_bytes(b'initial server') + receipt = make_receipt('local', versions) + receipt['server_receipt']['sha256'] = hashlib.sha256(weir.read_bytes()).hexdigest() + weir.with_suffix('.receipt.json').write_text(json.dumps(receipt['server_receipt'])) + arguments = ['run_performance_matrix.py', '--backend', 'mongo', '--weir', str(weir), '--output', str(output)] + + def run_command(command, **kwargs): + if command[0] == 'go': + Path(command[command.index('-o') + 1]).write_bytes(b'frozen client') + if when == 'before': + weir.write_bytes(b'replacement server') + else: + weir.write_bytes(b'replacement server') + result = subprocess.CompletedProcess(command, 0) + return result + + with mock.patch('sys.argv', arguments), mock.patch('sys.stdout', new_callable=io.StringIO), \ + mock.patch.object(runner.subprocess, 'run', side_effect=run_command) as run, \ + mock.patch.object(runner.subprocess, 'check_output', side_effect=['f' * 40, b'']): + with self.assertRaisesRegex(ValueError, 'server binary changed ' + when + ' read'): + runner.main() + self.assertEqual(run.call_count, 1 if when == 'before' else 2) + saved = json.loads((output / 'read.receipt.json').read_text()) + if when == 'during': + self.assertEqual(saved['exit_code'], 0) + self.assertNotEqual(saved['server_binary_after_sha256'], saved['server_binary_sha256']) + else: + self.assertNotEqual(saved['server_binary_before_sha256'], saved['server_binary_sha256']) + self.assertIn('changed ' + when + ' read', saved['validation_error']) + self.assertFalse((output / 'mixed.receipt.json').exists()) + + +class PhysicalCountTests(unittest.TestCase): + def test_only_complete_available_observations_are_aggregated(self): + first_counts, second_counts = dict(find=3, update=0), dict(find=4, update=0) + first = dict(timed_physical_database_command_deltas=first_counts) + second = dict(timed_physical_database_command_deltas=second_counts) + total, reasons = summary.summarize_physical_counts([first, second]) + expected = dict(find=7, update=0) + self.assertEqual(total, expected) + self.assertEqual(reasons, []) + reset_counts = dict(find=0, update=0) + unavailable = dict(timed_physical_database_command_deltas=reset_counts, + physical_database_command_observation_unavailable='counter reset during stage') + empty_counts = {} + empty = dict(timed_physical_database_command_deltas=empty_counts) + negative_counts = dict(find=-1, update=0) + negative = dict(timed_physical_database_command_deltas=negative_counts) + missing_counts = dict(find=4) + missing_command = dict(timed_physical_database_command_deltas=missing_counts) + missing = {} + for value, explanation in ((unavailable, 'counter reset during stage'), (empty, 'missing'), + (missing, 'missing'), (negative, 'reset'), (missing_command, 'differ')): + with self.subTest(explanation=explanation): + total, reasons = summary.summarize_physical_counts([first, value]) + self.assertIsNone(total) + self.assertIn(explanation, reasons[0]) + + +if __name__ == '__main__': + unittest.main() diff --git a/scripts/run_performance_matrix.py b/scripts/run_performance_matrix.py new file mode 100644 index 00000000..2ee724a5 --- /dev/null +++ b/scripts/run_performance_matrix.py @@ -0,0 +1,120 @@ +#!/usr/bin/env python3 +"""Run serialized, verified ordinary/Lua benchmarks with retained source and fixture evidence.""" +import argparse +import hashlib +import json +import os +from pathlib import Path +import shutil +import subprocess + +from build_local_server import freeze_source +from performance_evidence import validate_report_provenance, validate_server_receipt + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument('--backend', choices=('mongo', 'search'), required=True) + parser.add_argument('--weir', required=True, type=Path) + parser.add_argument('--server-receipt', type=Path) + parser.add_argument('--mongod', type=Path) + parser.add_argument('--output', required=True, type=Path) + parser.add_argument('--bulk', action='store_true', help='append read and write bulk32 comparisons') + parser.add_argument('--batch-limit', type=int, default=0, help='0 keeps the server default') + parser.add_argument('--warmup-duration', help='default: Search 20s, native Mongo 2s per path/stage') + parser.add_argument('--duration', help='default measurement: Search 20s, native Mongo 5s per path/stage') + args = parser.parse_args() + warmup = args.warmup_duration or ('20s' if args.backend == 'search' else '2s') + duration = args.duration or ('20s' if args.backend == 'search' else '5s') + root = Path(__file__).resolve().parent.parent + output = args.output.resolve() + output.mkdir(parents=True, exist_ok=False) + client = output / 'performance-lab' + weir = args.weir.resolve() + versions = json.loads((root / 'versions.json').read_text()) + server_sha256 = hashlib.sha256(weir.read_bytes()).hexdigest() + server_receipt_path = args.server_receipt.resolve() if args.server_receipt else weir.with_suffix('.receipt.json') + server_receipt = json.loads(server_receipt_path.read_text()) + receipt_kind = validate_server_receipt(server_receipt, server_sha256, versions) + archived_receipt = output / 'server.receipt.json' + archived_receipt.write_text(json.dumps(server_receipt, indent=2) + '\n') + source_sha256, source_files = freeze_source(root, output / 'client-source') + build_env = dict(os.environ, GOENV='off', GOWORK='off', GOFLAGS='-mod=readonly') + subprocess.run(['go', 'build', '-trimpath', '-buildvcs=false', '-o', str(client), './cmd/weir-lab'], + cwd=output / 'client-source', env=build_env, check=True, timeout=600) + provenance = dict(client_source_tree_sha256=source_sha256, client_source_files=source_files, + client_binary_sha256=hashlib.sha256(client.read_bytes()).hexdigest(), + server_binary_sha256=server_sha256, + test_source=subprocess.check_output(['git', 'rev-parse', 'HEAD'], cwd=root, text=True).strip(), + test_source_dirty=bool(subprocess.check_output(['git', 'status', '--porcelain'], cwd=root)), + versions=versions, server_receipt=server_receipt, server_receipt_kind=receipt_kind) + if args.mongod: + mongod = args.mongod.resolve() + provenance['native_mongodb_binary_sha256'] = hashlib.sha256(mongod.read_bytes()).hexdigest() + mongo_receipt = mongod.with_suffix('.receipt.json') + if mongo_receipt.exists(): + provenance['native_mongodb_receipt'] = json.loads(mongo_receipt.read_text()) + shutil.copy2(mongo_receipt, output / 'mongod.receipt.json') + cases = [('read', 0, False, 'saturation'), ('mixed', 10, False, 'saturation'), + ('write', 100, False, 'saturation'), ('lua', 100, True, 'saturation')] + if args.bulk: + cases += [('bulk-read', 0, False, 'bulk-saturation'), ('bulk-write', 100, False, 'bulk-saturation')] + for name, writes, lua, mode in cases: + directory = output / name + command = [str(client), '-mode', mode, '-weir', str(weir), '-backend', args.backend, + '-client-processes', '4', '-concurrency-levels', '8,32', + '-batch-sizes', '32' if mode == 'bulk-saturation' else '1', + '-records', '1024', '-payload-bytes', '1024', '-rounds', '3', + '-warmup-duration', warmup, '-duration', duration, '-database-cpus', '2', + '-write-percent', str(writes), '-backend-batch-limit', str(args.batch_limit), + '-output', str(directory)] + if args.mongod: + command += ['-mongod', str(args.mongod.resolve())] + if receipt_kind == 'local': + command += ['-server-receipt', str(archived_receipt)] + if lua: + command.append('-lua-mutations') + concurrency = 'total 8/32; four OS processes each have 2/8 single-record workers' + if mode == 'bulk-saturation': + concurrency = 'one client OS process with 8/32 workers; 32 records per call' + receipt = dict(provenance, command=command, case=name, concurrency=concurrency) + receipt_path = output / (name + '.receipt.json') + receipt['server_binary_before_sha256'] = hashlib.sha256(weir.read_bytes()).hexdigest() + if receipt['server_binary_before_sha256'] != server_sha256: + receipt['validation_error'] = 'server binary changed before ' + name + receipt_path.write_text(json.dumps(receipt, indent=2) + '\n') + raise ValueError(receipt['validation_error']) + receipt_path.write_text(json.dumps(receipt, indent=2) + '\n') + print('Starting ' + name, flush=True) + with (output / (name + '.log')).open('w') as log: + result = subprocess.run(command, cwd=root, stdout=log, stderr=subprocess.STDOUT) + receipt['exit_code'] = result.returncode + receipt['server_binary_after_sha256'] = hashlib.sha256(weir.read_bytes()).hexdigest() + if receipt['server_binary_after_sha256'] != server_sha256: + receipt['validation_error'] = 'server binary changed during ' + name + receipt_path.write_text(json.dumps(receipt, indent=2) + '\n') + raise ValueError(receipt['validation_error']) + report_path = directory / (args.backend + '.json') + if report_path.exists(): + report = json.loads(report_path.read_text()) + try: + validate_report_provenance(report, receipt) + except ValueError as error: + receipt['validation_error'] = str(error) + receipt_path.write_text(json.dumps(receipt, indent=2) + '\n') + raise + fixture = Path(report['provenance']['fixture_logs']) + evidence = output / (name + '-fixture') + evidence.mkdir() + for path in fixture.iterdir(): + if path.is_file() and (path.suffix in ('.json', '.log', '.txt')): + shutil.copy2(path, evidence / path.name) + receipt['fixture_evidence'] = str(evidence) + receipt_path.write_text(json.dumps(receipt, indent=2) + '\n') + print('Finished ' + name + ': exit ' + str(result.returncode), flush=True) + if result.returncode: + raise SystemExit(result.returncode) + + +if __name__ == '__main__': + main() diff --git a/scripts/summarize_performance.py b/scripts/summarize_performance.py new file mode 100644 index 00000000..e5fc71a5 --- /dev/null +++ b/scripts/summarize_performance.py @@ -0,0 +1,272 @@ +#!/usr/bin/env python3 +"""Recompute every-round before/after throughput, latency, RSS and command evidence.""" +import argparse +from datetime import datetime +import json +from pathlib import Path + +from performance_evidence import validate_report_provenance + + +def summarize_physical_counts(values): + commands, reasons = {}, [] + expected_commands = None + for index, value in enumerate(values, 1): + unavailable = value.get('physical_database_command_observation_unavailable') + observed = value.get('timed_physical_database_command_deltas') + reason = None + if unavailable: + reason = str(unavailable) + elif not isinstance(observed, dict) or not observed: + reason = 'missing physical database command deltas' + elif any(not isinstance(count, int) or isinstance(count, bool) or count < 0 for count in observed.values()): + reason = 'invalid or reset physical database command counter' + elif expected_commands is not None and set(observed) != expected_commands: + reason = 'physical database command counters differ between rounds' + if reason: + reasons.append('paired round ' + str(index) + ': ' + reason) + continue + expected_commands = set(observed) + for name, count in observed.items(): + commands[name] = commands.get(name, 0) + count + return (None if reasons else commands), reasons + + +def metric_delta(metrics, name): + counters = [] + empty = {} + for stage in ('before', 'after'): + matching = [item['value'] for item in metrics.get(stage, empty).get('metrics', []) + if item['name'] == name and not item.get('labels')] + if len(matching) != 1: + return None + counters.append(matching[0]) + delta = counters[1] - counters[0] + return delta if delta >= 0 else None + + +def comparison_parameters(report): + parameters = dict(report['parameters']) + dataset = dict(parameters['dataset']) + for fixture_field in ('weir_seed', 'namespace'): + dataset.pop(fixture_field, None) + parameters['dataset'] = dataset + return parameters + + +def measurement_arguments(receipt): + arguments = receipt['command'][1:] + result = [] + index = 0 + while index < len(arguments): + if arguments[index] in ('-weir', '-server-receipt', '-output'): + index += 2 + else: + result.append(arguments[index]) + index += 1 + return result + + +def summarize(report, concurrency, path): + values = [pair[path] for pair in report['pairs'] if pair['concurrency'] == concurrency] + empty = {} + succeeded = sum(value['succeeded'] for value in values) + elapsed = sum(value['elapsed_ns'] for value in values) + commands, command_reasons = summarize_physical_counts(values) + reads = sum(value['reads'] for value in values) + writes = sum(value['writes'] for value in values) + samples = [sample for value in values for sample in value['resources']['samples']] + rss = max((sample.get('weir_memory_bytes', 0) for sample in samples), default=0) + batch_count = sum(value.get('server_metrics', empty).get('timed_counter_deltas', empty).get('weir_store_batch_operations_count', 0) for value in values) + batch_operations = sum(value.get('server_metrics', empty).get('timed_counter_deltas', empty).get('weir_store_batch_operations_sum', 0) for value in values) + batch_buckets = {} + for value in values: + for bound, count in value.get('server_metrics', empty).get('timed_adapter_batch_cumulative_buckets', empty).items(): + batch_buckets[bound] = batch_buckets.get(bound, 0) + count + p95 = [value['latency']['p95_upper_bound_ns'] / 1e6 for value in values] + cpu_deltas, cpu_spans, cpu_per_operation, allocated_bytes, allocated_objects = [], [], [], [], [] + round_batch_averages = [] + gc_cycles, gc_pauses = [], [] + for value in values: + metrics = value.get('server_metrics', empty) + timed_deltas = metrics.get('timed_counter_deltas', empty) + count = timed_deltas.get('weir_store_batch_operations_count', 0) + operations = timed_deltas.get('weir_store_batch_operations_sum', 0) + round_batch_averages.append(operations / count if count else None) + delta = metric_delta(metrics, 'process_cpu_seconds_total') + allocations = metric_delta(metrics, 'go_memstats_alloc_bytes_total') + objects = metric_delta(metrics, 'go_memstats_mallocs_total') + allocated_bytes.append(allocations) + allocated_objects.append(objects) + gc_cycles.append(metric_delta(metrics, 'go_gc_duration_seconds_count')) + gc_pauses.append(metric_delta(metrics, 'go_gc_duration_seconds_sum')) + if delta is None: + cpu_per_operation.append(None) + continue + start = datetime.fromisoformat(metrics['before']['at_utc']) + end = datetime.fromisoformat(metrics['after']['at_utc']) + span = (end - start).total_seconds() + if span <= 0: + cpu_per_operation.append(None) + continue + cpu_deltas.append(delta) + cpu_spans.append(span) + cpu_per_operation.append(delta / value['succeeded'] * 1e6) + pooled_p95 = None + for comparison in report.get('observed_business_comparisons', []): + if comparison['concurrency'] == concurrency: + pooled_p95 = comparison[path + '_business_request_latency']['p95_upper_bound_ns'] / 1e6 + result = dict(successful_operations=succeeded, operations_per_second=succeeded * 1e9 / elapsed, + round_operations_per_second=[value['successful_operations_per_second'] for value in values], + round_p95_upper_ms=p95, pooled_p95_upper_ms=pooled_p95, + p95_round_range_ms=[min(p95), max(p95)], sampled_weir_peak_rss_mib=rss / 1048576, + adapter_batch_average=batch_operations / batch_count if batch_count else None, + round_adapter_batch_average=round_batch_averages, + adapter_batch_cumulative_buckets=batch_buckets, + adapter_batch_greater_than_16_fraction=(batch_count - batch_buckets['16']) / batch_count if batch_count and '16' in batch_buckets else None, + adapter_batch_exact_32_fraction=(batch_buckets['32'] - batch_buckets['31']) / batch_count if batch_count and '31' in batch_buckets and '32' in batch_buckets else None, + server_cpu_microseconds_per_operation=sum(cpu_deltas) / succeeded * 1e6 if len(cpu_deltas) == len(values) else None, + server_mean_cpu_cores=sum(cpu_deltas) / sum(cpu_spans) if cpu_spans else None, + round_server_cpu_microseconds_per_operation=cpu_per_operation, + server_allocated_bytes_per_operation=sum(allocated_bytes) / succeeded if all(value is not None for value in allocated_bytes) else None, + server_allocated_objects_per_operation=sum(allocated_objects) / succeeded if all(value is not None for value in allocated_objects) else None, + round_server_allocated_bytes_per_operation=[delta / value['succeeded'] if delta is not None else None for value, delta in zip(values, allocated_bytes)], + round_server_gc_cycles=gc_cycles, + round_server_gc_pause_seconds=gc_pauses, + server_gc_pause_microseconds_per_operation=sum(gc_pauses) / succeeded * 1e6 if all(value is not None for value in gc_pauses) else None, + server_resource_scope='Prometheus counter differences between snapshots wrapping each measured Weir stage; includes small metrics collection overhead; RSS is a sampled maximum', + physical_commands=commands, + physical_commands_available=not command_reasons, + physical_commands_unavailable_reasons=command_reasons, + reads=reads, writes=writes, + errors=sum(value['errors'] for value in values), + indeterminate=sum(value['indeterminate'] for value in values), + business_operation_timeouts=sum(value.get('business_operation_timeouts', 0) for value in values), + verified=all(value['verified'] for value in values) and not report.get('incomplete')) + return result + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument('--before', type=Path, required=True) + parser.add_argument('--after', type=Path, required=True) + parser.add_argument('--backend', choices=('mongo', 'search'), required=True) + parser.add_argument('--output', type=Path, required=True) + args = parser.parse_args() + rows, missing_cases, case_provenance = [], [], [] + expected_client = None + expected_servers = None + for case in ('read', 'mixed', 'write', 'lua', 'bulk-read', 'bulk-write'): + before_path = args.before / case / (args.backend + '.json') + after_path = args.after / case / (args.backend + '.json') + if not before_path.exists() or not after_path.exists(): + if case in ('read', 'mixed', 'write', 'lua') or before_path.exists() or after_path.exists(): + missing = [phase for phase, path in (('before', before_path), ('after', after_path)) if not path.exists()] + missing_case = dict(case=case, missing=missing) + missing_cases.append(missing_case) + continue + before, after = json.loads(before_path.read_text()), json.loads(after_path.read_text()) + before_receipt = json.loads((args.before / (case + '.receipt.json')).read_text()) + after_receipt = json.loads((args.after / (case + '.receipt.json')).read_text()) + receipts = (before_receipt, after_receipt) + for phase, report, receipt in (('before', before, before_receipt), ('after', after, after_receipt)): + try: + validate_report_provenance(report, receipt) + except ValueError as error: + raise SystemExit(case + ' ' + phase + ': ' + str(error)) from error + if any(receipt.get('exit_code') != 0 for receipt in receipts): + raise SystemExit(case + ': benchmark process did not exit successfully') + if before_receipt['versions'] != after_receipt['versions']: + raise SystemExit(case + ': before/after pinned version manifests differ') + servers = tuple(receipt['server_binary_sha256'] for receipt in receipts) + if expected_servers is None: + expected_servers = servers + if servers != expected_servers: + raise SystemExit(case + ': a server binary changed within the matrix') + clients = [(receipt['client_source_tree_sha256'], receipt['client_binary_sha256']) for receipt in receipts] + if expected_client is None: + expected_client = clients[0] + if any(client != expected_client for client in clients): + raise SystemExit(case + ': client source or binary differs between cases/phases') + if comparison_parameters(before) != comparison_parameters(after): + raise SystemExit(case + ': before/after benchmark parameters differ') + if len(before['parameters']['batch_sizes']) != 1: + raise SystemExit(case + ': summarize one batch size per report to avoid mixing workloads') + if measurement_arguments(before_receipt) != measurement_arguments(after_receipt): + raise SystemExit(case + ': before/after measurement command arguments differ') + paired_rounds = before['parameters']['paired_rounds'] + pair_keys = [(pair['concurrency'], pair['batch_size'], pair['round'], pair['order']) for pair in before['pairs']] + after_keys = [(pair['concurrency'], pair['batch_size'], pair['round'], pair['order']) for pair in after['pairs']] + if pair_keys != after_keys: + raise SystemExit(case + ': before/after paired round execution order differs') + for concurrency in before['parameters']['concurrency_levels']: + for batch in before['parameters']['batch_sizes']: + rounds = [pair['round'] for pair in before['pairs'] if pair['concurrency'] == concurrency and pair['batch_size'] == batch] + if sorted(rounds) != list(range(1, paired_rounds + 1)): + raise SystemExit(case + ': incomplete or duplicate paired rounds') + evidence = dict(case=case, parameters=comparison_parameters(before), + command_arguments=measurement_arguments(before_receipt), + before_server_binary_sha256=before_receipt['server_binary_sha256'], + after_server_binary_sha256=after_receipt['server_binary_sha256'], + before_receipt=str((args.before / (case + '.receipt.json')).resolve()), + after_receipt=str((args.after / (case + '.receipt.json')).resolve())) + case_provenance.append(evidence) + for concurrency in before['parameters']['concurrency_levels']: + left, right = {}, {} + for path in ('direct', 'weir'): + left[path] = summarize(before, concurrency, path) + right[path] = summarize(after, concurrency, path) + row = dict(case=case, concurrency=concurrency, before=left, after=right, + weir_change_percent=100 * (right['weir']['operations_per_second'] / left['weir']['operations_per_second'] - 1), + direct_change_percent=100 * (right['direct']['operations_per_second'] / left['direct']['operations_per_second'] - 1), + scope='all ' + str(paired_rounds) + ' paired rounds; p95 values are histogram upper bounds; bulk p95 remains per-round rather than averaging quantiles') + rows.append(row) + provenance = dict(client_source_tree_sha256=expected_client[0] if expected_client else None, + client_binary_sha256=expected_client[1] if expected_client else None, + cases=case_provenance) + status = 'partial' if missing_cases else 'complete' + summary = dict(status=status, missing_cases=missing_cases, backend=args.backend, before=str(args.before.resolve()), after=str(args.after.resolve()), provenance=provenance, rows=rows) + role_path = args.after / 'experiment-role.json' + if role_path.exists(): + summary['after_experiment_role'] = json.loads(role_path.read_text()) + args.output.with_suffix('.json').write_text(json.dumps(summary, indent=2) + '\n') + lines = ['# Observed before/after performance', '', + '**Status: ' + status + '.** ' + ('Missing evidence: ' + ', '.join(item['case'] + ' ' + '/'.join(item['missing']) for item in missing_cases) if missing_cases else 'All core cases have complete matching paired rounds.'), '', + 'All paired rounds are included. Throughput is total successful operations divided by total measured time. Latencies are histogram upper bounds. Native MongoDB runs have no enforced CPU quota; these measurements do not prove maximum database capacity.', '', + 'Server CPU and Go allocation costs use the Prometheus counter differences around each timed stage, divided by successful records. These include small metrics collection overhead. RSS is the maximum observed process sample. A batch-size histogram without separate 31 and 32 boundaries cannot establish the exact 32-record full-batch fraction.', '', + 'Each included case has the same frozen client source, binary, benchmark parameters and paired round order. Source tree: `' + str(provenance['client_source_tree_sha256']) + '`. Binary: `' + str(provenance['client_binary_sha256']) + '`.', '', + '| Case | Total workers | Before Weir ops/s | After Weir ops/s | Change | Direct drift | Before / after p95 ms | Before / after RSS MiB | Before / after CPU µs/op | Before / after allocation KiB/op | Before / after adapter batch | Verified |', + '| --- | ---: | ---: | ---: | ---: | ---: | --- | --- | --- | --- | --- | --- |'] + if 'after_experiment_role' in summary: + role = summary['after_experiment_role'] + lines[2:2] = ['**After experiment: ' + role['role'] + '.** ' + role['status'] + '.', ''] + for row in rows: + before, after = row['before']['weir'], row['after']['weir'] + latency = [] + for value in (before, after): + p95 = value['pooled_p95_upper_ms'] + if p95 is None: + low, high = value['p95_round_range_ms'] + latency.append(f'{low:.3f}–{high:.3f}') + else: + latency.append(f'{p95:.3f}') + verified = all(value['verified'] and value['errors'] == 0 and value['indeterminate'] == 0 and value['business_operation_timeouts'] == 0 + for phase in (row['before'], row['after']) for value in phase.values()) + cpu, allocation, batch = [], [], [] + for value in (before, after): + observed = value['server_cpu_microseconds_per_operation'] + cpu.append(f'{observed:.2f}' if observed is not None else 'unavailable') + observed = value['server_allocated_bytes_per_operation'] + allocation.append(f'{observed / 1024:.2f}' if observed is not None else 'unavailable') + observed = value['adapter_batch_average'] + batch.append(f'{observed:.2f}' if observed is not None else 'unavailable') + lines.append(f"| {row['case']} | {row['concurrency']} | {before['operations_per_second']:.1f} | {after['operations_per_second']:.1f} | {row['weir_change_percent']:+.2f}% | {row['direct_change_percent']:+.2f}% | {' / '.join(latency)} | {before['sampled_weir_peak_rss_mib']:.1f} / {after['sampled_weir_peak_rss_mib']:.1f} | {' / '.join(cpu)} | {' / '.join(allocation)} | {' / '.join(batch)} | {verified} |") + notes_path = args.after / 'comparison-notes.md' + if notes_path.exists(): + lines.extend(['', notes_path.read_text().strip()]) + args.output.with_suffix('.md').write_text('\n'.join(lines) + '\n') + print(args.output.with_suffix('.md')) + + +if __name__ == '__main__': + main() diff --git a/tests/integration/single_benchmark_test.go b/tests/integration/single_benchmark_test.go index 4fc6d300..ebf997b4 100644 --- a/tests/integration/single_benchmark_test.go +++ b/tests/integration/single_benchmark_test.go @@ -37,7 +37,7 @@ func TestIndependentSingleRequestClients(t *testing.T) { t.Fatalf("build real client executable: %v\n%s", err, output) } operationTimeout := 10 * time.Second - options := fixture.Options{WeirBinary: serverBinary, MongoBinary: os.Getenv("WEIR_TEST_MONGODB_BINARY"), Backends: []string{"mongo", "search"}, OwnerCount: 1, StoreConcurrency: 2, BatchSize: 32, BackendTimeout: operationTimeout, IngressSessions: 64, DatabaseCPUs: 1} + options := fixture.Options{WeirBinary: serverBinary, MongoBinary: os.Getenv("WEIR_TEST_MONGODB_BINARY"), Backends: []string{"mongo", "search"}, OwnerCount: 1, BatchSize: 32, DatabaseCPUs: 1} cluster, err := fixture.Start(ctx, options) if err != nil { t.Fatal(err) @@ -75,13 +75,13 @@ func TestIndependentSingleRequestClients(t *testing.T) { t.Error(err) } }() - params := benchmark.SaturationOptions{Dataset: dataset, Paths: paths, Concurrency: []int{2, 4}, BatchSizes: []int{1}, Warmup: time.Second, Duration: 5 * time.Second, Rounds: 1, WritePercent: 10, OperationTimeout: operationTimeout, Resources: cluster.Resources(backend), CPUThreshold: 90, ClientProcesses: 2, ClientExecutable: binary, Provenance: map[string]string{"client_operation_timeout": operationTimeout.String(), "weir_backend_timeout": options.BackendTimeout.String()}} + params := benchmark.SaturationOptions{Dataset: dataset, Paths: paths, Concurrency: []int{2, 4}, BatchSizes: []int{1}, Warmup: time.Second, Duration: 5 * time.Second, Rounds: 1, WritePercent: 10, OperationTimeout: operationTimeout, Resources: cluster.Resources(backend), CPUThreshold: 90, ClientProcesses: 2, ClientExecutable: binary, Provenance: map[string]string{"client_operation_timeout": operationTimeout.String()}} report, err := benchmark.RunSaturation(ctx, params) if err != nil || report == nil || !report.Verified() { t.Fatal("real multiprocess single-request workload failed", err) } - if report.Parameters.OperationTimeout != operationTimeout.String() || report.Provenance["weir_backend_timeout"] != operationTimeout.String() { - t.Fatal("report lost the matched request/backend timeout policy") + if report.Parameters.OperationTimeout != operationTimeout.String() { + t.Fatal("report lost the client operation timeout policy") } if len(report.BusinessComparisons) != 2 { t.Fatal("observed business performance missing") diff --git a/tests/integration/stream_memory_test.go b/tests/integration/stream_memory_test.go index b2e461a6..2d765acf 100644 --- a/tests/integration/stream_memory_test.go +++ b/tests/integration/stream_memory_test.go @@ -15,11 +15,68 @@ import ( "time" weir "github.com/batchstream/weir-go" + "github.com/batchstream/weir-protocol/api/protocol" "github.com/batchstream/weir-tests/internal/fixture" "github.com/batchstream/weir-tests/internal/observe" "go.mongodb.org/mongo-driver/v2/bson" ) +const streamPendingRecords = 32 + +// A ReadStream uses one Execute RPC with no other business RPCs active here. +// Each owner's reservations are bounded by its transport record window and the +// public protocol event size; these credits describe ownership rather than RSS. +func streamReservations(ctx context.Context, nodes []fixture.Node, store string) (bool, error) { + idle := true + labels := map[string]string{"store": store} + for _, node := range nodes { + if !node.Owner { + continue + } + snapshot := observe.Fetch(ctx, node.Diagnostics) + values := make(map[string]float64) + for _, name := range []string{ + "pending_entries", "pending_reserved_bytes", "result_reserved_entries", "result_reserved_bytes", + "working_reserved_bytes", "retained_results", "retained_result_reserved_bytes", "active_executions", "publishers", + } { + value, found := snapshot.Sum("weir_store_"+name, labels) + if !found || value < 0 { + return false, fmt.Errorf("Store stream metric %s unavailable or negative: value=%v error=%s", name, value, snapshot.Error) + } + values[name] = value + idle = idle && value == 0 + } + if values["result_reserved_entries"] > streamPendingRecords || values["result_reserved_bytes"] > streamPendingRecords*protocol.MaxEvent || + values["pending_entries"] > values["result_reserved_entries"] || values["retained_results"] > values["result_reserved_entries"] || + values["retained_result_reserved_bytes"] > values["result_reserved_bytes"] { + return false, fmt.Errorf("Store %s stream reservations exceeded transport/protocol ownership bounds: %v", store, values) + } + } + return idle, nil +} + +func (s *system) waitStreamIdle(t *testing.T, store string) { + t.Helper() + ctx, cancel := context.WithTimeout(s.ctx, 5*time.Second) + defer cancel() + ticker := time.NewTicker(20 * time.Millisecond) + defer ticker.Stop() + for { + idle, err := streamReservations(ctx, s.cluster.Nodes, store) + if err != nil { + t.Fatal(err) + } + if idle { + return + } + select { + case <-ticker.C: + case <-ctx.Done(): + t.Fatalf("Store %s retained execution or publication credits after stream completion: %v", store, ctx.Err()) + } + } +} + // Sample only processes started by this suite, including both possible owners. func ownerResidentBytes(ctx context.Context, nodes []fixture.Node) (uint64, error) { var largest uint64 @@ -88,6 +145,7 @@ func (s *system) testStreamMemory(t *testing.T, backend *backendData) { backend.assertPersisted(t, s.ctx, "stream_memory_1023", 42) var shortPeak uint64 for _, count := range []int{1024, 4096} { + s.waitStreamIdle(t, backend.name) before := s.nodeMetrics(t) ctx, stop := context.WithTimeout(s.ctx, 90*time.Second) produced, consumed := 0, uint64(0) @@ -98,19 +156,8 @@ func (s *system) testStreamMemory(t *testing.T, backend *backendData) { return err } peak = max(peak, resident) - for _, node := range s.cluster.Nodes { - if !node.Owner { - continue - } - snapshot := observe.Fetch(ctx, node.Diagnostics) - labels := map[string]string{"store": backend.name} - retained, found := snapshot.Sum("weir_store_result_reserved_bytes", labels) - limit, limitFound := snapshot.Sum("weir_store_result_reserved_bytes_limit", labels) - if !found || !limitFound || retained < 0 || retained > limit { - return fmt.Errorf("Store result credits invalid: retained=%v limit=%v error=%s", retained, limit, snapshot.Error) - } - } - return nil + _, err = streamReservations(ctx, s.cluster.Nodes, backend.name) + return err } if err := sample(); err != nil { stop() @@ -149,6 +196,7 @@ func (s *system) testStreamMemory(t *testing.T, backend *backendData) { if err != nil || consumed != uint64(count) || bytes < uint64(count)*payloadBytes { t.Fatalf("large call failed: count=%d consumed=%d bytes=%d error=%v", count, consumed, bytes, err) } + s.waitStreamIdle(t, backend.name) evidence := streamBatchEvidence{Before: before, After: s.nodeMetrics(t), Store: backend.name, Method: "read", Records: count} assertStreamBatchMetrics(t, evidence) if count == 1024 { diff --git a/tests/integration/system_test.go b/tests/integration/system_test.go index 4f398a89..f5c9a823 100644 --- a/tests/integration/system_test.go +++ b/tests/integration/system_test.go @@ -43,8 +43,8 @@ func TestSystemIntegration(t *testing.T) { options := fixture.Options{ WeirBinary: binary, MongoBinary: os.Getenv("WEIR_TEST_MONGODB_BINARY"), Backends: []string{"mongo", "search"}, - OwnerCount: 2, DiscoveryOnly: true, StoreConcurrency: 2, - BatchSize: 513, + OwnerCount: 2, DiscoveryOnly: true, + BatchSize: 513, PendingRecords: streamPendingRecords, } cluster, err := fixture.Start(ctx, options) if err != nil { diff --git a/versions.json b/versions.json index 0c17e7fe..ff3fd516 100644 --- a/versions.json +++ b/versions.json @@ -1,13 +1,13 @@ { "go": "1.27.1", - "sdk": "v0.9.0", - "protocol": "v0.7.0", - "weir_source": "d3ea541143a819d6e839b6ce398221d85a20ce90", + "sdk": "v0.10.0", + "protocol": "v0.8.0", + "weir_source": "d5ad652e30ccf381678b911816059fe36faff955", "mongodb_version": "8.0.32", "elasticsearch_version": "8.19.22", "elasticsearch_image": "docker.elastic.co/elasticsearch/elasticsearch@sha256:e98f9c3b09beb2fbb9eaf667d602df3f0e00bd3644138b8458dc17ba1a675595", - "weir_version": "v0.1.1-0.20261007083022-d3ea541143a8", - "weir_sum": "h1:+d+Fs+R1rCYpQHindXtE8cNiMms8mlCJ5fMFWw6GMbw=", + "weir_version": "v0.1.1-0.20261010150620-d5ad652e30cc", + "weir_sum": "h1:+y3CjKbh9uvma53J0tGPYBTQWXDGP9aNkTiXTM70iE4=", "mongodb_image": "mongo@sha256:d0d926f94df099bff534b7ee5b5986458131a22489dfff8664509af0c1e2ca9c", "mongodb_darwin_arm64_url": "https://fastdl.mongodb.org/osx/mongodb-macos-arm64-8.0.32.tgz", "mongodb_darwin_arm64_sha256": "f81cb258434d548dca7244d599c82eb339043d8dedd0b1b807870c9d263117f2",