From e5c60482b97789bbaa309fe964c039d03ddbdf51 Mon Sep 17 00:00:00 2001 From: Randy <857271370@qq.com> Date: Thu, 15 Oct 2026 15:01:30 +0800 Subject: [PATCH 1/3] =?UTF-8?q?=F0=9F=94=A7=20chore:=20pin=20benchmarks=20?= =?UTF-8?q?to=20the=20current=20Weir=20contract?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- go.mod | 4 ++-- go.sum | 12 ++++-------- internal/workload/weir_batch_test.go | 5 ++++- tests/integration/stream_memory_test.go | 9 ++++----- versions.json | 10 +++++----- 5 files changed, 19 insertions(+), 21 deletions(-) diff --git a/go.mod b/go.mod index 11c6d51b..ec368b09 100644 --- a/go.mod +++ b/go.mod @@ -3,8 +3,8 @@ module github.com/batchstream/weir-tests go 1.27.1 require ( - github.com/batchstream/weir-go v0.10.0 - github.com/batchstream/weir-protocol v0.8.0 + github.com/batchstream/weir-go v0.15.0 + github.com/batchstream/weir-protocol v0.13.0 go.mongodb.org/mongo-driver/v2 v2.9.1 google.golang.org/grpc v1.83.2 ) diff --git a/go.sum b/go.sum index 67c1b9e3..c51fec8e 100644 --- a/go.sum +++ b/go.sum @@ -1,11 +1,7 @@ -github.com/batchstream/weir-go v0.9.0 h1:a4pGdTJd9/5Y8QMTz37ipHpfSXztpPB84Wm4Wl0fc7A= -github.com/batchstream/weir-go v0.9.0/go.mod h1:lAlK3m/CBl+WZUhMIKwhGdqoeb4wnoteKWppyCfgZLo= -github.com/batchstream/weir-go v0.10.0 h1:D4sv8YoXqUP44W+jd/DsbsW6NkZqK5FTbNFWqhDaABA= -github.com/batchstream/weir-go v0.10.0/go.mod h1:ud7wNpCOrnNuIfwu7CPxkveku9qNi9NrBxuNv44NBfk= -github.com/batchstream/weir-protocol v0.7.0 h1:+LSSLPCWv8GL8RgkH06LYNT8wDN5tFRAs0rle1Ym4sY= -github.com/batchstream/weir-protocol v0.7.0/go.mod h1:hHmwPoE9yHYAz7Auc3a14Ijo/uxAef4sQ9nBtyXNcGc= -github.com/batchstream/weir-protocol v0.8.0 h1:A+diMm2VPP0YoAsnVQJTW71rf1vV+IVMyUQ02zta2Oo= -github.com/batchstream/weir-protocol v0.8.0/go.mod h1:hHmwPoE9yHYAz7Auc3a14Ijo/uxAef4sQ9nBtyXNcGc= +github.com/batchstream/weir-go v0.15.0 h1:Y63iiiiimksBp6lslITghcnQxmqp51ytb4jT8fASm6o= +github.com/batchstream/weir-go v0.15.0/go.mod h1:wwaM3ZwPIv2LN9W+BjTUoKNRlGVkwj04tHxg1xaIe5k= +github.com/batchstream/weir-protocol v0.13.0 h1:kXxZOq30a8rgYFrxpV0pdXalNH6ttvM6qgCkrxsxi9U= +github.com/batchstream/weir-protocol v0.13.0/go.mod h1:hHmwPoE9yHYAz7Auc3a14Ijo/uxAef4sQ9nBtyXNcGc= github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs= github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= diff --git a/internal/workload/weir_batch_test.go b/internal/workload/weir_batch_test.go index 1939b62a..5cd112a3 100644 --- a/internal/workload/weir_batch_test.go +++ b/internal/workload/weir_batch_test.go @@ -19,7 +19,6 @@ import ( ) type acknowledgementPeer struct { - pb.UnimplementedStoreServiceServer mode string calls atomic.Int64 applied atomic.Int64 @@ -27,6 +26,10 @@ type acknowledgementPeer struct { firstMutation chan struct{} } +func (p *acknowledgementPeer) ResolveStore(context.Context, *pb.ResolveStoreRequest) (*pb.ResolveStoreResponse, error) { + return nil, status.Error(codes.Unimplemented, "test peer handles direct execution only") +} + func (p *acknowledgementPeer) Execute(stream grpc.BidiStreamingServer[pb.ExecuteRequest, pb.ExecuteResponse]) error { p.calls.Add(1) for { diff --git a/tests/integration/stream_memory_test.go b/tests/integration/stream_memory_test.go index 2d765acf..b7c4f929 100644 --- a/tests/integration/stream_memory_test.go +++ b/tests/integration/stream_memory_test.go @@ -15,7 +15,6 @@ import ( "time" weir "github.com/batchstream/weir-go" - "github.com/batchstream/weir-protocol/api/protocol" "github.com/batchstream/weir-tests/internal/fixture" "github.com/batchstream/weir-tests/internal/observe" "go.mongodb.org/mongo-driver/v2/bson" @@ -24,8 +23,8 @@ import ( const streamPendingRecords = 32 // A ReadStream uses one Execute RPC with no other business RPCs active here. -// Each owner's reservations are bounded by its transport record window and the -// public protocol event size; these credits describe ownership rather than RSS. +// Each owner's retained records fit its transport window. Byte reservations +// reflect backend document limits and live results rather than a protocol cap. func streamReservations(ctx context.Context, nodes []fixture.Node, store string) (bool, error) { idle := true labels := map[string]string{"store": store} @@ -46,10 +45,10 @@ func streamReservations(ctx context.Context, nodes []fixture.Node, store string) values[name] = value idle = idle && value == 0 } - if values["result_reserved_entries"] > streamPendingRecords || values["result_reserved_bytes"] > streamPendingRecords*protocol.MaxEvent || + if values["result_reserved_entries"] > streamPendingRecords || values["pending_entries"] > values["result_reserved_entries"] || values["retained_results"] > values["result_reserved_entries"] || values["retained_result_reserved_bytes"] > values["result_reserved_bytes"] { - return false, fmt.Errorf("Store %s stream reservations exceeded transport/protocol ownership bounds: %v", store, values) + return false, fmt.Errorf("Store %s stream reservations exceeded transport ownership bounds: %v", store, values) } } return idle, nil diff --git a/versions.json b/versions.json index ff3fd516..a066835b 100644 --- a/versions.json +++ b/versions.json @@ -1,13 +1,13 @@ { "go": "1.27.1", - "sdk": "v0.10.0", - "protocol": "v0.8.0", - "weir_source": "d5ad652e30ccf381678b911816059fe36faff955", + "sdk": "v0.15.0", + "protocol": "v0.13.0", + "weir_source": "8955cde10e574dae55d890b0c9c824fd39219d1c", "mongodb_version": "8.0.32", "elasticsearch_version": "8.19.22", "elasticsearch_image": "docker.elastic.co/elasticsearch/elasticsearch@sha256:e98f9c3b09beb2fbb9eaf667d602df3f0e00bd3644138b8458dc17ba1a675595", - "weir_version": "v0.1.1-0.20261010150620-d5ad652e30cc", - "weir_sum": "h1:+y3CjKbh9uvma53J0tGPYBTQWXDGP9aNkTiXTM70iE4=", + "weir_version": "v0.2.1-0.20261015053327-8955cde10e57", + "weir_sum": "h1:u1YqjXPBmo6sIxaZUGaGW4IEx8Z5XUzq2BdNDnN9/zE=", "mongodb_image": "mongo@sha256:d0d926f94df099bff534b7ee5b5986458131a22489dfff8664509af0c1e2ca9c", "mongodb_darwin_arm64_url": "https://fastdl.mongodb.org/osx/mongodb-macos-arm64-8.0.32.tgz", "mongodb_darwin_arm64_sha256": "f81cb258434d548dca7244d599c82eb339043d8dedd0b1b807870c9d263117f2", From 99d7951bc935c811bc155a40e390ef7a200e3fa5 Mon Sep 17 00:00:00 2001 From: Randy <857271370@qq.com> Date: Thu, 15 Oct 2026 15:01:30 +0800 Subject: [PATCH 2/3] =?UTF-8?q?=E2=9C=85=20test:=20measure=20sustained=20w?= =?UTF-8?q?orkloads=20against=20native=20controls?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .github/workflows/saturation.yml | 85 ++++++++++----- Makefile | 2 +- README.md | 38 +++++-- cmd/weir-lab/main.go | 2 +- internal/benchmark/command_counters.go | 17 +++ internal/benchmark/command_counters_test.go | 27 ++++- internal/benchmark/saturation.go | 22 ++++ internal/benchmark/single_saturation.go | 15 +-- internal/workload/dataset.go | 3 + internal/workload/dataset_test.go | 17 +++ internal/workload/lua.go | 109 +++----------------- internal/workload/lua_test.go | 88 ++++++++++------ internal/workload/paths.go | 8 +- scripts/performance_tools_test.py | 13 ++- scripts/run_performance_matrix.py | 26 +++-- 15 files changed, 279 insertions(+), 193 deletions(-) create mode 100644 internal/benchmark/command_counters.go diff --git a/.github/workflows/saturation.yml b/.github/workflows/saturation.yml index 7362343c..c76a81f3 100644 --- a/.github/workflows/saturation.yml +++ b/.github/workflows/saturation.yml @@ -17,10 +17,9 @@ on: options: ['1', '0.5', '2'] client-levels: description: Total concurrency across four independent client processes - default: '8,32,128' + default: '4,8,32,128,256' required: true - type: choice - options: ['8,32,128', '8,32,128,512'] + type: string backend-batch-limit: description: Server adapter grouping limit (1 disables cross-RPC aggregation) default: '32' @@ -32,6 +31,23 @@ on: default: false required: true type: boolean + backend: + description: Owned database backend + default: all + required: true + type: choice + options: [all, mongo, search] + mode: + description: Single-record business requests or supplemental client bulk calls + default: saturation + required: true + type: choice + options: [saturation, bulk-saturation] + batch-sizes: + description: Records per call; saturation requires 1, bulk-saturation accepts 1 through 64 + default: '1' + required: true + type: string pull_request: paths-ignore: - 'results/**' @@ -40,7 +56,7 @@ permissions: contents: read concurrency: - group: saturation-${{ github.ref }}-${{ inputs.write-percent || 'matrix' }}-${{ inputs.database-cpus || '1' }}-${{ inputs.client-levels || '8,32,128' }}-${{ inputs.backend-batch-limit || '32' }}-${{ inputs.lua-mutations || false }} + group: saturation-${{ github.ref }}-${{ inputs.write-percent || 'matrix' }}-${{ inputs.database-cpus || '1' }}-${{ inputs.client-levels || '4,8,32,128,256' }}-${{ inputs.backend-batch-limit || '32' }}-${{ inputs.lua-mutations || false }}-${{ inputs.backend || 'all' }}-${{ inputs.mode || 'saturation' }}-${{ inputs.batch-sizes || '1' }} cancel-in-progress: true defaults: @@ -61,7 +77,7 @@ jobs: matrix: write-percent: ${{ fromJSON(github.event_name == 'workflow_dispatch' && format('[{0}]', inputs.write-percent) || '[0,10,100]') }} runs-on: ubuntu-24.04 - timeout-minutes: 55 + timeout-minutes: 120 steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 with: @@ -81,34 +97,50 @@ jobs: GOSUMDB: 'off' WRITE_PERCENT: ${{ matrix.write-percent }} DATABASE_CPUS: ${{ inputs.database-cpus || '1' }} - CLIENT_LEVELS: ${{ inputs.client-levels || '8,32,128' }} + CLIENT_LEVELS: ${{ inputs.client-levels || '4,8,32,128,256' }} BACKEND_BATCH_LIMITS: ${{ inputs.backend-batch-limit || '32' }} LUA_MUTATIONS: ${{ inputs.lua-mutations || false }} + BACKEND: ${{ inputs.backend || 'all' }} + MODE: ${{ inputs.mode || 'saturation' }} + BATCH_SIZES: ${{ inputs.batch-sizes || '1' }} run: | mkdir -p results/local - go build -o .tools/weir-lab ./cmd/weir-lab - IFS=',' read -r -a batch_limits <<< "$BACKEND_BATCH_LIMITS" - lua_flags=() - if [ "$LUA_MUTATIONS" = true ]; then - lua_flags=(-lua-mutations) - fi - for backend_limit in "${batch_limits[@]}"; do - export BACKEND_BATCH_LIMIT="$backend_limit" - export MEASUREMENT_OUTPUT="results/local/saturation-batch-$backend_limit" - export MEASUREMENT_RECEIPT="results/local/measurement-batch-$backend_limit.receipt.json" - python3 - <<'PY' - import hashlib, json, os, pathlib, subprocess + go build -trimpath -buildvcs=false -o .tools/weir-lab ./cmd/weir-lab + python3 - <<'PY' + import hashlib, json, os, pathlib, platform, subprocess binary = pathlib.Path('.tools/weir-lab') - command = [str(binary), '-mode', 'saturation', '-weir', '.tools/weir', '-backend', 'all', '-database-cpus', os.environ['DATABASE_CPUS'], '-backend-batch-limit', os.environ['BACKEND_BATCH_LIMIT'], '-client-processes', '4', '-concurrency-levels', os.environ['CLIENT_LEVELS'], '-batch-sizes', '1', '-records', '2048', '-payload-bytes', '1024', '-write-percent', os.environ['WRITE_PERCENT'], '-rounds', '3', '-warmup-duration', '10s', '-duration', '20s', '-output', os.environ['MEASUREMENT_OUTPUT']] - if os.environ['LUA_MUTATIONS'] == 'true': - command.append('-lua-mutations') - receipt = {'client_operation_timeout': '10s', 'host_memory_bytes': int(next(line for line in pathlib.Path('/proc/meminfo').read_text().splitlines() if line.startswith('MemTotal:')).split()[1]) * 1024, 'source': subprocess.check_output(['git', 'rev-parse', 'HEAD'], text=True).strip(), 'source_dirty': bool(subprocess.check_output(['git', 'status', '--porcelain'])), 'client_binary_sha256': hashlib.sha256(binary.read_bytes()).hexdigest(), 'client_build': subprocess.check_output(['go', 'version', '-m', str(binary)], text=True), 'versions': json.loads(pathlib.Path('versions.json').read_text()), 'command': command} - if receipt['source_dirty']: + server = pathlib.Path('.tools/weir') + source = subprocess.check_output(['git', 'rev-parse', 'HEAD'], text=True).strip() + if subprocess.check_output(['git', 'status', '--porcelain']): raise SystemExit('measurement requires a clean source checkout') - pathlib.Path(os.environ['MEASUREMENT_RECEIPT']).write_text(json.dumps(receipt, indent=2) + '\n') + server_receipt = json.loads(server.with_suffix('.receipt.json').read_text()) + server_sha = hashlib.sha256(server.read_bytes()).hexdigest() + if server_receipt['sha256'] != server_sha: + raise SystemExit('server binary does not match its preparation receipt') + for backend_limit in os.environ['BACKEND_BATCH_LIMITS'].split(','): + if not backend_limit.isdecimal() or not 1 <= int(backend_limit) <= 1024: + raise SystemExit('backend batch limits must be integers from 1 through 1024') + output = 'results/local/saturation-batch-' + backend_limit + command = [str(binary), '-mode', os.environ['MODE'], '-weir', str(server), '-backend', os.environ['BACKEND'], '-database-cpus', os.environ['DATABASE_CPUS'], '-backend-batch-limit', backend_limit, '-client-processes', '4', '-concurrency-levels', os.environ['CLIENT_LEVELS'], '-batch-sizes', os.environ['BATCH_SIZES'], '-records', '8192', '-payload-bytes', '1024', '-write-percent', os.environ['WRITE_PERCENT'], '-rounds', '3', '-warmup-duration', '10s', '-duration', '20s', '-output', output] + if os.environ['LUA_MUTATIONS'] == 'true': + command.append('-lua-mutations') + receipt = {'client_operation_timeout': '10s', 'host_memory_bytes': int(next(line for line in pathlib.Path('/proc/meminfo').read_text().splitlines() if line.startswith('MemTotal:')).split()[1]) * 1024, 'host_cpus': os.cpu_count(), 'host_cpu_model': next(line.split(':', 1)[1].strip() for line in pathlib.Path('/proc/cpuinfo').read_text().splitlines() if line.startswith('model name')), 'go': subprocess.check_output(['go', 'version'], text=True).strip(), 'host': platform.uname()._asdict(), 'source': source, 'source_dirty': False, 'client_binary_sha256': hashlib.sha256(binary.read_bytes()).hexdigest(), 'client_build': subprocess.check_output(['go', 'version', '-m', str(binary)], text=True), 'server_binary_sha256': server_sha, 'server_receipt': server_receipt, 'versions': json.loads(pathlib.Path('versions.json').read_text()), 'command': command, 'workflow_run': 'https://github.com/' + os.environ['GITHUB_REPOSITORY'] + '/actions/runs/' + os.environ['GITHUB_RUN_ID']} + receipt_path = pathlib.Path('results/local/measurement-batch-' + backend_limit + '.receipt.json') + receipt_path.write_text(json.dumps(receipt, indent=2) + '\n') + with pathlib.Path(output + '.log').open('x') as log: + process = subprocess.Popen(command, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, text=True) + for line in process.stdout: + print(line, end='', flush=True) + log.write(line) + receipt['exit_code'] = process.wait() + receipt['server_binary_after_sha256'] = hashlib.sha256(server.read_bytes()).hexdigest() + receipt['client_binary_after_sha256'] = hashlib.sha256(binary.read_bytes()).hexdigest() + receipt_path.write_text(json.dumps(receipt, indent=2) + '\n') + if receipt['server_binary_after_sha256'] != server_sha or receipt['client_binary_after_sha256'] != receipt['client_binary_sha256']: + raise SystemExit('benchmark binary changed during measurement') + if receipt['exit_code']: + raise SystemExit(receipt['exit_code']) PY - .tools/weir-lab -mode saturation -weir .tools/weir -backend all -database-cpus "$DATABASE_CPUS" -backend-batch-limit "$BACKEND_BATCH_LIMIT" -client-processes 4 -concurrency-levels "$CLIENT_LEVELS" -batch-sizes 1 -records 2048 -payload-bytes 1024 -write-percent "$WRITE_PERCENT" -rounds 3 -warmup-duration 10s -duration 20s -output "$MEASUREMENT_OUTPUT" "${lua_flags[@]}" 2>&1 | tee "results/local/saturation-batch-$backend_limit.log" - done git diff --exit-code - name: Preserve raw resource samples, workload validation and cleanup if: always() @@ -117,6 +149,7 @@ jobs: name: saturation-evidence-writes-${{ matrix.write-percent }}-${{ github.sha }} path: | results/local/ + .tools/weir.receipt.json /tmp/weir-tests-*/manifest.json /tmp/weir-tests-*/cleanup.json /tmp/weir-tests-*/*config.json diff --git a/Makefile b/Makefile index 304bddc3..f3107e46 100644 --- a/Makefile +++ b/Makefile @@ -13,7 +13,7 @@ integration: WEIR_TEST_BINARY="$(CURDIR)/.tools/weir" GOWORK=off go test -race -tags=integration -count=1 -timeout=10m -v ./tests/integration benchmark: - GOWORK=off go run ./cmd/weir-lab -mode saturation -weir "$(CURDIR)/.tools/weir" -backend all -database-cpus 1 -client-processes 4 -concurrency-levels 8,32,128 -batch-sizes 1 -records 2048 -write-percent 0 -rounds 3 -warmup-duration 10s -duration 20s -output results/local/benchmark + GOWORK=off go run ./cmd/weir-lab -mode saturation -weir "$(CURDIR)/.tools/weir" -backend all -database-cpus 1 -client-processes 4 -concurrency-levels 4,8,32,128,256 -batch-sizes 1 -records 8192 -write-percent 0 -rounds 3 -warmup-duration 10s -duration 20s -output results/local/benchmark diagnostic: GOWORK=off go run ./cmd/weir-lab -mode fixed -weir "$(CURDIR)/.tools/weir" -output results/local/diagnostic diff --git a/README.md b/README.md index 32b07926..61a791ac 100644 --- a/README.md +++ b/README.md @@ -87,8 +87,8 @@ make benchmark go run ./cmd/weir-lab -mode saturation \ -weir .tools/weir -backend all -database-cpus 1 \ -client-processes 4 \ - -concurrency-levels 8,32,128 -batch-sizes 1 \ - -records 2048 -payload-bytes 1024 -write-percent 0 \ + -concurrency-levels 4,8,32,128,256 -batch-sizes 1 \ + -records 8192 -payload-bytes 1024 -write-percent 0 \ -rounds 3 -warmup-duration 10s -duration 20s \ -output results/local/read ``` @@ -103,13 +103,20 @@ AB/BA order alternates across rounds. Failed calls or uncertain acknowledgements invalidate the comparison. For Lua read-modify-write, add `-lua-mutations -write-percent 100`. Each native -MongoDB request uses a snapshot transaction with one read, revision computation, -replacement and majority commit. Each native Search request uses one real-time -read and a write conditioned on its observed sequence number and primary term. +MongoDB request uses one atomic `UpdateOne` aggregation pipeline, with +`$set: {revision: {$subtract: [int32(1), "$revision"]}}` and majority acknowledgement. +Each native Search request uses one `POST _update` Painless script with +`ctx._source.revision = 1 - ctx._source.revision` and `retry_on_conflict=0`. +Neither baseline performs a client read or transaction. These are native controls +for this fixed transformation, rather than a claim about arbitrary Lua programs. The SDK sends one Lua `AtomicTransform` per request with the same computation. The function updates `current.revision = 1 - current.revision` and returns the -current document; existing BSON int32 revision fields retain their width. +current document; independent postflight checks require the BSON revision to +retain its int32 type and verify every field and final revision on both paths. Fixture documents are identical on both paths; Weir adds no document metadata. +The logical request/result document-byte counters exclude scripts and commands, +so they are zero for transform-only calls. Backend reads and replacements still +contribute to the measured network and CPU costs. Repeat with `-backend-batch-limit 1` to disable server aggregation. Keep other settings identical and use a fresh output directory. The dispatch workflow can @@ -120,7 +127,9 @@ Reports retain per-round throughput, pooled latency histograms, client process identities, start lag, CPU/RSS, database CPU/memory/I/O/network samples and raw Weir metrics. Metrics report stream completions, terminal records, queue wait, adapter time, rejections and operations per adapter invocation. MongoDB also -records physical command deltas. Adapter invocations can split by namespace, +records selected business-command deltas at the timed boundaries for both +single-record and bulk calls. These counters exclude handshakes and monitoring +commands. Adapter invocations can split by namespace, action and byte limits; their counts are distinct from database wire commands. Missing samples and reset counters are reported as unavailable. `business_operation_timeouts` counts failed logical operations whose request context @@ -169,12 +178,17 @@ prove Linux database saturation or a production capacity limit. `scripts/run_performance_matrix.py` serializes read, mixed (10%), write and Lua cases at total concurrency 8/32, with three alternating native/Weir rounds, -2s warmup and 5s measurement for native MongoDB, or 20s warmup and 20s +10s warmup and 20s measurement for MongoDB, or 20s warmup and 20s measurement for Elasticsearch per stage. Longer Search stages gave better resource sampling coverage and avoided a short-run regression that did not reproduce in the adjacent longer comparison. These are measurement defaults; the server batching and transport defaults remain 32 records. -`--warmup-duration` and `--duration` allow explicit overrides. It retains exact commands, client source +`--warmup-duration`, `--duration`, `--database-cpus`, `--concurrency-levels`, +`--client-processes`, `--records`, `--payload-bytes` and `--rounds` expose the +experiment controls. Docker databases default to an enforced 1 CPU quota and an +8192-document hot working set. +`--bulk-concurrency-levels` separately controls bulk comparisons, whose maximum +concurrency is 64. It retains exact commands, client source snapshots, binary hashes, JSON/Markdown reports and copied fixture cleanup/log evidence. The runner uses an explicit `--server-receipt` or automatically loads the adjacent `.receipt.json`, passing frozen local receipts to the lab while @@ -236,7 +250,11 @@ an explicit flag; credential-bearing URLs are rejected. CI runs offline unit/race/vet and dependency checks, real multi-node integration, and verified ordinary and Lua comparison smoke tests on both databases. `saturation.yml` runs read/mixed/write matrices on PRs and accepts CPU quota, -concurrency and grouping controls on dispatch. Workflow artifacts preserve raw +concurrency, backend, workload mode, records per call and grouping controls on +dispatch. Its default independent-client curve is 4/8/32/128/256, with three +alternating rounds and 10s warmup plus 20s measurement per path. Supplemental +`bulk-saturation` runs require explicit concurrency at most 64 and a batch size +such as 32. Workflow artifacts preserve raw reports and owned-resource cleanup receipts. Generated measurements belong in ignored output directories under `results/`. diff --git a/cmd/weir-lab/main.go b/cmd/weir-lab/main.go index 45640a1e..5efb4418 100644 --- a/cmd/weir-lab/main.go +++ b/cmd/weir-lab/main.go @@ -76,7 +76,7 @@ func main() { flag.IntVar(&opts.concurrency, "concurrency", 8, "client workers for finite diagnostics") flag.IntVar(&opts.rounds, "rounds", 3, "alternating paired rounds") flag.IntVar(&opts.writePercent, "write-percent", 10, "write fraction from 0 through 100") - flag.BoolVar(&opts.luaMutations, "lua-mutations", false, "single-record read-modify-write: native transaction/OCC versus Weir Lua; no client batching") + flag.BoolVar(&opts.luaMutations, "lua-mutations", false, "single-record mutation: native atomic update pipeline/server script versus Weir Lua; no client batching") flag.StringVar(&opts.mode, "mode", "saturation", "saturation sends independent single requests from OS client processes; bulk-saturation measures bulk overhead; fixed is a finite diagnostic") flag.StringVar(&opts.levels, "concurrency-levels", "8,32,128", "ascending total client concurrency, at most 512 in single-request mode") flag.StringVar(&opts.batches, "batch-sizes", "1", "records per call: saturation requires 1; bulk-saturation accepts 1 through 64") diff --git a/internal/benchmark/command_counters.go b/internal/benchmark/command_counters.go new file mode 100644 index 00000000..5a673d52 --- /dev/null +++ b/internal/benchmark/command_counters.go @@ -0,0 +1,17 @@ +package benchmark + +func (result *SaturationResult) calculatePhysicalCommands() { + if result.DatabaseBefore == nil || result.DatabaseAfter == nil { + return + } + result.PhysicalCommandUnavailable = mergeCommandUnavailable(result.PhysicalCommandUnavailable, result.DatabaseBefore.CommandUnavailable, result.DatabaseAfter.CommandUnavailable) + result.PhysicalCommands = make(map[string]uint64) + for name, after := range result.DatabaseAfter.Commands { + before, found := result.DatabaseBefore.Commands[name] + if !found || after < before { + result.PhysicalCommandUnavailable = mergeCommandUnavailable(result.PhysicalCommandUnavailable, "unavailable or reset "+name) + continue + } + result.PhysicalCommands[name] = after - before + } +} diff --git a/internal/benchmark/command_counters_test.go b/internal/benchmark/command_counters_test.go index c2e431f2..f2a01dfd 100644 --- a/internal/benchmark/command_counters_test.go +++ b/internal/benchmark/command_counters_test.go @@ -1,6 +1,31 @@ package benchmark -import "testing" +import ( + "strings" + "testing" + + "github.com/batchstream/weir-tests/internal/workload" +) + +func TestPhysicalCommandDeltaPreservesUnavailableAndRejectsResets(t *testing.T) { + before := &workload.DatabaseCounters{Commands: map[string]uint64{"find": 100, "update": 30}, CommandUnavailable: "missing bulkWrite"} + after := &workload.DatabaseCounters{Commands: map[string]uint64{"find": 115, "update": 2, "getMore": 1}, CommandUnavailable: "missing bulkWrite"} + result := SaturationResult{DatabaseBefore: before, DatabaseAfter: after, PhysicalCommandUnavailable: "sampling warning"} + result.calculatePhysicalCommands() + if len(result.PhysicalCommands) != 1 || result.PhysicalCommands["find"] != 15 { + t.Fatal("missing or reset counters counted as measured work", result.PhysicalCommands) + } + for _, reason := range []string{"sampling warning", "missing bulkWrite", "unavailable or reset update", "unavailable or reset getMore"} { + if strings.Count(result.PhysicalCommandUnavailable, reason) != 1 { + t.Fatal("counter evidence was hidden or duplicated", reason, result.PhysicalCommandUnavailable) + } + } + missing := SaturationResult{DatabaseAfter: after} + missing.calculatePhysicalCommands() + if missing.PhysicalCommands != nil { + t.Fatal("missing initial observation fabricated command counts") + } +} func TestCommandUnavailableMergesBeforeAfterReasonsWithoutDuplicates(t *testing.T) { before := "missing bulkWrite; missing getMore; " diff --git a/internal/benchmark/saturation.go b/internal/benchmark/saturation.go index 74ef4c3f..dd82a35f 100644 --- a/internal/benchmark/saturation.go +++ b/internal/benchmark/saturation.go @@ -264,6 +264,16 @@ type durationMeasureOptions struct { } func measureDuration(ctx context.Context, options durationMeasureOptions) SaturationResult { + var beforeDatabase, afterDatabase *workload.DatabaseCounters + var counterErrors []string + if options.Sample && options.Paths != nil { + counters, err := options.Paths.DatabaseCounters(ctx) + if err != nil { + counterErrors = append(counterErrors, err.Error()) + } else { + beforeDatabase = &counters + } + } var metricsBefore observe.Snapshot measureServer := options.Sample && options.Executor.Name() == "weir" if measureServer { @@ -386,9 +396,21 @@ func measureDuration(ctx context.Context, options durationMeasureOptions) Satura if measureServer { metricsAfter = fetchServerMetrics(ctx, options.Target.Diagnostics) } + if options.Sample && options.Paths != nil { + counters, err := options.Paths.DatabaseCounters(ctx) + if err != nil { + counterErrors = append(counterErrors, err.Error()) + } else { + afterDatabase = &counters + } + } base := Result{Path: options.Executor.Name(), ElapsedNS: elapsed.Nanoseconds(), Evidence: options.Executor.Evidence()} base.Evidence.ClientBatch = fmt.Sprintf("native database bulk / same-Store SDK batch, %d distinct records per request", options.State.batch) result := SaturationResult{Result: base, Resources: resources} + result.DatabaseBefore = beforeDatabase + result.DatabaseAfter = afterDatabase + result.PhysicalCommandUnavailable = mergeCommandUnavailable(counterErrors...) + result.calculatePhysicalCommands() if measureServer { result.ServerMetrics = serverMetricDelta(metricsBefore, metricsAfter, options.State.dataset.Config.StoreName) } diff --git a/internal/benchmark/single_saturation.go b/internal/benchmark/single_saturation.go index a5ec4d5a..bb8f7fb5 100644 --- a/internal/benchmark/single_saturation.go +++ b/internal/benchmark/single_saturation.go @@ -71,7 +71,7 @@ func runSingleSaturation(ctx context.Context, options SaturationOptions) (*Satur provenance["client_process_model"] = "independent OS client processes; one SDK/driver pool each; sequential single-request goroutine workers; total native pool limit equals total client concurrency" report := &SaturationReport{Schema: 3, Started: time.Now().UTC().Format(time.RFC3339Nano), Parameters: params, Provenance: provenance, CPUQuota: quota, Method: "independent OS client processes send one record per native call or streaming SDK RPC; native FindOne/ReplaceOne or single-ID mget/PUT; no client aggregation; worker-owned disjoint IDs; common phase barriers and unchanged pools across warmup and measurement; alternating AB/BA; latency is each business request including validation, queueing and any server aggregation; independent persisted postflight; sustained database CPU requires >=5 intervals, >=80% measurement coverage, mean CPU>=threshold and CPU>=threshold during >=80% measurement duration in every round, plus <=10% next-level throughput gain"} if options.Dataset.Config.LuaMutations { - report.Method = strings.Replace(report.Method, "native FindOne/ReplaceOne or single-ID mget/PUT", "native single-record snapshot transaction FindOne/compute/ReplaceOne/commit or real-time single-ID mget/compute/version-conditional PUT; SDK Lua AtomicTransform computes the same revision toggle from the current source", 1) + report.Method = strings.Replace(report.Method, "native FindOne/ReplaceOne or single-ID mget/PUT", "native single-record atomic MongoDB update pipeline or Elasticsearch server-side update script; SDK Lua AtomicTransform computes the same revision toggle from the current source", 1) } for level, workers := range options.Concurrency { for round := range options.Rounds { @@ -238,18 +238,7 @@ func runSingleStage(ctx context.Context, opts singleStageOptions) (result Satura } else { result.PhysicalCommandUnavailable = mergeCommandUnavailable(result.PhysicalCommandUnavailable, counterErr.Error()) } - if result.DatabaseBefore != nil && result.DatabaseAfter != nil { - result.PhysicalCommandUnavailable = mergeCommandUnavailable(result.DatabaseBefore.CommandUnavailable, result.DatabaseAfter.CommandUnavailable) - result.PhysicalCommands = make(map[string]uint64) - for name, after := range result.DatabaseAfter.Commands { - before, found := result.DatabaseBefore.Commands[name] - if !found || after < before { - result.PhysicalCommandUnavailable = mergeCommandUnavailable(result.PhysicalCommandUnavailable, "unavailable or reset "+name) - continue - } - result.PhysicalCommands[name] = after - before - } - } + result.calculatePhysicalCommands() plan := &workload.Plan{Expected: make([]int, options.Dataset.Config.Records)} covered := make([]bool, len(plan.Expected)) var histogram histogram diff --git a/internal/workload/dataset.go b/internal/workload/dataset.go index 00d558b6..3ddb937a 100644 --- a/internal/workload/dataset.go +++ b/internal/workload/dataset.go @@ -184,6 +184,9 @@ func (d *Dataset) Validate(raw []byte, operation Operation) error { if !ok || id != d.ID(operation.Record) { return errors.New("persisted BSON _id mismatch") } + if _, ok := bson.Raw(raw).Lookup("revision").Int32OK(); !ok { + return errors.New("persisted BSON revision must retain int32 type") + } err = bson.Unmarshal(raw, &actual) } else { var fields map[string]json.RawMessage diff --git a/internal/workload/dataset_test.go b/internal/workload/dataset_test.go index 606032ce..4f2f6470 100644 --- a/internal/workload/dataset_test.go +++ b/internal/workload/dataset_test.go @@ -5,6 +5,8 @@ import ( "encoding/json" "reflect" "testing" + + "go.mongodb.org/mongo-driver/v2/bson" ) func testDataset(t *testing.T, backend string) *Dataset { @@ -170,3 +172,18 @@ func TestWorkloadBoundsAndOwnedNamespace(t *testing.T) { t.Fatal("accepted fewer operations than workers") } } + +func TestMongoValidationRejectsChangedRevisionType(t *testing.T) { + dataset := testDataset(t, "mongo") + operation := Operation{Record: 2, Revision: 1} + for _, revision := range []any{int64(1), float64(1), "1"} { + fields := bson.D{{Key: "_id", Value: dataset.ID(2)}, {Key: "fixture_id", Value: dataset.ID(2)}, {Key: "revision", Value: revision}, {Key: "padding", Value: dataset.padding}} + raw, err := bson.Marshal(fields) + if err != nil { + t.Fatal(err) + } + if err := dataset.Validate(raw, operation); err == nil { + t.Fatalf("accepted BSON revision type %T instead of int32", revision) + } + } +} diff --git a/internal/workload/lua.go b/internal/workload/lua.go index 31727278..f52a7fd4 100644 --- a/internal/workload/lua.go +++ b/internal/workload/lua.go @@ -7,15 +7,9 @@ import ( "fmt" "net/http" "net/url" - "strconv" - "time" "go.mongodb.org/mongo-driver/v2/bson" "go.mongodb.org/mongo-driver/v2/mongo" - "go.mongodb.org/mongo-driver/v2/mongo/options" - "go.mongodb.org/mongo-driver/v2/mongo/readconcern" - "go.mongodb.org/mongo-driver/v2/mongo/readpref" - "go.mongodb.org/mongo-driver/v2/mongo/writeconcern" ) func (d *Dataset) luaSource() string { @@ -26,103 +20,34 @@ func (d *Dataset) luaSource() string { end` } -// The native path performs the same single-record read-modify-write, without -// client batching or write replay. Partitioned worker IDs avoid planned conflicts. +// This fixed transformation has a native atomic implementation on both backends. +// It is not a baseline for arbitrary Lua programs. func (p *directPath) executeMongoTransform(ctx context.Context, operation Operation) Outcome { - session, err := p.mongo.StartSession() - if err != nil { - return failed(err, false) - } - defer func() { - cleanup, cancel := context.WithTimeout(context.Background(), time.Second) - defer cancel() - session.EndSession(cleanup) - }() - txnOptions := options.Transaction().SetReadConcern(readconcern.Snapshot()).SetWriteConcern(writeconcern.Majority()).SetReadPreference(readpref.Primary()) - if err := session.StartTransaction(txnOptions); err != nil { - return failed(err, false) - } - txctx := mongo.NewSessionContext(ctx, session) collection := p.mongo.Database(p.dataset.Config.Namespace).Collection("records") filter := bson.D{{Key: "_id", Value: p.dataset.ID(operation.Record)}} - raw, err := collection.FindOne(txctx, filter).Raw() - if err != nil { - return failed(err, false) - } - previous := operation - previous.Revision = 1 - operation.Revision - if err := p.dataset.Validate(raw, previous); err != nil { - return failed(err, false) - } - var replacement bson.D - if err := bson.Unmarshal(raw, &replacement); err != nil { - return failed(err, false) - } - for index := range replacement { - if replacement[index].Key == "revision" { - replacement[index].Value = int32(operation.Revision) - } - } - result, err := collection.ReplaceOne(txctx, filter, replacement) + subtract := bson.D{{Key: "$subtract", Value: bson.A{int32(1), "$revision"}}} + fields := bson.D{{Key: "revision", Value: subtract}} + stage := bson.D{{Key: "$set", Value: fields}} + pipeline := mongo.Pipeline{stage} + result, err := collection.UpdateOne(ctx, filter, pipeline) if err != nil { - return failed(err, false) + var rejection mongo.WriteException + uncertain := !errors.As(err, &rejection) || rejection.WriteConcernError != nil || len(rejection.WriteErrors) == 0 + return failed(err, uncertain) } if result == nil || !result.Acknowledged || result.MatchedCount != 1 || result.ModifiedCount != 1 { - return failed(errors.New("native transform did not change exactly one record"), false) - } - if err := session.CommitTransaction(txctx); err != nil { - return failed(err, true) + return failed(errors.New("native transform did not acknowledge exactly one changed record"), true) } - outcome := Outcome{Status: Success, Applied: true, RequestBytes: uint64(len(raw)), ResponseBytes: uint64(len(raw))} + outcome := Outcome{Status: Success, Applied: true} return outcome } func (p *directPath) executeSearchTransform(ctx context.Context, operation Operation) Outcome { id := p.dataset.ID(operation.Record) - readBody := map[string]any{"ids": []string{id}} - encoded, err := json.Marshal(readBody) - if err != nil { - return failed(err, false) - } - status, raw, err := p.request(ctx, http.MethodPost, "/"+p.dataset.Config.Namespace+"/_mget?realtime=true", encoded) - if err != nil || status != http.StatusOK { - return failed(errors.Join(fmt.Errorf("native transform read HTTP status %d", status), err), false) - } - var envelope struct { - Docs []struct { - ID string `json:"_id"` - Index string `json:"_index"` - Found *bool `json:"found"` - Source json.RawMessage `json:"_source"` - Sequence *int64 `json:"_seq_no"` - PrimaryTerm *int64 `json:"_primary_term"` - Error json.RawMessage `json:"error"` - } `json:"docs"` - } - if json.Unmarshal(raw, &envelope) != nil || len(envelope.Docs) != 1 { - return failed(errors.New("native transform requires exactly one observed record"), false) - } - observed := envelope.Docs[0] - if observed.ID != id || observed.Index != p.dataset.Config.Namespace || observed.Found == nil || !*observed.Found || observed.Sequence == nil || *observed.Sequence < 0 || observed.PrimaryTerm == nil || *observed.PrimaryTerm < 1 || len(observed.Error) != 0 { - return failed(errors.New("native transform lacks matching source and version evidence"), false) - } - previous := operation - previous.Revision = 1 - operation.Revision - if err := p.dataset.Validate(observed.Source, previous); err != nil { - return failed(err, false) - } - var replacement document - if err := json.Unmarshal(observed.Source, &replacement); err != nil { - return failed(err, false) - } - replacement.Revision = int32(operation.Revision) - payload, err := json.Marshal(replacement) - if err != nil { - return failed(err, false) - } - query := url.Values{"pipeline": {"_none"}, "refresh": {"false"}, "wait_for_active_shards": {"1"}, "timeout": {"1s"}, "if_seq_no": {strconv.FormatInt(*observed.Sequence, 10)}, "if_primary_term": {strconv.FormatInt(*observed.PrimaryTerm, 10)}} - path := "/" + p.dataset.Config.Namespace + "/_doc/" + url.PathEscape(id) + "?" + query.Encode() - status, raw, err = p.request(ctx, http.MethodPut, path, payload) + payload := []byte(`{"script":{"lang":"painless","source":"ctx._source.revision = 1 - ctx._source.revision"}}`) + query := url.Values{"refresh": {"false"}, "wait_for_active_shards": {"1"}, "timeout": {"1s"}, "retry_on_conflict": {"0"}} + path := "/" + p.dataset.Config.Namespace + "/_update/" + url.PathEscape(id) + "?" + query.Encode() + status, raw, err := p.request(ctx, http.MethodPost, path, payload) if err != nil { return failed(err, true) } @@ -141,6 +66,6 @@ func (p *directPath) executeSearchTransform(ctx context.Context, operation Opera if json.Unmarshal(raw, &reply) != nil || reply.ID != id || reply.Index != p.dataset.Config.Namespace || reply.Result != "updated" || reply.Shards.Successful < 1 || reply.Shards.Failed != 0 { return failed(errors.New("native transform write lacks a complete acknowledgement"), true) } - outcome := Outcome{Status: Success, Applied: true, RequestBytes: uint64(len(payload)), ResponseBytes: uint64(len(observed.Source))} + outcome := Outcome{Status: Success, Applied: true} return outcome } diff --git a/internal/workload/lua_test.go b/internal/workload/lua_test.go index 6976a8a2..22666307 100644 --- a/internal/workload/lua_test.go +++ b/internal/workload/lua_test.go @@ -11,57 +11,42 @@ import ( "testing" ) -func TestNativeSearchTransformConditionsWriteOnObservedVersion(t *testing.T) { +func TestNativeSearchTransformUsesOneAtomicServerScript(t *testing.T) { dataset := testDataset(t, "search") dataset.Config.LuaMutations = true operation := Operation{Record: 2, Write: true, Revision: 1} - var reads, writes atomic.Int32 + var calls atomic.Int32 server := httptest.NewServer(http.HandlerFunc(func(writer http.ResponseWriter, request *http.Request) { - if request.Method == http.MethodPost { - reads.Add(1) - var body struct { - IDs []string `json:"ids"` - } - if json.NewDecoder(request.Body).Decode(&body) != nil || len(body.IDs) != 1 || body.IDs[0] != dataset.ID(operation.Record) || request.URL.Path != "/"+dataset.Config.Namespace+"/_mget" || request.URL.Query().Get("realtime") != "true" { - t.Error("native transform did not issue exactly one real-time point read") - } - previous := operation - previous.Revision = 0 - fmt.Fprintf(writer, `{"docs":[{"_index":%q,"_id":%q,"found":true,"_seq_no":7,"_primary_term":2,"_source":%s}]}`, dataset.Config.Namespace, dataset.ID(operation.Record), dataset.Document(previous)) - return + calls.Add(1) + if request.Method != http.MethodPost || request.URL.Path != "/"+dataset.Config.Namespace+"/_update/"+dataset.ID(operation.Record) || request.URL.Query().Get("retry_on_conflict") != "0" || request.URL.Query().Get("refresh") != "false" || request.URL.Query().Get("wait_for_active_shards") != "1" || request.URL.Query().Get("timeout") != "1s" { + t.Error("native transform lost its single-update or acknowledgement policy") } - writes.Add(1) - if request.Method != http.MethodPut || request.URL.Query().Get("if_seq_no") != "7" || request.URL.Query().Get("if_primary_term") != "2" || request.URL.Query().Get("pipeline") != "_none" || request.URL.Query().Get("refresh") != "false" || request.URL.Query().Get("wait_for_active_shards") != "1" || request.URL.Query().Get("timeout") != "1s" { - t.Error("native transform lost its version or acknowledgement policy") + var body struct { + Script struct { + Lang string `json:"lang"` + Source string `json:"source"` + } `json:"script"` } - raw, err := io.ReadAll(request.Body) - if err != nil || dataset.Validate(raw, operation) != nil { - t.Error("native transform changed business document shape or produced the wrong revision", err) + if json.NewDecoder(request.Body).Decode(&body) != nil || body.Script.Lang != "painless" || body.Script.Source != "ctx._source.revision = 1 - ctx._source.revision" { + t.Error("native transform did not compute the revision from the server document") } fmt.Fprintf(writer, `{"_index":%q,"_id":%q,"result":"updated","_shards":{"successful":1,"failed":0}}`, dataset.Config.Namespace, dataset.ID(operation.Record)) })) defer server.Close() path := &directPath{dataset: dataset, search: server.Client(), baseURL: server.URL} result := path.Execute(t.Context(), operation) - if result.Status != Success || !result.Applied || reads.Load() != 1 || writes.Load() != 1 { - t.Fatal("native RMW was not one read and one acknowledged conditional write", result, reads.Load(), writes.Load()) + if result.Status != Success || !result.Applied || calls.Load() != 1 || result.RequestBytes != 0 || result.ResponseBytes != 0 { + t.Fatal("native RMW was not one acknowledged atomic update without document transfer", result, calls.Load()) } } -func TestNativeSearchTransformLostWriteAcknowledgementIsNotReplayed(t *testing.T) { +func TestNativeSearchTransformLostAcknowledgementIsNotReplayed(t *testing.T) { dataset := testDataset(t, "search") dataset.Config.LuaMutations = true operation := Operation{Record: 2, Write: true, Revision: 1} - var reads, writes atomic.Int32 + var calls atomic.Int32 server := httptest.NewServer(http.HandlerFunc(func(writer http.ResponseWriter, request *http.Request) { - if request.Method == http.MethodPost { - reads.Add(1) - previous := operation - previous.Revision = 0 - fmt.Fprintf(writer, `{"docs":[{"_index":%q,"_id":%q,"found":true,"_seq_no":7,"_primary_term":2,"_source":%s}]}`, dataset.Config.Namespace, dataset.ID(operation.Record), dataset.Document(previous)) - return - } - writes.Add(1) + calls.Add(1) _, _ = io.Copy(io.Discard, request.Body) connection, _, err := writer.(http.Hijacker).Hijack() if err != nil { @@ -73,8 +58,43 @@ func TestNativeSearchTransformLostWriteAcknowledgementIsNotReplayed(t *testing.T defer server.Close() path := &directPath{dataset: dataset, search: server.Client(), baseURL: server.URL} result := path.Execute(t.Context(), operation) - if result.Status != Indeterminate || result.Applied || reads.Load() != 1 || writes.Load() != 1 { - t.Fatal("lost write acknowledgement invented success or replayed RMW", result, reads.Load(), writes.Load()) + if result.Status != Indeterminate || result.Applied || calls.Load() != 1 { + t.Fatal("lost acknowledgement invented success or replayed the toggle", result, calls.Load()) + } +} + +func TestNativeSearchTransformRequiresCompleteAcknowledgement(t *testing.T) { + cases := []struct { + name string + status int + body string + want Status + }{ + {name: "conflict", status: http.StatusConflict, body: `{}`, want: Failed}, + {name: "missing", status: http.StatusNotFound, body: `{}`, want: Failed}, + {name: "server failure", status: http.StatusServiceUnavailable, body: `{}`, want: Indeterminate}, + {name: "wrong record", status: http.StatusOK, body: `{"_index":"%s","_id":"other","result":"updated","_shards":{"successful":1,"failed":0}}`, want: Indeterminate}, + {name: "noop", status: http.StatusOK, body: `{"_index":"%s","_id":"record-00000002","result":"noop","_shards":{"successful":1,"failed":0}}`, want: Indeterminate}, + {name: "shard failure", status: http.StatusOK, body: `{"_index":"%s","_id":"record-00000002","result":"updated","_shards":{"successful":0,"failed":1}}`, want: Indeterminate}, + } + for _, test := range cases { + t.Run(test.name, func(t *testing.T) { + dataset := testDataset(t, "search") + dataset.Config.LuaMutations = true + var calls atomic.Int32 + server := httptest.NewServer(http.HandlerFunc(func(writer http.ResponseWriter, request *http.Request) { + calls.Add(1) + writer.WriteHeader(test.status) + fmt.Fprint(writer, strings.ReplaceAll(test.body, "%s", dataset.Config.Namespace)) + })) + defer server.Close() + path := &directPath{dataset: dataset, search: server.Client(), baseURL: server.URL} + operation := Operation{Record: 2, Write: true, Revision: 1} + result := path.Execute(t.Context(), operation) + if result.Status != test.want || result.Applied || calls.Load() != 1 { + t.Fatal("failed update acknowledgement was accepted or replayed", result, calls.Load()) + } + }) } } diff --git a/internal/workload/paths.go b/internal/workload/paths.go index abce2465..0ef55579 100644 --- a/internal/workload/paths.go +++ b/internal/workload/paths.go @@ -113,7 +113,7 @@ func Open(ctx context.Context, dataset *Dataset) (*Paths, error) { } func openDirect(ctx context.Context, dataset *Dataset) (*directPath, error) { - info := Evidence{ByteScope: "logical document bytes from APPLIED write acknowledgements and validated reads; excludes URI, commands, acknowledgement bodies, framing, compression and discovery; unavailable evidence contributes zero", ClientBatch: "one logical operation per client call; no client bulk API", RetryPolicy: "no application retries"} + info := Evidence{ByteScope: "logical document bytes from APPLIED write acknowledgements and validated reads; excludes transform scripts, URI, commands, acknowledgement bodies, framing, compression and discovery; unavailable evidence contributes zero", ClientBatch: "one logical operation per client call; no client bulk API", RetryPolicy: "no application retries"} direct := &directPath{dataset: dataset, info: info} if dataset.Config.Backend == "mongo" { address, err := url.Parse(dataset.Config.MongoURI) @@ -149,7 +149,7 @@ func openDirect(ctx context.Context, dataset *Dataset) (*directPath, error) { direct.info.Version = build.Version direct.info.WritePolicy = "ReplaceOne upsert=true; primary reads; w=majority; no compression; pool=worker concurrency" if dataset.Config.LuaMutations { - direct.info.WritePolicy = "one snapshot transaction per business mutation: FindOne, compute revision toggle, ReplaceOne, majority commit; no client bulk; pool=worker concurrency" + direct.info.WritePolicy = "one atomic UpdateOne aggregation pipeline per business mutation: $set revision=$subtract(int32(1), current revision); w=majority; no client read, transaction or bulk; pool=worker concurrency" } direct.info.RetryPolicy = "retryReads=false; retryWrites=false; maxAdaptiveRetries=0" return direct, nil @@ -180,7 +180,7 @@ func openDirect(ctx context.Context, dataset *Dataset) (*directPath, error) { direct.info.Version = root.Version.Number direct.info.WritePolicy = "PUT _doc; pipeline=_none; refresh=false; wait_for_active_shards=1; timeout=1s; POST _mget?realtime=true with one ID; no compression; connections=worker concurrency" if dataset.Config.LuaMutations { - direct.info.WritePolicy = "single-ID real-time _mget, compute revision toggle, PUT _doc with observed if_seq_no/if_primary_term; pipeline=_none; refresh=false; wait_for_active_shards=1; timeout=1s; no client bulk" + direct.info.WritePolicy = "one POST _update Painless script per business mutation: ctx._source.revision=1-ctx._source.revision; retry_on_conflict=0; refresh=false; wait_for_active_shards=1; timeout=1s; no client read or bulk" } direct.info.RetryPolicy = "no application retries; business POST/PUT have no idempotency key and are not transparently replayed by http.Transport" return direct, nil @@ -486,7 +486,7 @@ func (p *weirPath) Execute(ctx context.Context, operation Operation) Outcome { request := &weir.AtomicTransformRequest{Resource: p.dataset.Resource(operation.Record), Lua: program} opts := weir.AtomicTransformOptions{StoreName: p.dataset.Config.StoreName, Request: request} result, err := p.client.AtomicTransform(ctx, opts) - return mutationOutcome(result, len(program.Source), err) + return mutationOutcome(result, 0, err) } document := &weir.Document{ContentType: p.dataset.ContentType(), Data: p.dataset.Document(operation)} request := &weir.WriteRequest{Resource: p.dataset.Resource(operation.Record), Document: document} diff --git a/scripts/performance_tools_test.py b/scripts/performance_tools_test.py index b318870d..8bddd9dd 100644 --- a/scripts/performance_tools_test.py +++ b/scripts/performance_tools_test.py @@ -147,7 +147,10 @@ def test_automatic_and_explicit_receipts_select_local_or_locked_lab_flags(self): receipt['server_binary_sha256'] = receipt['server_receipt']['sha256'] server_receipt_path = weir.with_suffix('.receipt.json') server_receipt_path.write_text(json.dumps(receipt['server_receipt'])) - arguments = ['run_performance_matrix.py', '--backend', 'mongo', '--weir', str(weir), '--output', str(output)] + arguments = ['run_performance_matrix.py', '--backend', 'mongo', '--weir', str(weir), '--output', str(output), + '--database-cpus', '1', '--concurrency-levels', '4,16,64,256', + '--client-processes', '4', '--records', '2048', '--payload-bytes', '512', '--rounds', '5', + '--bulk', '--bulk-concurrency-levels', '4,16'] if explicit: arguments += ['--server-receipt', str(server_receipt_path)] commands = [] @@ -171,9 +174,15 @@ def run_command(command, **kwargs): mock.patch.object(runner.subprocess, 'run', side_effect=run_command), \ mock.patch.object(runner.subprocess, 'check_output', side_effect=['f' * 40, b'']): runner.main() - self.assertEqual(len(commands), 4) + self.assertEqual(len(commands), 6) for command in commands: self.assertEqual('-server-receipt' in command, kind == 'local') + self.assertEqual(command[command.index('-database-cpus') + 1], '1.0') + self.assertEqual(command[command.index('-records') + 1], '2048') + self.assertEqual(command[command.index('-payload-bytes') + 1], '512') + self.assertEqual(command[command.index('-rounds') + 1], '5') + expected_levels = '4,16' if command[command.index('-mode') + 1] == 'bulk-saturation' else '4,16,64,256' + self.assertEqual(command[command.index('-concurrency-levels') + 1], expected_levels) if kind == 'local': self.assertEqual(command[command.index('-server-receipt') + 1], str(output.resolve() / 'server.receipt.json')) saved = json.loads((output / 'read.receipt.json').read_text()) diff --git a/scripts/run_performance_matrix.py b/scripts/run_performance_matrix.py index 2ee724a5..80d25112 100644 --- a/scripts/run_performance_matrix.py +++ b/scripts/run_performance_matrix.py @@ -21,11 +21,18 @@ def main(): parser.add_argument('--output', required=True, type=Path) parser.add_argument('--bulk', action='store_true', help='append read and write bulk32 comparisons') parser.add_argument('--batch-limit', type=int, default=0, help='0 keeps the server default') - parser.add_argument('--warmup-duration', help='default: Search 20s, native Mongo 2s per path/stage') - parser.add_argument('--duration', help='default measurement: Search 20s, native Mongo 5s per path/stage') + parser.add_argument('--database-cpus', type=float, default=1, help='enforced CPU quota for each owned Docker database') + parser.add_argument('--concurrency-levels', default='8,32', help='ascending total client concurrency') + parser.add_argument('--bulk-concurrency-levels', default='8,32', help='ascending bulk client concurrency, at most 64') + parser.add_argument('--client-processes', type=int, default=4, help='independent client OS processes for single-record cases') + parser.add_argument('--records', type=int, default=8192, help='working-set document count') + parser.add_argument('--payload-bytes', type=int, default=1024, help='document payload size') + parser.add_argument('--rounds', type=int, default=3, help='alternating paired rounds') + parser.add_argument('--warmup-duration', help='default: Search 20s, Mongo 10s per path/stage') + parser.add_argument('--duration', help='default measurement: 20s per path/stage') args = parser.parse_args() - warmup = args.warmup_duration or ('20s' if args.backend == 'search' else '2s') - duration = args.duration or ('20s' if args.backend == 'search' else '5s') + warmup = args.warmup_duration or ('20s' if args.backend == 'search' else '10s') + duration = args.duration or '20s' root = Path(__file__).resolve().parent.parent output = args.output.resolve() output.mkdir(parents=True, exist_ok=False) @@ -61,11 +68,12 @@ def main(): cases += [('bulk-read', 0, False, 'bulk-saturation'), ('bulk-write', 100, False, 'bulk-saturation')] for name, writes, lua, mode in cases: directory = output / name + levels = args.bulk_concurrency_levels if mode == 'bulk-saturation' else args.concurrency_levels command = [str(client), '-mode', mode, '-weir', str(weir), '-backend', args.backend, - '-client-processes', '4', '-concurrency-levels', '8,32', + '-client-processes', str(args.client_processes), '-concurrency-levels', levels, '-batch-sizes', '32' if mode == 'bulk-saturation' else '1', - '-records', '1024', '-payload-bytes', '1024', '-rounds', '3', - '-warmup-duration', warmup, '-duration', duration, '-database-cpus', '2', + '-records', str(args.records), '-payload-bytes', str(args.payload_bytes), '-rounds', str(args.rounds), + '-warmup-duration', warmup, '-duration', duration, '-database-cpus', str(args.database_cpus), '-write-percent', str(writes), '-backend-batch-limit', str(args.batch_limit), '-output', str(directory)] if args.mongod: @@ -74,9 +82,9 @@ def main(): command += ['-server-receipt', str(archived_receipt)] if lua: command.append('-lua-mutations') - concurrency = 'total 8/32; four OS processes each have 2/8 single-record workers' + concurrency = 'total ' + levels + '; ' + str(args.client_processes) + ' independent OS client processes' if mode == 'bulk-saturation': - concurrency = 'one client OS process with 8/32 workers; 32 records per call' + concurrency = 'one client OS process with ' + levels + ' workers; 32 records per call' receipt = dict(provenance, command=command, case=name, concurrency=concurrency) receipt_path = output / (name + '.receipt.json') receipt['server_binary_before_sha256'] = hashlib.sha256(weir.read_bytes()).hexdigest() From 0d7b89de44c4be669a6b24f9b31e7d0e2490fa2b Mon Sep 17 00:00:00 2001 From: Randy <857271370@qq.com> Date: Thu, 15 Oct 2026 15:09:15 +0800 Subject: [PATCH 3/3] =?UTF-8?q?=E2=9C=85=20test:=20use=20current=20Native?= =?UTF-8?q?=20targets=20and=20opt=20in=20to=20long=20benchmarks?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .github/workflows/saturation.yml | 3 --- README.md | 2 +- tests/integration/lifecycle_test.go | 5 ++--- tests/integration/semantics_test.go | 29 ++++++++++++++++++++++++++--- 4 files changed, 29 insertions(+), 10 deletions(-) diff --git a/.github/workflows/saturation.yml b/.github/workflows/saturation.yml index c76a81f3..00a19872 100644 --- a/.github/workflows/saturation.yml +++ b/.github/workflows/saturation.yml @@ -48,9 +48,6 @@ on: default: '1' required: true type: string - pull_request: - paths-ignore: - - 'results/**' permissions: contents: read diff --git a/README.md b/README.md index 61a791ac..398196f2 100644 --- a/README.md +++ b/README.md @@ -249,7 +249,7 @@ an explicit flag; credential-bearing URLs are rejected. CI runs offline unit/race/vet and dependency checks, real multi-node integration, and verified ordinary and Lua comparison smoke tests on both databases. -`saturation.yml` runs read/mixed/write matrices on PRs and accepts CPU quota, +`saturation.yml` runs only on explicit dispatch and accepts write percentage, CPU quota, concurrency, backend, workload mode, records per call and grouping controls on dispatch. Its default independent-client curve is 4/8/32/128/256, with three alternating rounds and 10s warmup plus 20s measurement per path. Supplemental diff --git a/tests/integration/lifecycle_test.go b/tests/integration/lifecycle_test.go index 2eff34da..9f56d39a 100644 --- a/tests/integration/lifecycle_test.go +++ b/tests/integration/lifecycle_test.go @@ -10,7 +10,6 @@ import ( "fmt" "net/http" "slices" - "strings" "testing" weir "github.com/batchstream/weir-go" @@ -213,9 +212,9 @@ func testNative(t *testing.T, ctx context.Context, client *weir.Client, backend t.Helper() request := &weir.NativeRequest{Resource: backend.collection} if backend.name == "mongo" { - parts := strings.Split(backend.collection, "/") filter := bson.D{{Key: "_id", Value: "primary"}} - command := bson.D{{Key: "count", Value: parts[len(parts)-1]}, {Key: "query", Value: filter}} + command := bson.D{{Key: "count", Value: backend.mongo.Name()}, {Key: "query", Value: filter}} + request.Resource = weir.EncodeSegment(backend.mongo.Database().Name()) request.Request = bsonDocument(t, command) } else { request = nativeHTTPGet(t, backend.collection, "/_doc/primary") diff --git a/tests/integration/semantics_test.go b/tests/integration/semantics_test.go index cf7a6ce3..2ac6bec3 100644 --- a/tests/integration/semantics_test.go +++ b/tests/integration/semantics_test.go @@ -228,6 +228,9 @@ func (s *system) testAdapterOwnedEnvelopes(t *testing.T, target *backendData) { t.Helper() payload := &weir.Document{ContentType: "application/vnd.example.operation", Data: []byte("opaque request")} native := &weir.NativeRequest{Resource: target.collection, Request: payload} + if target.name == "mongo" { + native.Resource = weir.EncodeSegment(target.mongo.Database().Name()) + } options := weir.NativeOptions{StoreName: target.name, Request: native, Consume: func(context.Context, *weir.NativeResponse, []byte) error { return nil }} result, err := s.client.Native(s.ctx, options) if err != nil || result == nil || result.Response != nil || result.Completion != weir.NativeNotStarted || result.Failure.GetCode() != weir.FailureUnsupported { @@ -308,14 +311,34 @@ func (s *system) testMissingTargets(t *testing.T, target *backendData) { nativeMissing := &weir.NativeRequest{Resource: missing.collection} if target.name == "mongo" { command := bson.D{{Key: "count", Value: target.mongo.Name() + ".never_created"}} + nativeMissing.Resource = weir.EncodeSegment(target.mongo.Database().Name()) nativeMissing.Request = bsonDocument(t, command) } else { nativeMissing = nativeHTTPGet(t, missing.collection, "/_doc/absent") } - nativeMissingOptions := weir.NativeOptions{StoreName: target.name, Request: nativeMissing, Consume: func(context.Context, *weir.NativeResponse, []byte) error { return nil }} + var nativeBody []byte + nativeMissingOptions := weir.NativeOptions{StoreName: target.name, Request: nativeMissing} + nativeMissingOptions.Consume = func(_ context.Context, _ *weir.NativeResponse, chunk []byte) error { + if len(nativeBody)+len(chunk) > 1<<20 { + return fmt.Errorf("Native response exceeded fixture bound") + } + nativeBody = append(nativeBody, chunk...) + return nil + } missingResult, err := s.client.Native(s.ctx, nativeMissingOptions) - if err != nil || missingResult == nil || missingResult.Response != nil || missingResult.Completion != weir.NativeNotStarted || missingResult.Failure.GetCode() != weir.FailureTargetNotFound { - t.Fatalf("missing Native target: %v %v", missingResult, err) + if err != nil || missingResult == nil || missingResult.Response == nil || missingResult.Completion != weir.NativeResponseComplete || missingResult.Failure != nil { + t.Fatalf("missing Native target must retain the backend response: %v %v", missingResult, err) + } + if target.name == "mongo" { + raw := bson.Raw(nativeBody) + if raw.Validate() != nil || raw.Lookup("n").AsInt64() != 0 || raw.Lookup("ok").AsFloat64() != 1 { + t.Fatalf("Native count on missing MongoDB collection: %s", raw) + } + } else { + response, err := weir.ParseHTTPNativeResponse(missingResult.Response) + if err != nil || response.StatusCode != http.StatusNotFound { + t.Fatalf("Native missing Search index HTTP response: %v %v", response, err) + } } if target.name == "search" { native := nativeHTTPGet(t, target.collection, "/_doc/absent")