Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
88 changes: 59 additions & 29 deletions .github/workflows/saturation.yml
Original file line number Diff line number Diff line change
Expand Up @@ -17,10 +17,9 @@ on:
options: ['1', '0.5', '2']
client-levels:
description: Total concurrency across four independent client processes
default: '8,32,128'
default: '4,8,32,128,256'
required: true
type: choice
options: ['8,32,128', '8,32,128,512']
type: string
backend-batch-limit:
description: Server adapter grouping limit (1 disables cross-RPC aggregation)
default: '32'
Expand All @@ -32,15 +31,29 @@ on:
default: false
required: true
type: boolean
pull_request:
paths-ignore:
- 'results/**'
backend:
description: Owned database backend
default: all
required: true
type: choice
options: [all, mongo, search]
mode:
description: Single-record business requests or supplemental client bulk calls
default: saturation
required: true
type: choice
options: [saturation, bulk-saturation]
batch-sizes:
description: Records per call; saturation requires 1, bulk-saturation accepts 1 through 64
default: '1'
required: true
type: string

permissions:
contents: read

concurrency:
group: saturation-${{ github.ref }}-${{ inputs.write-percent || 'matrix' }}-${{ inputs.database-cpus || '1' }}-${{ inputs.client-levels || '8,32,128' }}-${{ inputs.backend-batch-limit || '32' }}-${{ inputs.lua-mutations || false }}
group: saturation-${{ github.ref }}-${{ inputs.write-percent || 'matrix' }}-${{ inputs.database-cpus || '1' }}-${{ inputs.client-levels || '4,8,32,128,256' }}-${{ inputs.backend-batch-limit || '32' }}-${{ inputs.lua-mutations || false }}-${{ inputs.backend || 'all' }}-${{ inputs.mode || 'saturation' }}-${{ inputs.batch-sizes || '1' }}
cancel-in-progress: true

defaults:
Expand All @@ -61,7 +74,7 @@ jobs:
matrix:
write-percent: ${{ fromJSON(github.event_name == 'workflow_dispatch' && format('[{0}]', inputs.write-percent) || '[0,10,100]') }}
runs-on: ubuntu-24.04
timeout-minutes: 55
timeout-minutes: 120
steps:
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1
with:
Expand All @@ -81,34 +94,50 @@ jobs:
GOSUMDB: 'off'
WRITE_PERCENT: ${{ matrix.write-percent }}
DATABASE_CPUS: ${{ inputs.database-cpus || '1' }}
CLIENT_LEVELS: ${{ inputs.client-levels || '8,32,128' }}
CLIENT_LEVELS: ${{ inputs.client-levels || '4,8,32,128,256' }}
BACKEND_BATCH_LIMITS: ${{ inputs.backend-batch-limit || '32' }}
LUA_MUTATIONS: ${{ inputs.lua-mutations || false }}
BACKEND: ${{ inputs.backend || 'all' }}
MODE: ${{ inputs.mode || 'saturation' }}
BATCH_SIZES: ${{ inputs.batch-sizes || '1' }}
run: |
mkdir -p results/local
go build -o .tools/weir-lab ./cmd/weir-lab
IFS=',' read -r -a batch_limits <<< "$BACKEND_BATCH_LIMITS"
lua_flags=()
if [ "$LUA_MUTATIONS" = true ]; then
lua_flags=(-lua-mutations)
fi
for backend_limit in "${batch_limits[@]}"; do
export BACKEND_BATCH_LIMIT="$backend_limit"
export MEASUREMENT_OUTPUT="results/local/saturation-batch-$backend_limit"
export MEASUREMENT_RECEIPT="results/local/measurement-batch-$backend_limit.receipt.json"
python3 - <<'PY'
import hashlib, json, os, pathlib, subprocess
go build -trimpath -buildvcs=false -o .tools/weir-lab ./cmd/weir-lab
python3 - <<'PY'
import hashlib, json, os, pathlib, platform, subprocess
binary = pathlib.Path('.tools/weir-lab')
command = [str(binary), '-mode', 'saturation', '-weir', '.tools/weir', '-backend', 'all', '-database-cpus', os.environ['DATABASE_CPUS'], '-backend-batch-limit', os.environ['BACKEND_BATCH_LIMIT'], '-client-processes', '4', '-concurrency-levels', os.environ['CLIENT_LEVELS'], '-batch-sizes', '1', '-records', '2048', '-payload-bytes', '1024', '-write-percent', os.environ['WRITE_PERCENT'], '-rounds', '3', '-warmup-duration', '10s', '-duration', '20s', '-output', os.environ['MEASUREMENT_OUTPUT']]
if os.environ['LUA_MUTATIONS'] == 'true':
command.append('-lua-mutations')
receipt = {'client_operation_timeout': '10s', 'host_memory_bytes': int(next(line for line in pathlib.Path('/proc/meminfo').read_text().splitlines() if line.startswith('MemTotal:')).split()[1]) * 1024, 'source': subprocess.check_output(['git', 'rev-parse', 'HEAD'], text=True).strip(), 'source_dirty': bool(subprocess.check_output(['git', 'status', '--porcelain'])), 'client_binary_sha256': hashlib.sha256(binary.read_bytes()).hexdigest(), 'client_build': subprocess.check_output(['go', 'version', '-m', str(binary)], text=True), 'versions': json.loads(pathlib.Path('versions.json').read_text()), 'command': command}
if receipt['source_dirty']:
server = pathlib.Path('.tools/weir')
source = subprocess.check_output(['git', 'rev-parse', 'HEAD'], text=True).strip()
if subprocess.check_output(['git', 'status', '--porcelain']):
raise SystemExit('measurement requires a clean source checkout')
pathlib.Path(os.environ['MEASUREMENT_RECEIPT']).write_text(json.dumps(receipt, indent=2) + '\n')
server_receipt = json.loads(server.with_suffix('.receipt.json').read_text())
server_sha = hashlib.sha256(server.read_bytes()).hexdigest()
if server_receipt['sha256'] != server_sha:
raise SystemExit('server binary does not match its preparation receipt')
for backend_limit in os.environ['BACKEND_BATCH_LIMITS'].split(','):
if not backend_limit.isdecimal() or not 1 <= int(backend_limit) <= 1024:
raise SystemExit('backend batch limits must be integers from 1 through 1024')
output = 'results/local/saturation-batch-' + backend_limit
command = [str(binary), '-mode', os.environ['MODE'], '-weir', str(server), '-backend', os.environ['BACKEND'], '-database-cpus', os.environ['DATABASE_CPUS'], '-backend-batch-limit', backend_limit, '-client-processes', '4', '-concurrency-levels', os.environ['CLIENT_LEVELS'], '-batch-sizes', os.environ['BATCH_SIZES'], '-records', '8192', '-payload-bytes', '1024', '-write-percent', os.environ['WRITE_PERCENT'], '-rounds', '3', '-warmup-duration', '10s', '-duration', '20s', '-output', output]
if os.environ['LUA_MUTATIONS'] == 'true':
command.append('-lua-mutations')
receipt = {'client_operation_timeout': '10s', 'host_memory_bytes': int(next(line for line in pathlib.Path('/proc/meminfo').read_text().splitlines() if line.startswith('MemTotal:')).split()[1]) * 1024, 'host_cpus': os.cpu_count(), 'host_cpu_model': next(line.split(':', 1)[1].strip() for line in pathlib.Path('/proc/cpuinfo').read_text().splitlines() if line.startswith('model name')), 'go': subprocess.check_output(['go', 'version'], text=True).strip(), 'host': platform.uname()._asdict(), 'source': source, 'source_dirty': False, 'client_binary_sha256': hashlib.sha256(binary.read_bytes()).hexdigest(), 'client_build': subprocess.check_output(['go', 'version', '-m', str(binary)], text=True), 'server_binary_sha256': server_sha, 'server_receipt': server_receipt, 'versions': json.loads(pathlib.Path('versions.json').read_text()), 'command': command, 'workflow_run': 'https://github.com/' + os.environ['GITHUB_REPOSITORY'] + '/actions/runs/' + os.environ['GITHUB_RUN_ID']}
receipt_path = pathlib.Path('results/local/measurement-batch-' + backend_limit + '.receipt.json')
receipt_path.write_text(json.dumps(receipt, indent=2) + '\n')
with pathlib.Path(output + '.log').open('x') as log:
process = subprocess.Popen(command, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, text=True)
for line in process.stdout:
print(line, end='', flush=True)
log.write(line)
receipt['exit_code'] = process.wait()
receipt['server_binary_after_sha256'] = hashlib.sha256(server.read_bytes()).hexdigest()
receipt['client_binary_after_sha256'] = hashlib.sha256(binary.read_bytes()).hexdigest()
receipt_path.write_text(json.dumps(receipt, indent=2) + '\n')
if receipt['server_binary_after_sha256'] != server_sha or receipt['client_binary_after_sha256'] != receipt['client_binary_sha256']:
raise SystemExit('benchmark binary changed during measurement')
if receipt['exit_code']:
raise SystemExit(receipt['exit_code'])
PY
.tools/weir-lab -mode saturation -weir .tools/weir -backend all -database-cpus "$DATABASE_CPUS" -backend-batch-limit "$BACKEND_BATCH_LIMIT" -client-processes 4 -concurrency-levels "$CLIENT_LEVELS" -batch-sizes 1 -records 2048 -payload-bytes 1024 -write-percent "$WRITE_PERCENT" -rounds 3 -warmup-duration 10s -duration 20s -output "$MEASUREMENT_OUTPUT" "${lua_flags[@]}" 2>&1 | tee "results/local/saturation-batch-$backend_limit.log"
done
git diff --exit-code
- name: Preserve raw resource samples, workload validation and cleanup
if: always()
Expand All @@ -117,6 +146,7 @@ jobs:
name: saturation-evidence-writes-${{ matrix.write-percent }}-${{ github.sha }}
path: |
results/local/
.tools/weir.receipt.json
/tmp/weir-tests-*/manifest.json
/tmp/weir-tests-*/cleanup.json
/tmp/weir-tests-*/*config.json
Expand Down
2 changes: 1 addition & 1 deletion Makefile
Original file line number Diff line number Diff line change
Expand Up @@ -13,7 +13,7 @@ integration:
WEIR_TEST_BINARY="$(CURDIR)/.tools/weir" GOWORK=off go test -race -tags=integration -count=1 -timeout=10m -v ./tests/integration

benchmark:
GOWORK=off go run ./cmd/weir-lab -mode saturation -weir "$(CURDIR)/.tools/weir" -backend all -database-cpus 1 -client-processes 4 -concurrency-levels 8,32,128 -batch-sizes 1 -records 2048 -write-percent 0 -rounds 3 -warmup-duration 10s -duration 20s -output results/local/benchmark
GOWORK=off go run ./cmd/weir-lab -mode saturation -weir "$(CURDIR)/.tools/weir" -backend all -database-cpus 1 -client-processes 4 -concurrency-levels 4,8,32,128,256 -batch-sizes 1 -records 8192 -write-percent 0 -rounds 3 -warmup-duration 10s -duration 20s -output results/local/benchmark

diagnostic:
GOWORK=off go run ./cmd/weir-lab -mode fixed -weir "$(CURDIR)/.tools/weir" -output results/local/diagnostic
40 changes: 29 additions & 11 deletions README.md
Original file line number Diff line number Diff line change
Expand Up @@ -87,8 +87,8 @@ make benchmark
go run ./cmd/weir-lab -mode saturation \
-weir .tools/weir -backend all -database-cpus 1 \
-client-processes 4 \
-concurrency-levels 8,32,128 -batch-sizes 1 \
-records 2048 -payload-bytes 1024 -write-percent 0 \
-concurrency-levels 4,8,32,128,256 -batch-sizes 1 \
-records 8192 -payload-bytes 1024 -write-percent 0 \
-rounds 3 -warmup-duration 10s -duration 20s \
-output results/local/read
```
Expand All @@ -103,13 +103,20 @@ AB/BA order alternates across rounds. Failed calls or uncertain acknowledgements
invalidate the comparison.

For Lua read-modify-write, add `-lua-mutations -write-percent 100`. Each native
MongoDB request uses a snapshot transaction with one read, revision computation,
replacement and majority commit. Each native Search request uses one real-time
read and a write conditioned on its observed sequence number and primary term.
MongoDB request uses one atomic `UpdateOne` aggregation pipeline, with
`$set: {revision: {$subtract: [int32(1), "$revision"]}}` and majority acknowledgement.
Each native Search request uses one `POST _update` Painless script with
`ctx._source.revision = 1 - ctx._source.revision` and `retry_on_conflict=0`.
Neither baseline performs a client read or transaction. These are native controls
for this fixed transformation, rather than a claim about arbitrary Lua programs.
The SDK sends one Lua `AtomicTransform` per request with the same computation.
The function updates `current.revision = 1 - current.revision` and returns the
current document; existing BSON int32 revision fields retain their width.
current document; independent postflight checks require the BSON revision to
retain its int32 type and verify every field and final revision on both paths.
Fixture documents are identical on both paths; Weir adds no document metadata.
The logical request/result document-byte counters exclude scripts and commands,
so they are zero for transform-only calls. Backend reads and replacements still
contribute to the measured network and CPU costs.

Repeat with `-backend-batch-limit 1` to disable server aggregation. Keep other
settings identical and use a fresh output directory. The dispatch workflow can
Expand All @@ -120,7 +127,9 @@ Reports retain per-round throughput, pooled latency histograms, client process
identities, start lag, CPU/RSS, database CPU/memory/I/O/network samples and raw
Weir metrics. Metrics report stream completions, terminal records, queue wait,
adapter time, rejections and operations per adapter invocation. MongoDB also
records physical command deltas. Adapter invocations can split by namespace,
records selected business-command deltas at the timed boundaries for both
single-record and bulk calls. These counters exclude handshakes and monitoring
commands. Adapter invocations can split by namespace,
action and byte limits; their counts are distinct from database wire commands.
Missing samples and reset counters are reported as unavailable.
`business_operation_timeouts` counts failed logical operations whose request context
Expand Down Expand Up @@ -169,12 +178,17 @@ prove Linux database saturation or a production capacity limit.

`scripts/run_performance_matrix.py` serializes read, mixed (10%), write and Lua
cases at total concurrency 8/32, with three alternating native/Weir rounds,
2s warmup and 5s measurement for native MongoDB, or 20s warmup and 20s
10s warmup and 20s measurement for MongoDB, or 20s warmup and 20s
measurement for Elasticsearch per stage. Longer Search stages gave better
resource sampling coverage and avoided a short-run regression that did not
reproduce in the adjacent longer comparison. These are measurement defaults;
the server batching and transport defaults remain 32 records.
`--warmup-duration` and `--duration` allow explicit overrides. It retains exact commands, client source
`--warmup-duration`, `--duration`, `--database-cpus`, `--concurrency-levels`,
`--client-processes`, `--records`, `--payload-bytes` and `--rounds` expose the
experiment controls. Docker databases default to an enforced 1 CPU quota and an
8192-document hot working set.
`--bulk-concurrency-levels` separately controls bulk comparisons, whose maximum
concurrency is 64. It retains exact commands, client source
snapshots, binary hashes, JSON/Markdown reports and copied fixture cleanup/log
evidence. The runner uses an explicit `--server-receipt` or automatically loads
the adjacent `.receipt.json`, passing frozen local receipts to the lab while
Expand Down Expand Up @@ -235,8 +249,12 @@ an explicit flag; credential-bearing URLs are rejected.

CI runs offline unit/race/vet and dependency checks, real multi-node integration,
and verified ordinary and Lua comparison smoke tests on both databases.
`saturation.yml` runs read/mixed/write matrices on PRs and accepts CPU quota,
concurrency and grouping controls on dispatch. Workflow artifacts preserve raw
`saturation.yml` runs only on explicit dispatch and accepts write percentage, CPU quota,
concurrency, backend, workload mode, records per call and grouping controls on
dispatch. Its default independent-client curve is 4/8/32/128/256, with three
alternating rounds and 10s warmup plus 20s measurement per path. Supplemental
`bulk-saturation` runs require explicit concurrency at most 64 and a batch size
such as 32. Workflow artifacts preserve raw
reports and owned-resource cleanup receipts.

Generated measurements belong in ignored output directories under `results/`.
Expand Down
2 changes: 1 addition & 1 deletion cmd/weir-lab/main.go
Original file line number Diff line number Diff line change
Expand Up @@ -76,7 +76,7 @@ func main() {
flag.IntVar(&opts.concurrency, "concurrency", 8, "client workers for finite diagnostics")
flag.IntVar(&opts.rounds, "rounds", 3, "alternating paired rounds")
flag.IntVar(&opts.writePercent, "write-percent", 10, "write fraction from 0 through 100")
flag.BoolVar(&opts.luaMutations, "lua-mutations", false, "single-record read-modify-write: native transaction/OCC versus Weir Lua; no client batching")
flag.BoolVar(&opts.luaMutations, "lua-mutations", false, "single-record mutation: native atomic update pipeline/server script versus Weir Lua; no client batching")
flag.StringVar(&opts.mode, "mode", "saturation", "saturation sends independent single requests from OS client processes; bulk-saturation measures bulk overhead; fixed is a finite diagnostic")
flag.StringVar(&opts.levels, "concurrency-levels", "8,32,128", "ascending total client concurrency, at most 512 in single-request mode")
flag.StringVar(&opts.batches, "batch-sizes", "1", "records per call: saturation requires 1; bulk-saturation accepts 1 through 64")
Expand Down
4 changes: 2 additions & 2 deletions go.mod
Original file line number Diff line number Diff line change
Expand Up @@ -3,8 +3,8 @@ module github.com/batchstream/weir-tests
go 1.27.1

require (
github.com/batchstream/weir-go v0.10.0
github.com/batchstream/weir-protocol v0.8.0
github.com/batchstream/weir-go v0.15.0
github.com/batchstream/weir-protocol v0.13.0
go.mongodb.org/mongo-driver/v2 v2.9.1
google.golang.org/grpc v1.83.2
)
Expand Down
12 changes: 4 additions & 8 deletions go.sum
Original file line number Diff line number Diff line change
@@ -1,11 +1,7 @@
github.com/batchstream/weir-go v0.9.0 h1:a4pGdTJd9/5Y8QMTz37ipHpfSXztpPB84Wm4Wl0fc7A=
github.com/batchstream/weir-go v0.9.0/go.mod h1:lAlK3m/CBl+WZUhMIKwhGdqoeb4wnoteKWppyCfgZLo=
github.com/batchstream/weir-go v0.10.0 h1:D4sv8YoXqUP44W+jd/DsbsW6NkZqK5FTbNFWqhDaABA=
github.com/batchstream/weir-go v0.10.0/go.mod h1:ud7wNpCOrnNuIfwu7CPxkveku9qNi9NrBxuNv44NBfk=
github.com/batchstream/weir-protocol v0.7.0 h1:+LSSLPCWv8GL8RgkH06LYNT8wDN5tFRAs0rle1Ym4sY=
github.com/batchstream/weir-protocol v0.7.0/go.mod h1:hHmwPoE9yHYAz7Auc3a14Ijo/uxAef4sQ9nBtyXNcGc=
github.com/batchstream/weir-protocol v0.8.0 h1:A+diMm2VPP0YoAsnVQJTW71rf1vV+IVMyUQ02zta2Oo=
github.com/batchstream/weir-protocol v0.8.0/go.mod h1:hHmwPoE9yHYAz7Auc3a14Ijo/uxAef4sQ9nBtyXNcGc=
github.com/batchstream/weir-go v0.15.0 h1:Y63iiiiimksBp6lslITghcnQxmqp51ytb4jT8fASm6o=
github.com/batchstream/weir-go v0.15.0/go.mod h1:wwaM3ZwPIv2LN9W+BjTUoKNRlGVkwj04tHxg1xaIe5k=
github.com/batchstream/weir-protocol v0.13.0 h1:kXxZOq30a8rgYFrxpV0pdXalNH6ttvM6qgCkrxsxi9U=
github.com/batchstream/weir-protocol v0.13.0/go.mod h1:hHmwPoE9yHYAz7Auc3a14Ijo/uxAef4sQ9nBtyXNcGc=
github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs=
github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs=
github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
Expand Down
17 changes: 17 additions & 0 deletions internal/benchmark/command_counters.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,17 @@
package benchmark

func (result *SaturationResult) calculatePhysicalCommands() {
if result.DatabaseBefore == nil || result.DatabaseAfter == nil {
return
}
result.PhysicalCommandUnavailable = mergeCommandUnavailable(result.PhysicalCommandUnavailable, result.DatabaseBefore.CommandUnavailable, result.DatabaseAfter.CommandUnavailable)
result.PhysicalCommands = make(map[string]uint64)
for name, after := range result.DatabaseAfter.Commands {
before, found := result.DatabaseBefore.Commands[name]
if !found || after < before {
result.PhysicalCommandUnavailable = mergeCommandUnavailable(result.PhysicalCommandUnavailable, "unavailable or reset "+name)
continue
}
result.PhysicalCommands[name] = after - before
}
}
Loading
Loading