From c0b9b72e5769ccacd53ef3502c1a2c8066c27a8b Mon Sep 17 00:00:00 2001 From: Taewoon Kim Date: Tue, 10 Nov 2026 03:52:42 +0900 Subject: [PATCH] benchmarks: tooling for the post1 rerun of the paper's embedded cells (build-only mode, wheel label, one wheel per cell, embedded-only stages) (#583) The post1 rows override the 26.11.1 rows of the cells that are rerun, all repetitions of a cell from one wheel (DECISIONS #297, #298). The harness had no way to stop a cell after build(), rows did not say which wheel file they ran on, the generated stages mix embedded and served arms, and nothing refused a cell with repetitions from two wheels. All of it is additive and off by default, because comparator stages pull main at every start and a mid-run change splits rows: no variable set means no behaviour change. - post1_rows.py (new): build-only helpers for the lanes, the opt-in wheel stamp, resolve() (one wheel per cell, problems instead of a silent mix), strip (rollback). - runner.py: --build-only (refused unless the results file ends _post.jsonl), BENCH_BUILD_ONLY forwarded when set, stamp_row() before the row is written. - l1_tpc.py, l2_graph.py, e2_hybrid.py, l3d_dense.py: one guarded hook after build() and before any timed work. - land_stage.py: --post1-reps N (default 0) pulls the post1 rows file and lands the resolved rows; build-only rows are refused as cells. - engine_pin.py: an X.Y.Z.postN pin accepts the X.Y.Z engine jar (the check refused every post1 cell). - make_post1_rerun_stages.py (new): embedded-only stages on top of make_2611_stages.stage_script, cell list, dry run with an estimate, selftest, driver. - POST1-RERUN.md, CAMPAIGN.md paragraph, test_post1_rerun.py. Co-Authored-By: Claude Sonnet 5.5 --- benchmarks/experiments/CAMPAIGN.md | 2 + benchmarks/experiments/POST1-RERUN.md | 49 ++ benchmarks/experiments/e2_hybrid.py | 5 + benchmarks/experiments/engine_pin.py | 10 +- benchmarks/experiments/l1_tpc.py | 5 + benchmarks/experiments/l2_graph.py | 10 + benchmarks/experiments/l3d_dense.py | 5 + benchmarks/experiments/land_stage.py | 19 + .../experiments/make_post1_rerun_stages.py | 526 ++++++++++++++++++ benchmarks/experiments/post1_rows.py | 403 ++++++++++++++ benchmarks/experiments/runner.py | 17 +- benchmarks/experiments/test_post1_rerun.py | 39 ++ 12 files changed, 1087 insertions(+), 3 deletions(-) create mode 100644 benchmarks/experiments/POST1-RERUN.md create mode 100644 benchmarks/experiments/make_post1_rerun_stages.py create mode 100644 benchmarks/experiments/post1_rows.py create mode 100644 benchmarks/experiments/test_post1_rerun.py diff --git a/benchmarks/experiments/CAMPAIGN.md b/benchmarks/experiments/CAMPAIGN.md index 155bab603d1..53d8d5918e6 100644 --- a/benchmarks/experiments/CAMPAIGN.md +++ b/benchmarks/experiments/CAMPAIGN.md @@ -242,6 +242,8 @@ The detail behind item 8: **ONE LINE OF DEVELOPMENT.** The instrument is developed on `main`, never on a long-lived branch (BUGS F65): mini pulls `main` at every stage start, so the instrument is landed on `main` before the first cell and never during the run, and a change that must not reach a running campaign waits for the next one. Anything else lands on `main`, behind a flag if it must, or waits. A later instrument is added the same way: `BENCH_INSTRUMENT` (`bench_common.INSTRUMENT`, default `2026-11`) names it, and code for it asks `instrument_at_least(...)` (`test_instrument_selector.py`). **The next campaign's stages are generated by `make_2611_stages.py`.** One template carries every guard the campaign learned (cap table, shadowed definitions, budgets against the cap, whole-cell cost, images present and current, comparator digests frozen at generation, the JPype pin read out of the image, the pinned wheel by sha256, the cap rule, the ALL-DONE trap); the instrument, `REPS` and the banner are its parameters. The generator selects the multi-model subset, narrowed by a profile (next paragraph). The subset is `ENGINES` in the file: ArcadeDB (embedded and served), SurrealDB (embedded and served), ArangoDB, Memgraph, PostgreSQL with pgvector, AGE and TimescaleDB (and plain PostgreSQL where a lane uses it), MongoDB with MongoDB Search, DuckDB with VSS and DuckPGQ, SQLite with sqlite-vec, the composed Qdrant plus Neo4j stack, Neo4j and Qdrant on their own (graph and dense; sparse on the second list), and LadybugDB on every lane it supports. Every registered arm is either in the subset or in `OUTSIDE`, so a newly registered arm forces a decision (`--check`). The core lanes are graph (interactive and analytics), cross-model, documents and dense; under `--profile full`, `--second-list` adds lifecycle, then sparse and time series, after them. TimescaleDB runs whenever the time-series lane does (so not under the default profile); `--without-timescaledb` leaves it out. +**The wheels-only post1 re-measure has its own tooling (POST1-RERUN.md; the plan is row 93, issue #581).** `make_post1_rerun_stages.py` writes embedded-only stages on top of this generator, with `runner.py --build-only` cells where only the load figures can change, the post1 wheel pinned by name and sha256, and rows in `runs_page__2611_post1.jsonl`; `land_stage.py --post1-reps N` lands, per cell, the rows of one wheel and refuses a cell that would mix wheels. None of it is active for a stage of the live pass: every new flag and variable is off by default. + **THE PAPER PROFILE (the owner, 2026-10-25: the paper's experiments must finish by 2026-11-10; amended 2026-11-03, DECISIONS #232 and #235: the paper's numbers land about 2026-11-15, and the submission is 2026-11-24 for the SIGMOD 2027 industrial track or 2026-12-01 for the ICDE 2027 industry track, the fallback).** `--profile paper-2611` is the default and narrows the subset in the open (`PROFILES`, `DROP_WHY` in the file). (1) Core lanes only: graph, cross-model, documents and dense. The second list (lifecycle, sparse, time series) is out of scope for the paper, for the comparators and for the ArcadeDB chain, and `--second-list` is refused unless `--profile full` is given. (2) Neo4j and Memgraph stay in the graph lane (and cross-model where they are); their dense-lane cells (`neo4j_dense`, `neo4j_dense_int8`, `memgraph_dense`, `memgraph_dense_int8`) are out. The composed embedded stack (SQLite + LanceDB + LadybugDB in one process, `composed_embedded_sqlite_lance_ladybug`, the owner, 2026-10-25) is a Tier 1 cross-model arm: engine `composed-embedded`, right after the served composed stack, six lane cells per pass (hybrid in both durability classes and atomicity, at e2 and e2_500k), on the lane `e2` only; it runs in the dense image, so rebuild `dbbench:dense` (`./build_images.sh dense`) before the first pass. (3) The dense lane is fp32 only, for every engine: no `_int8` and no `_pq` arm, the comparators' and ArcadeDB's alike (`arangodb_dense_int8`, `arcadedb_dense_embedded_int8`, `arcadedb_dense_server_int8`, `mongodb_dense_int8`, `qdrant_dense_int8`, `sqlite_vec_dense_int8`, and the PQ arms when `BENCH_DENSE_PQ=1`). Quantized arms are not like-for-like across engines, and quantization is out of the paper's scope: fp32 for every engine, ArcadeDB's own INT8 and BINARY out by the owner's rule, jvector PQ, BQ and NVQ studied after the paper. ArcadeDB keeps its fp32 dense arms, embedded and served. (4) The target is N=3 (`TARGET_REPS`; `--project` prices 3 repetitions by default). The chain keeps the passes 4 and 5 after pass 3, in #503's order, so they run only if time is left; stopping after the first six groups is a complete result. Since 2026-11-03 the order on the bench host is: pass 3 (qC48), then the re-runs (qC14 at turbo off, qC15..qC23 at the 10 us idle hold, DECISIONS #234), then the four-arm ArcadeDB pass on the released 26.11.1 wheels, then passes 4 and 5 (qC49..qC80, which wait for `PASS45.go`). Amended 2026-11-05: if the 26.11.1 wheels are on PyPI and the four-arm runner is deployed when `REORDER reruns finished` is logged in the host's `STATUS.txt`, the four-arm pass (n=3 for the four ArcadeDB arms) starts first, and the window re-runs of qC14..qC17 follow it, still before `PASS45.go`; otherwise the order above stands. So the paper's comparators are N=3 for sure and N=5 where passes 4 and 5 finish before the tables freeze (DECISIONS #232, amendment of 2026-11-03). `--check` prints every dropped cell with its reason and count (lane cells per pass: 148 second-list, 8 Neo4j and Memgraph dense, 12 quantized dense) and proves that none is scheduled, in both chains whatever the arguments select (`profile_problems`); `--check --drafts DIR` lints written scripts for the same. `--profile full` is the whole subset as it was before the profile. Every generated stage exports `BENCH_INSTRUMENT=2026-11` before its first Python call, reads the selection back from `bench_common` and aborts if the tree does not select it, writes `runs_page__2611.jsonl` (and overlay directories with the same suffix) so one rows file never holds two instruments, and fixes `REPS` at generation (`--reps`, default 5, at most 5; the target is 3). The generator re-executes itself with the variable set and refuses a different value in the environment. `land_stage.py` lands exactly these files (DECISIONS #196; the 2026-10-25 change that closed the gap between the pull and the freeze): it pulls `runs_page__2611.jsonl` and the overlay directories with the same suffix, and `screen_pulled_rows` refuses the file before anything is merged unless every row is stamped `2026-11` and names the pin's engine commit, so a retired 2026-10 file or a file that mixes instruments never reaches `results/runs.jsonl`. The first landing meets no store (the October one is retired) and `merge_campaign.py` creates it. `export_web` no longer demands the `e4decomp_` artifacts of a landing scoped away from e4, because the chain has no e4 stage and `--only-lanes l2` would otherwise wait for one. A rehearsal restores what it wrote, the site payload (removed when the run created it) and the PAGE-SPEC.md block included. Failed cells (`rc != 0` or an error) merge as failures and are not frozen; censored queries keep their flag and publish no number. No stage script calls `land_stage.py` (the generated `qC` scripts only name it in a comment about the rows file); a landing is run by hand on the laptop. The comparators are the default mode and run on the release pair already installed (`ARCADEDB_WHEEL`, `ARCADEDB_SERVER_IMAGE`, `ARCADEDB_ENGINE_COMMIT`): `python3 make_2611_stages.py --after --status-file STATUS.txt --id-prefix qS --out DIR`. `--after` is required, with no default (a marker name, or `none`), and the first stage waits for `grep -qx " ALL-DONE"`, each later one for the stage before it. The marker must be the LAST item still to run, one that writes its line whether it runs or is skipped: a marker already in the machine's STATUS.txt makes the first stage start at once, beside the running jobs (2026-10-23: `qMA06 ALL-DONE`, written by a skipped study). So `--status-file` is required with a marker (a copy of that STATUS.txt, or `none` when there is no queue) and the generator refuses a marker the file already holds; `--check --drafts DIR --status-file STATUS.txt` makes the same test on drafts that exist. The subset is emitted in ENGINE TIERS in the user's exact pass order (2026-10-24; `--tiers`, on by default, `--no-tiers` for the plain rep-major chain). N defaults to 5, the target is 3 (the paper profile), and the order is `schedule_order`: n=1 T1, n=2 T1, n=3 T1, n=1 T2, n=2 T2, n=3 T2, n=4 T1, n=4 T2, n=5 T1, n=5 T2. Every prefix of it is a complete set of equal replicates for the tiers covered so far (each tier has run passes 1..k, none skipped, and a pass is whole before the next starts); `--reps 3` stops after the first six groups, which is the chain of Tier 1 passes 1-3 and then Tier 2 passes 1-3 as before. Tier 1 is ArcadeDB (the four 26.11.1 arms, when `--arcadedb-release` adds them), SurrealDB, ArangoDB, PostgreSQL with pgvector and AGE, MongoDB with Search, the composed Qdrant plus Neo4j stack, and the embedded references SQLite, LadybugDB and DuckDB; Tier 2 is Memgraph, Neo4j and Qdrant on their own, Redis 8, CrateDB and TimescaleDB (`TIER_ENGINES`, with `ARM_TIER` for TimescaleDB, an arm of the `postgresql` engine that the user lists in Tier 2). Every stage holds the arms of one tier, and stage ids stay one run of consecutive numbers (a stage that mixed both tiers is split in two, so the stage count grows, and each stage repeats its guards and image checks; past 99 stages the ids take three digits, so `qS100` cannot sort before `qS11`). `--check` prints the schedule and proves the order and the prefixes (the plan collapsed into (tier, pass) groups equals `schedule_order`, each group holds the whole pass), and `--check --drafts DIR` lints a draft folder for the same. An arm with no tier runs after Tier 2 in each block. The ArcadeDB arms are Tier 1 but not in this chain: DBBench keeps one ArcadeDB version per host, so they run only once the benchmark pin is 26.11.1 (next paragraph). Inside a tier the order is the one below: a pass is pass k of every stage of the tier (core lanes, then the second list), and each stage runs ONE repetition (`--only-reps $PASS`). With `--no-tiers` the chain is pass 1 of every stage, then pass 2, up to pass N. If time runs out, every engine of a tier has the same n; repetitions days apart show drift. A stage whose environment declares fewer reps (`REPS=k`) is left out of the later passes. Inside a lane the engines run in `ENGINE_PRIORITY` (SurrealDB, ArangoDB, PostgreSQL, MongoDB, Redis 8, composed Qdrant+Neo4j, Neo4j, Qdrant, DuckDB, SQLite, Memgraph, LadybugDB, CrateDB; ArcadeDB's arms lead it when they join; Redis and CrateDB are skipped until their adapters are in `ENGINES`). The rep-1-first rule and the cap rule hold across passes: a pass above 1 runs a cell only when rep 1 of it was clean, and only a timeout of rep 1 censors the larger sizes. WHICH REPS EXIST is `~/rep-state-2026-11-.tsv` on the host (one `label, rep, ok|fail` line per finished repetition; the rows themselves carry the rep number in the runner's results file): stop after any pass, and the next pass, or a restarted stage, picks up from it with no regeneration; N can go to 3 or stop at 2. Delete that file to start a chain over. The ArcadeDB pass is `--arcadedb-release X.Y.Z` with `--engine-commit`, `--image-digest sha256:...`, `--wheel-name`, `--wheel-sha256` and `--wheel-url`; it refuses unless all five are given, because the release may not exist when the drafts are made. It writes `00`, the pair install (official image by digest, wheel by sha256, `build_matched_pair.sh`, `verify_pair_c25.sh`), and ArcadeDB's lane stages behind it. THE ARCADEDB GATE: the ArcadeDB chain is its own chain (Tier 1 only, passes 1..N in order), generated with `--after` the last comparator stage, so the comparators run the schedule first. `00` waits after its `--after` for the marker file `$HOME/conf/go-arcadedb-2611` on the host (it never creates it; it sits before the ALL-DONE trap, so a killed waiter hands nothing on), and every ArcadeDB lane stage aborts without it. The user or the agent touches that file when `CURRENT_PIN` moves to 26.11.1, and not before; `--check --drafts` lints that the gate is in the pair install and in every ArcadeDB stage and in no comparator stage. diff --git a/benchmarks/experiments/POST1-RERUN.md b/benchmarks/experiments/POST1-RERUN.md new file mode 100644 index 00000000000..1dadd53e475 --- /dev/null +++ b/benchmarks/experiments/POST1-RERUN.md @@ -0,0 +1,49 @@ +# Re-measuring paper cells on a wheels-only post1 + +The procedure for rerunning the paper's ArcadeDB embedded cells on `26.11.1.post1`, the wheels-only release whose engine is byte-identical to 26.11.1 (DECISIONS #297). The plan and the open question of which cells qualify are CAMPAIGN section 7 row 93 (issue #581). This file is the how. Nothing in it runs by itself, and nothing in it touches the live pass: the rerun writes its own rows file and its own rep state, from its own worktrees. + +## What was built + +| piece | file | what it does | +|---|---|---| +| build-only mode | `runner.py --build-only`, a hook in `l1_tpc.py`, `l2_graph.py`, `e2_hybrid.py`, `l3d_dense.py`, `post1_rows.py` | A cell stops right after `build()` (load and index, before any timed work) and writes a row with `row_kind = "build_only"`. The runner refuses the flag unless the results file is named `runs_page__2611_post.jsonl`, so such a row cannot land in the pass's own rows file. | +| wheel label | `post1_rows.stamp_row`, called by the runner | A stage that exports `BENCH_WHEEL_LABEL`, `BENCH_WHEEL_NAME` and `BENCH_WHEEL_SHA256` gets `wheel_label`, `wheel_name` and `wheel_sha256` on every row. A row without them is labelled by its `engine_version`. The build is `engine_backend` (`native` or `jvm`), as before. | +| the choice of one wheel per cell | `post1_rows.resolve`, used by `land_stage.py --post1-reps N` | Per in-process ArcadeDB cell, the rows of ONE wheel: the newest wheel that has rows for the cell, only when it is complete at N repetitions. A cell that would mix wheels, or whose newer wheel is short, is a problem and the landing exits nonzero before anything is merged. | +| stage generator | `make_post1_rerun_stages.py` | Writes the wheel stage and the lane stages of the rerun, embedded arms only, on top of `make_2611_stages.stage_script` (so the TEMP GATE, the guards, the cap rule and the rep gate are the live chain's own), plus `ORDER.txt` and the sequential driver `run_post1.sh`. | +| tests | `test_post1_rerun.py` | Pure Python, no docker, no timing. `uv run --no-project python test_post1_rerun.py`. | + +## What a build-only row may replace + +Only these fields of the same cell and repetition, and only when the glue change does not touch a query path (DECISIONS #299: the ingest changes #318, #324 and #325; not the read-path changes #321 and #323, which need a `full` cell): + +`build_s`, `ingest_s`, `index_s`, `index_before_load`, `setup_s`, `gav_build_s`, and the load-transport facts `lineitem_buckets`, `async_writers`, `async_sync`, `load_call_rows`, `columnar_insert`. + +Every timed workload figure of the cell stays at its 26.11.1 value. The memory, disk and cpu figures of the build window are carried beside the full row's own as `build_window_*` and never over them, because the full row's figures cover the whole container life. The merged row says where each half came from: `timed_wheel`, `timed_ts_utc`, `build_figures_wheel`, `build_figures_wheel_sha256`, `build_figures_run_id`, `build_figures_ts_utc`, `build_figures_replaced`. Its `ts_utc` is the newer of the two, because the freeze keeps the newest row per cell key. + +A cell list entry has one of three modes: `build-only`, `full` (a whole cell, with its overlay) or `skip`. The default list is every embedded cell of the paper profile (22 jobs per pass per chain): lane cells `build-only`, the two multipass overlays `skip` (an overlay times queries, there is nothing to stop after). + +## The order of the real rerun + +1. **Preconditions.** The live n=3 pass and the PG-wire catch-up have ended; `bench/arcadedb-import-26111` (the four-arm generator, PR #554) is merged, because the native chain needs it (`make_post1_rerun_stages.py` refuses a native chain without it); the post1 wheels are released (tag and PyPI) and the byte-identity guard of DECISIONS #297 passed; the user said go. Nobody syncs the mini during the live pass: a dirty or moved tree aborts its queues. +2. **Pin check.** The engine commit is the pass's (`ENGINE_COMMIT` of the pins file), the wheel label is `26.11.1.post1`, and each wheel's file name carries that version. The stage reads the version out of the built image and refuses on a difference, so a wheel whose metadata says another version cannot run (the candidate JVM wheel of the glue A/B is one: its metadata says 26.11.1). `engine_pin.jar_pin` makes the engine-jar check accept the `X.Y.Z` jar under an `X.Y.Z.postN` pin. +3. **Cell list.** `python3 make_post1_rerun_stages.py --write-cells post1_cells.tsv`, then edit the mode column from the measured A/B (never by judgment, DECISIONS #298). +4. **Dry run, laptop.** `python3 make_post1_rerun_stages.py --dry-run --cells post1_cells.tsv --hours HOURS.tsv --reps 3` prints the stage list and an ESTIMATE per chain. Generate the scripts with the real pins, `bash -n` is part of the selftest. Run ONE build-only cell on the laptop with a synced wheel into a scratch results file (`runner.py --lanes l1tpc --workloads oltp --scale tpch1 --backends arcadedb_embedded --build-only --results-file runs_page_dry_2611_post1.jsonl`) and read the row back: `row_kind`, `wheel_sha256`, `build_s`, and no timed figure. The laptop verifies, the mini measures. +5. **Mini, only on the go.** Worktrees at the harness commit, one per chain (`git worktree add --detach ~/post1-wt/ `, `results` linked as `fourarm/run.sh` does), the generated folder copied to `~/stages-2611c/post1/`, then `touch ~/POST1.go` and `setsid nohup bash run_post1.sh`. One timed job at a time, `ORDER.txt` is the order, removing the go file stops it at the next stage boundary. +6. **Landing.** `land_stage.py --pin --post1-reps 3 --dry-run`, read the per-cell decisions, then `--apply`. A build-only row anywhere else is refused. +7. **Rows and state of the live pass are never touched.** The rerun writes `runs_page__2611_post1.jsonl` and `rep-state-2026-11-[-]-post1.tsv`. + +## What the paper may say + +For a rerun cell, only the post1 figures of the fields above, labelled with the wheel (`26.11.1.post1`, sha256 in the artifact), and the 26.11.1 figures for everything else of the cell, labelled as 26.11.1. The 26.11.1 rows of a rerun cell stay in the raw logs and are not printed (DECISIONS #298). A cell that was not rerun keeps its 26.11.1 rows. The served arms never change (they do not use the wheels). + +## n and repetitions + +DECISIONS #240: a speed claim needs 3 runs per arm at 2x or more, 5 or more at 1.3x to 2x, 10 or more below 1.3x, pre-registered in the README and the brief. The landing's `--post1-reps N` is that n: a cell with fewer post1 repetitions than N is refused, never filled from 26.11.1. The paper's n=3 is `--post1-reps 3`; a pass 4 or 5 of the live chain would need the same number of post1 repetitions. + +## Rollback + +`uv run --no-project python post1_rows.py strip --label 26.11.1.post1 results/runs.jsonl` deletes every row that carries the wheel label (a full post1 row, or a cell merged from build-only rows), then `land_stage.py` without `--post1-reps` lands the live rows file again, which restores the replaced 26.11.1 rows, and the page is rebuilt by the landing. + +## Not verified here + +Nothing was run on the mini or in docker. The freeze's handling of the added columns (`row_kind`, `wheel_*`, `build_*`, `timed_*`, `build_window_*`) and of two `engine_version` strings in one freeze (the one-version gate of the four-arm freeze) is checked on the landing's dry run, not before. `verify_pair_c25.sh` may compare the wheel version with the engine jar version and has not been read for `.postN`. diff --git a/benchmarks/experiments/e2_hybrid.py b/benchmarks/experiments/e2_hybrid.py index afcff8881ba..be24033602e 100644 --- a/benchmarks/experiments/e2_hybrid.py +++ b/benchmarks/experiments/e2_hybrid.py @@ -2377,6 +2377,11 @@ def main(): # strong claim while demonstrating the weak one. Declared per adapter; # "not declared" is honest about the ones that have not been asked yet. out["txn_scope"] = getattr(b, "TXN_SCOPE", "not declared") + # BUILD-ONLY MODE (post1_rows.py, runner.py --build-only): stop here, after build() and before any timed work, and write the row + # marked row_kind=build_only. Unset by default, so nothing below changes for any run that does not ask for it. + if os.environ.get("BENCH_BUILD_ONLY") == "1": + import post1_rows + post1_rows.finish_build_only(out, args.out, b) if args.workload == "hybrid": # ------------------------------------------------------------------ diff --git a/benchmarks/experiments/engine_pin.py b/benchmarks/experiments/engine_pin.py index d258eee5fbd..9fde3cf5c09 100644 --- a/benchmarks/experiments/engine_pin.py +++ b/benchmarks/experiments/engine_pin.py @@ -70,6 +70,12 @@ def check_image_tag(image, pin=None): return None +def jar_pin(pin): + """The engine jar version a pin stands for: a wheels-only X.Y.Z.postN carries the engine jars of X.Y.Z (DECISIONS #297; the byte-identity + of the jars is checked at the release), so its jar is named X.Y.Z. Every other pin is its own jar version (post1_rows.py, the post1 rerun).""" + return re.sub(r"\.post\d+$", "", pin or "") + + def normalize_jar_version(v): """The jar of a snapshot build says 26.11.1-SNAPSHOT where its wheel says 26.11.1.dev0.""" return re.sub(r"-SNAPSHOT$", ".dev0", v) @@ -123,13 +129,13 @@ def verify(pin, embedded_images=(), served_images=(), run=None): continue if got["wheel"] != pin: problems.append(f"{image} has arcadedb-embedded {got['wheel']}, the stage pins {pin}") - if got["engine_jar"] != [pin]: + if got["engine_jar"] != [jar_pin(pin)]: problems.append(f"{image} ships engine jar {','.join(got['engine_jar']) or '(none)'}, the stage pins {pin}") for image in sorted(set(served_images)): got, err = read_served(image, run) readback[image] = got if err: problems.append(f"{image}: {err}") - elif got["engine_jar"] != [pin]: + elif got["engine_jar"] != [jar_pin(pin)]: problems.append(f"{image} loads engine jar {','.join(got['engine_jar']) or '(none)'}, the stage pins {pin}") return readback, problems diff --git a/benchmarks/experiments/l1_tpc.py b/benchmarks/experiments/l1_tpc.py index 9159f7f64ed..60a47d96064 100644 --- a/benchmarks/experiments/l1_tpc.py +++ b/benchmarks/experiments/l1_tpc.py @@ -1998,6 +1998,11 @@ def main(): # from one place (DECISIONS #90). bench_common.stamp_durability(out, getattr(b, "durability", None)) out["instrument"] = bench_common.INSTRUMENT + # BUILD-ONLY MODE (post1_rows.py, runner.py --build-only): stop here, after build() and before any timed work, and write the row + # marked row_kind=build_only. Unset by default, so nothing below changes for any run that does not ask for it. + if os.environ.get("BENCH_BUILD_ONLY") == "1": + import post1_rows + post1_rows.finish_build_only(out, args.out, b) if args.workload == "olap": out["olap_iters"] = OLAP_ITER # THE Q1 THIS ROW RAN (row 57, BUGS F170): all ten of TPC-H Q1's output columns. The page's diff --git a/benchmarks/experiments/l2_graph.py b/benchmarks/experiments/l2_graph.py index e6c955d7cca..528c9715aaa 100644 --- a/benchmarks/experiments/l2_graph.py +++ b/benchmarks/experiments/l2_graph.py @@ -3570,6 +3570,16 @@ def gen_edges(n, *a, **kw): # noqa: F811 - deliberate shadow, counted # adapter that read its value out of its own engine wins over the map. bench_common.stamp_durability(out, getattr(ad, "durability", None) or DURABILITY.get(args.backend)) + # BUILD-ONLY MODE (post1_rows.py, runner.py --build-only): stop here, after build() and before any timed work, and write the row + # marked row_kind=build_only. Unset by default, so nothing below changes for any run that does not ask for it. + if os.environ.get("BENCH_BUILD_ONLY") == "1": + # the GAV twin keeps its own canonical key on a build-only row too (see the stamp at the end of the olap branch) + if args.workload == "olap": + out["gav"] = os.environ.get("BENCH_GAV", "1") != "0" + if not out["gav"]: + out["backend_arm"] = "nogav" + import post1_rows + post1_rows.finish_build_only(out, args.out, ad) if args.workload == "oltp": ids = pick_query_ids(n_persons, n_q) diff --git a/benchmarks/experiments/l3d_dense.py b/benchmarks/experiments/l3d_dense.py index 9d163e26f7f..dd247a63c83 100644 --- a/benchmarks/experiments/l3d_dense.py +++ b/benchmarks/experiments/l3d_dense.py @@ -3332,6 +3332,11 @@ def main(): # #6858's trigger fires from the QUERY path: a build-time snapshot alone # cannot show a rebuild that the searches provoked. out["engine_stats_after_build"] = b.engine_stats() + # BUILD-ONLY MODE (post1_rows.py, runner.py --build-only): stop here, after build() and before any timed work, and write the row + # marked row_kind=build_only. Unset by default, so nothing below changes for any run that does not ask for it. + if os.environ.get("BENCH_BUILD_ONLY") == "1": + import post1_rows + post1_rows.finish_build_only(out, args.out, b) WARM = 20 timed_n = len(test) - WARM diff --git a/benchmarks/experiments/land_stage.py b/benchmarks/experiments/land_stage.py index 5f33484a7e8..41ab2a06aee 100644 --- a/benchmarks/experiments/land_stage.py +++ b/benchmarks/experiments/land_stage.py @@ -413,6 +413,9 @@ def main(): help="dense multipass arm token to pull at both sizes (e.g. neo4jvec, pgvector)") ap.add_argument("--message", required=True, help="one-line commit subject for both repos") ap.add_argument("--apply", action="store_true", help="build, commit and push; default stops after the diff") + ap.add_argument("--post1-reps", type=int, default=0, metavar="N", + help="also pull runs_page__2611_post1.jsonl (the wheels-only rerun) and land, per cell, the rows of ONE wheel " + "(post1_rows.resolve at N repetitions); exits nonzero on a cell that would mix wheels or is short of repetitions. Off by default") ap.add_argument("--dry-run", action="store_true", help="pull and filter, then STOP before the merge. Without this, " "a run with no --apply still merges: --apply governs the " @@ -523,6 +526,22 @@ def main(): excl = {b.strip() for b in args.exclude_backends.split(",") if b.strip()} lanes = {l.strip() for l in args.only_lanes.split(",") if l.strip()} rows = [json.loads(l) for l in pulled.read_text().splitlines() if l.strip()] + if args.post1_reps: + import post1_rows + _p1 = SCRATCH / "runs_page_host_post1.jsonl" + sh(["scp", "-q", f"{HOST}:{REMOTE}/{host_name('runs_page', args.pin, '_post1.jsonl')}", str(_p1)]) + rows, _dec, _prob = post1_rows.resolve(rows + post1_rows.load_jsonl(str(_p1)), args.post1_reps) + for _d in _dec: + print(" post1: " + _d) + if _prob: + sys.exit("REFUSING: the post1 rows do not resolve to one wheel per cell, nothing was merged:\n " + "\n ".join(_prob)) + pulled = SCRATCH / "runs_page_host_resolved.jsonl" + pulled.write_text("".join(json.dumps(r) + "\n" for r in rows)) + # A BUILD-ONLY ROW IS NOT A CELL (post1_rows.py): it has no timed figure, and the canonical rule (newest ts_utc per key) would let it + # shadow a full row. It reaches the freeze only merged into a full row by --post1-reps; any other way is refused. + _bo = [r for r in rows if r.get("row_kind") == "build_only"] + if _bo: + sys.exit(f"REFUSING: {len(_bo)} build-only row(s) in the pulled rows; land them with --post1-reps N (post1_rows.py), never as cells") # Before the --exclude filters and before any merge: the whole file is one instrument at one pin or it is not landed. screen_pulled_rows(rows, args.pin, host_name("runs_page", args.pin, ".jsonl") + f" on {HOST}") unless = [parse_drop_unless(x) for x in args.drop_unless] diff --git a/benchmarks/experiments/make_post1_rerun_stages.py b/benchmarks/experiments/make_post1_rerun_stages.py new file mode 100644 index 00000000000..0497e02b6aa --- /dev/null +++ b/benchmarks/experiments/make_post1_rerun_stages.py @@ -0,0 +1,526 @@ +#!/usr/bin/env python3 +"""Emit the stage scripts of the wheels-only post1 re-measure of the paper's ArcadeDB EMBEDDED cells (DECISIONS #293 to #306, CAMPAIGN section 7 +row 93, humemai/arcadedb-embedded-python#581): the embedded arms only, build-only where the cell list says so, the post1 wheel pinned by name and +sha256, and nothing of it touching the live pass. + +It does not template anything itself. Every stage is what make_2611_stages.stage_script writes (the TEMP GATE, the guards, the cap rule, the rep +gate), on a stage table narrowed to the embedded arms (the served arms never use the wheels: DECISIONS #297), then four exact-string changes +that each assert their anchor count, the way fourarm/run.sh patches its stages: + + 1. the rows go to `runs_page__2611_post1.jsonl` and the rep state to `rep-state--[-]-post1.tsv`, so no row or + state of the live pass is read, skipped or shadowed (rep_done would skip every cell already `ok`); + 2. `--build-only` is passed to the runner for the build-only cells (the runner refuses it for any other results file); + 3. BENCH_WHEEL_LABEL, BENCH_WHEEL_NAME and BENCH_WHEEL_SHA256 are exported, which makes the runner stamp them on every row (post1_rows.py); + 4. with --tree and --harness-sha, the live chain's own patches: run from a worktree at a pinned commit (no git pull), the idle-state hold, the + turbo gate, and a deadline on every predecessor wait. + +A wheel stage `00` per chain puts the wheel into the worktree's dist/ by URL and sha256 and builds the arm image from it. The pair (the +server image, the engine) is NOT reinstalled: the engine is byte-identical (DECISIONS #297), and the stages read the pair that the pass installed. + +THE CELL LIST is a TSV: lane workload scale backend class arm kind mode. `--write-cells FILE` writes the default (every embedded cell of the paper +profile: lane cells `build-only`, overlays `skip`); edit the mode column (build-only, full, skip) to say which cells are rerun and how. A +`build-only` row may replace only post1_rows.BUILD_FIELDS; a glue change that touches a query path (#321, #323) needs `full`. + +Usage: + python3 make_post1_rerun_stages.py --selftest + python3 make_post1_rerun_stages.py --write-cells post1_cells.tsv + python3 make_post1_rerun_stages.py --dry-run [--cells post1_cells.tsv] [--hours HOURS.tsv] --chains jvm,native --reps 3 + python3 make_post1_rerun_stages.py --out DIR --chains jvm,native --reps 3 --engine-commit <40 hex> \\ + --wheel-label 26.11.1.post1 --jvm-wheel-name arcadedb_jvm-26.11.1.post1-.whl --jvm-wheel-sha256 <64 hex> --jvm-wheel-url \\ + --native-wheel-name arcadedb-26.11.1.post1-.whl --native-wheel-sha256 <64 hex> --native-wheel-url \\ + --native-image-digest sha256:<64 hex> --tree-root ~/post1-wt --harness-sha <40 hex> [--cells post1_cells.tsv] [--after none] +""" +from __future__ import annotations + +import argparse +import collections +import os +import re +import subprocess +import sys +import tempfile + +HERE = os.path.dirname(os.path.abspath(__file__)) +sys.path.insert(0, HERE) +import make_2611_stages as M # noqa: E402 + +ROWS_SUFFIX_POST1 = "_2611_post1" +STATE_SUFFIX = "-post1" +MODES = ("build-only", "full", "skip") +CHAINS = ("jvm", "native") +DEFAULT_PREFIX = {"jvm": "qPV", "native": "qPN"} +CELL_HEADER = ["lane", "workload", "scale", "backend", "class", "arm", "kind", "mode"] + + +# ------------------------------------------------------------------------------------------------------ the cells +def embedded(backend): + return "server" not in backend and "http_client" not in backend and "imgdefaults" not in backend + + +def base_specs(): + """The paper profile's ArcadeDB stage pieces (as the live chain's generator builds them), narrowed to the embedded arms.""" + profile = M.DEFAULT_PROFILE + specs = M.subset_specs(["arcadedb"], list(M.profile_lanes(profile)), "biggest-last", M.profile_skip(profile), False) + out = [] + for s in specs: + arms = [b for b in s[8] if embedded(b)] + if arms: + p = M._piece(M._spec10(s), s[1], arms, s[4]) + out.append(p) + return out + + +def default_cells(): + """[(Cell, mode)] for every embedded cell of the profile: lane cells build-only, overlays skip (an overlay times queries).""" + cells = [] + for spec in base_specs(): + for c in M.stage_cells(spec): + cells.append((c, "skip" if c.kind == "overlay" else "build-only")) + return cells + + +def cell_row(c, mode): + return [c.lane, c.workload, c.scale, c.backend, c.cls, c.arm or "-", c.kind, mode] + + +def write_cells(path): + with open(path, "w", encoding="utf-8") as fh: + fh.write("# post1 rerun cell list: mode is build-only, full or skip (make_post1_rerun_stages.py)\n") + fh.write("\t".join(CELL_HEADER) + "\n") + for c, mode in default_cells(): + fh.write("\t".join(cell_row(c, mode)) + "\n") + + +def read_cells(path): + """{(lane, workload, scale, backend, class, arm, kind): mode}; every default cell must be named, none may be invented.""" + rows = [ln.rstrip("\n").split("\t") for ln in open(path, encoding="utf-8") if ln.strip() and not ln.startswith("#")] + if rows[0] != CELL_HEADER: + raise SystemExit(f"{path}: header {rows[0]}, expected {CELL_HEADER}") + got = {} + for r in rows[1:]: + if len(r) != 8 or r[7] not in MODES: + raise SystemExit(f"{path}: bad row {r} (mode must be one of {MODES})") + got[tuple(r[:7])] = r[7] + want = {tuple(cell_row(c, "x")[:7]) for c, _m in default_cells()} + if set(got) - want: + raise SystemExit(f"{path}: cells the paper profile does not run as embedded cells: {sorted(set(got) - want)[:3]}") + if want - set(got): + raise SystemExit(f"{path}: {len(want - set(got))} cell(s) missing (name every cell, set mode skip to leave one out): {sorted(want - set(got))[:3]}") + return got + + +def plan_pieces(modes): + """[(piece spec, mode)] covering exactly the cells whose mode is not skip, one mode per piece. A stage piece keeps the table's shape + (one lane, several workloads and one or more scales) when its cells are all wanted in one mode, else it is split down to (scale, workload, + class) pieces, and the pieces' cells are proven equal to the wanted ones.""" + want = {k: m for k, m in modes.items() if m != "skip"} + out, covered = [], set() + for spec in base_specs(): + cells = M.stage_cells(spec) + keys = [tuple(cell_row(c, "x")[:7]) for c in cells] + used = {modes[k] for k in keys if modes[k] != "skip"} + if not used: + continue + lane_cells_skipped = [k for k in keys if modes[k] == "skip" and k[6] == "lane"] + if len(used) == 1 and not lane_cells_skipped: + out.append((spec, next(iter(used)))) + covered |= {k for k in keys if modes[k] != "skip"} + continue + for mode in sorted(used): + for sc in spec[4]: + for wl in spec[3]: + for dur in (None, "relaxed", "strict"): + p = list(spec) + p[3], p[4], p[9] = [wl], [sc], dur + ks = [tuple(cell_row(c, "x")[:7]) for c in M.stage_cells(p)] + if dur is None and not ks: + continue + if ks and all(modes[k] == mode for k in ks): + if dur is None and any(k in covered for k in ks): + continue + if not any(k in covered for k in ks): + out.append((p, mode)) + covered |= set(ks) + # the overlay is part of a piece of an overlay lane unless the piece runs build-only: the stage text drops it for build-only pieces + have = {tuple(cell_row(c, "x")[:7]) for p, m in out for c in M.stage_cells(p) if not (m == "build-only" and c.kind == "overlay")} + if not set(want) <= have: + raise SystemExit(f"internal: the pieces cover {len(have)} cells, the list wants {len(want)}: {sorted(set(want) - have)[:3]}") + extra = {k for k in have if k not in want} + if extra: + raise SystemExit(f"internal: the pieces would also run {sorted(extra)[:3]}, which the cell list skips; split the list differently") + return out + + +# --------------------------------------------------------------------------------------------------- the patches +def _sub(text, old, new, must=1, what=""): + n = text.count(old) + if n != must: + raise SystemExit(f"generator drift: anchor {what or old[:60]!r} found {n}x, expected {must}; make_2611_stages.py changed, fix this script") + return text.replace(old, new) + + +def post1_patch(text, label, wheel_name, wheel_sha, build_only, chain): + text = _sub(text, "#!/bin/bash\n", "#!/bin/bash\n# POST1 RERUN (make_post1_rerun_stages.py): embedded arms only, wheel " + label + ", " + + ("BUILD-ONLY cells" if build_only else "full cells") + ", rows in runs_page_" + ROWS_SUFFIX_POST1 + ".jsonl\n", what="shebang") + text, n = re.subn(r"(REPSTATE=\$HOME/rep-state-[^\n]*?)\.tsv", r"\1" + STATE_SUFFIX + ".tsv", text) + if n != 1: + raise SystemExit(f"generator drift: REPSTATE line matched {n}x, expected 1") + text = _sub(text, 'say "$ID START:', f'BUILD_ONLY_FLAG="{"--build-only" if build_only else ""}" # post1_rows.py; the runner refuses it for any results file not named _postN\nsay "$ID START:', what="START") + text = _sub(text, '--only-reps "$PASS" --reps "$REPS" --results-file "$RF" >> "$S" 2>&1', + '--only-reps "$PASS" --reps "$REPS" --results-file "$RF" $BUILD_ONLY_FLAG >> "$S" 2>&1', what="run_cell runner line") + text = _sub(text, "export BENCH_CPUSET=0-11 BENCH_GRAPH_SOURCE=ldbc\n", + "export BENCH_CPUSET=0-11 BENCH_GRAPH_SOURCE=ldbc\n" + f"export BENCH_WHEEL_LABEL={label} BENCH_WHEEL_NAME={wheel_name} BENCH_WHEEL_SHA256={wheel_sha} # every row says which wheel it ran on (post1_rows.stamp_row)\n", + what="cpuset export") + return text + + +def tree_patch(text, tree, hsha, wheel_stage=False): + """The live chain's patches (fourarm/run.sh patch_stage), for a stage run from a worktree at a pinned commit.""" + text = _sub(text, "REPO=$HOME/repos/humemai/arcadedb-embedded-python", f"REPO={tree}", what="REPO") + pull = 'cd "$REPO" && git pull -q --ff-only || { say "$ID ABORT: pull"; exit 1; }' + text = _sub(text, pull, f'cd "$REPO" && [ "$(git rev-parse HEAD)" = "{hsha}" ] || {{ say "$ID ABORT: worktree is not at the harness pin {hsha}"; exit 1; }}', what="git pull") + text = _sub(text, "set -u\n", "set -u\nexport BENCH_CPU_DMA_LATENCY_US=${BENCH_CPU_DMA_LATENCY_US:-10} # DECISIONS #234: unset = 0 us in the runner\n", what="set -u") \ + if not wheel_stage else text + if not wheel_stage: + trap = "trap 'echo \"$ID ALL-DONE\" >> \"$S\"' EXIT\n" + text = _sub(text, trap, trap + '# TURBO GATE (the user, 2026-10-24; same text as fourarm/run.sh)\n' + '. "${AB2610:-$HOME/ab-2610}/lanes/turbo_gate.sh" 2>/dev/null || { say "$ID ABORT: ab-2610/lanes/turbo_gate.sh missing"; exit 1; }\n' + 'turbo_gate "$ID" || exit 0\n', what="ALL-DONE trap") + text = re.sub(r'while ! grep -qx "(q[A-Z]+\d+) ALL-DONE" "\$S" 2>/dev/null; do sleep 300; done', + r'_w0=$SECONDS; while ! grep -qx "\1 ALL-DONE" "$S" 2>/dev/null; do [ $((SECONDS - _w0)) -lt 600 ] || { say "$ID ABORT: marker \1 missing after 600 s"; exit 1; }; sleep 5; done', text) + for leak in ("git pull", "$HOME/repos/humemai/arcadedb-embedded-python", "sleep 300; done"): + if leak in text: + raise SystemExit(f"{leak!r} still in a patched stage") + return text + + +WHEEL_STAGE = '''#!/bin/bash +# {id}. POST1 wheel stage ({chain}): {wheel_name} into {tree}/bindings/python/dist by sha256, and the arm image built from it. +# +# GENERATED by make_post1_rerun_stages.py. The pair (server image, engine) is NOT reinstalled: the engine is byte-identical to 26.11.1 +# (DECISIONS #297) and the lane stages read the pair the pass installed. Nothing here is found, everything is named. +set -u +ID={id}; S=$HOME/STATUS.txt; REPO={tree}; R=$REPO/benchmarks/experiments +say() {{ echo "[$(date -Is)] $*" >> "$S"; }} +{wait} +trap 'echo "$ID ALL-DONE" >> "$S"' EXIT +SHA={sha}; IMG=arcadedb-c25:${{SHA:0:9}} +WN={wheel_name}; WSHA={wheel_sha256}; WURL={wheel_url} +say "$ID START: post1 wheel $WN ({chain}), pin ${{SHA:0:9}}" +cd "$REPO" && [ "$(git rev-parse HEAD)" = "{hsha}" ] || {{ say "$ID ABORT: worktree is not at the harness pin {hsha}"; exit 1; }} +docker image inspect "$IMG" > /dev/null 2>&1 || {{ say "$ID ABORT: no pair image $IMG (the pass installs it; this stage does not)"; exit 1; }} +mkdir -p "$REPO/bindings/python/dist" +[ -z "$(ls "$REPO"/bindings/python/dist/*.whl 2>/dev/null)" ] || {{ say "$ID ABORT: dist/ of this worktree is not empty; one wheel, or say which"; exit 1; }} +T=$(mktemp -d); curl -sfL -o "$T/$WN" "$WURL" && [ "$(sha256sum "$T/$WN" | cut -c1-64)" = "$WSHA" ] || {{ say "$ID ABORT: wheel download or sha256"; exit 1; }} +mv "$T/$WN" "$REPO/bindings/python/dist/" || {{ say "$ID ABORT: dist move"; exit 1; }} +say "$ID: dist/ holds $WN only" +cd "$R" && ARCADEDB_WHEEL="$REPO/bindings/python/dist/$WN" ARCADEDB_SERVER_IMAGE="$IMG" BENCH_ALLOW_DEV=0 bash build_images.sh arcadedb >> "$S" 2>&1 || {{ say "$ID ABORT: build_images.sh arcadedb"; exit 1; }} +say "$ID finished: the {chain} arm image carries $WN; the lane stages start" +''' + + +DRIVER = '''#!/usr/bin/env bash +# Sequential driver of the post1 rerun (make_post1_rerun_stages.py). One timed job at a time: each stage of ORDER.txt runs after the one before it. +# Needs the user's go as a FILE ($POST1_GO, default ~/POST1.go); removing it stops the run at the next stage boundary. Restartable: stages that +# finished are in done.list, and a restarted stage skips every repetition its rep-state file holds as clean. +set -u +HERE=$(cd "$(dirname "$0")" && pwd); S=${POST1_STATUS:-$HOME/STATUS.txt}; GO=${POST1_GO:-$HOME/POST1.go}; DONE=$HERE/done.list +say() { echo "[$(date -Is)] $*" >> "$S"; } +[ -e "$GO" ] || { echo "no $GO: the go is a file only the user writes" >&2; exit 2; } +touch "$DONE" +while IFS=$'\\t' read -r item pass mode n; do + [ -n "$item" ] || continue + grep -qxF "$item" "$DONE" && continue + [ -e "$GO" ] || { say "POST1 STOPPED before $item: $GO removed"; exit 3; } + say "POST1 driver: $item (pass $pass, $mode)" + bash "$HERE/$item.sh" + tail -200 "$S" | grep -q "\\] ${item#*/} finished" || { say "POST1 STOPPED at $item: no finished line (see STATUS)"; exit 4; } + echo "$item" >> "$DONE" +done < "$HERE/ORDER.txt" +say "POST1 driver finished: every stage of ORDER.txt ran" +''' + + +# ------------------------------------------------------------------------------------------------------ generation +def chain_pins(a, chain): + name = getattr(a, f"{chain}_wheel_name") + sha = getattr(a, f"{chain}_wheel_sha256") + url = getattr(a, f"{chain}_wheel_url") + if not (name and sha and url): + raise SystemExit(f"REFUSING: the {chain} chain needs --{chain}-wheel-name, --{chain}-wheel-sha256 and --{chain}-wheel-url") + if not re.fullmatch(r"[0-9a-f]{64}", sha): + raise SystemExit(f"REFUSING: --{chain}-wheel-sha256 must be 64 hex characters") + pre = "arcadedb_jvm" if chain == "jvm" else "arcadedb" + if not name.startswith(f"{pre}-{a.wheel_label}-") or not name.endswith(".whl"): + raise SystemExit(f"REFUSING: {name} is not a {pre} wheel of {a.wheel_label} (the file name carries the version the stage pins)") + if not (url.startswith("https://") or url.startswith("file:///")) or not url.endswith(name): + raise SystemExit("REFUSING: the wheel URL must be https (or file:///) and end in the wheel's file name") + pins = {"ARCADEDB_WHEEL": os.path.join("bindings", "python", "dist", name), "ARCADEDB_WHEEL_SHA256": sha, + "ARCADEDB_SERVER_IMAGE": f"arcadedb-c25:{a.engine_commit[:9]}", "ARCADEDB_ENGINE_COMMIT": a.engine_commit, + "wheel_name": name, "wheel_sha256": sha} + if chain == "native": + if not a.native_image_digest: + raise SystemExit("REFUSING: the native chain needs --native-image-digest (the pass's own pin; the stage re-verifies the image)") + pins["ARCADEDB_NATIVE_SERVER_IMAGE"] = f"arcadedata/arcadedb@{a.native_image_digest}" + return pins, url + + +def emit_chain(a, chain, modes, out, first_after, start_index=0): + """Write the chain's wheel stage and lane stages. Returns [(stage id, pass, mode, [cells])] in run order.""" + pins, url = chain_pins(a, chain) + if chain == "native": + if not hasattr(M, "CHAIN_BUILD"): + raise SystemExit("REFUSING: this tree's make_2611_stages.py has no native chain (the four-arm generator, PR #554); generate from a tree that has it") + M.CHAIN_BUILD = "native" + elif hasattr(M, "CHAIN_BUILD"): + M.CHAIN_BUILD = "" + pieces = plan_pieces(modes) + prefix = a.id_prefix_jvm if chain == "jvm" else a.id_prefix_native + os.makedirs(out, exist_ok=True) + order = [] + wid = f"{prefix}00" + tree = os.path.join(a.tree_root or "$HOME/post1-wt", chain) + with open(os.path.join(out, f"{wid}.sh"), "w", encoding="utf-8") as fh: + fh.write(WHEEL_STAGE.format(id=wid, chain=chain, tree=tree, wait=M._wait(first_after), sha=a.engine_commit, wheel_name=pins["wheel_name"], + wheel_sha256=pins["wheel_sha256"], wheel_url=url, hsha=a.harness_sha or "UNPINNED")) + os.chmod(os.path.join(out, f"{wid}.sh"), 0o755) + order.append((wid, 0, "wheel", [])) + total = len(pieces) * a.reps + after = wid + idx = 0 + saved_overlay, saved_suffix = dict(M.OVERLAY), M.ROWS_SUFFIX + M.ROWS_SUFFIX = ROWS_SUFFIX_POST1 + try: + for pass_no in range(1, a.reps + 1): + for spec, mode in pieces: + idx += 1 + sid = f"{prefix}{idx:02d}" + title = ("POST1 build-only, " if mode == "build-only" else "POST1 ") + spec[1] + s = list(spec) + s[0], s[1] = sid, title + M.OVERLAY.clear() + if mode == "full": + M.OVERLAY.update(saved_overlay) + params = dict(M.stage_params(a.reps), pass_no=pass_no) + text = M.stage_script(idx - 1, total, tuple(s), after, pins, params) + text = post1_patch(text, a.wheel_label, pins["wheel_name"], pins["wheel_sha256"], mode == "build-only", chain) + if a.tree_root: + text = tree_patch(text, tree, a.harness_sha) + p = os.path.join(out, f"{sid}.sh") + with open(p, "w", encoding="utf-8") as fh: + fh.write(text) + os.chmod(p, 0o755) + order.append((sid, pass_no, mode, [c for c in M.stage_cells(tuple(s)) if not (mode == "build-only" and c.kind == "overlay")])) + after = sid + finally: + M.OVERLAY.clear() + M.OVERLAY.update(saved_overlay) + M.ROWS_SUFFIX = saved_suffix + return order + + +# ----------------------------------------------------------------------------------------------------- the estimate +def load_hours(path): + """{(chain, lane, workload, scale, backend, class, arm, kind): (full hours per rep, build-only hours per rep)} from a TSV with the header + chain lane workload scale backend class arm kind h_full h_build. ESTIMATES: scheduling facts from the mini's STATUS file, never timings.""" + rows = [ln.rstrip("\n").split("\t") for ln in open(path, encoding="utf-8") if ln.strip() and not ln.startswith("#")] + if rows[0][:10] != ["chain"] + CELL_HEADER[:7] + ["h_full", "h_build"]: + raise SystemExit(f"{path}: header {rows[0]}") + return {tuple(r[:8]): (float(r[8]), float(r[9])) for r in rows[1:]} + + +def estimate(order, chain, reps, hours): + """[(stage id, pass, mode, hours, priced, unpriced)]; hours for ONE pass of a stage, so n repetitions are the sum over passes.""" + out = [] + for sid, pass_no, mode, cells in order: + if mode == "wheel": + continue + h, n, miss = 0.0, 0, [] + for c in cells: + v = hours.get((chain, c.lane, c.workload, c.scale, c.backend, c.cls, c.arm or "-", c.kind)) + if v is None: + miss.append(c.label) + continue + h += v[1] if mode == "build-only" else v[0] + n += 1 + out.append((sid, pass_no, mode, h, n, miss)) + return out + + +def print_dry_run(a, modes, hours): + for chain in a.chains: + order = emit_chain(a, chain, modes, tempfile.mkdtemp(prefix="post1dry-"), M._after_arg(a.after) if hasattr(M, "_after_arg") else None) \ + if a.wheel_label and a.engine_commit and getattr(a, f"{chain}_wheel_name") else dry_order(a, chain, modes) + print(f"\n== {chain} chain: {len(order) - 1} lane stage(s), reps {a.reps}") + est = estimate(order, chain, a.reps, hours) if hours else [] + by_id = {e[0]: e for e in est} + for sid, pass_no, mode, cells in order: + if mode == "wheel": + print(f" {sid} wheel stage (download by sha256, build the arm image)") + continue + e = by_id.get(sid) + print(f" {sid} pass {pass_no} {mode:10s} {len(cells):2d} job(s) " + (f"{e[3]:6.2f} h" if e else "") + " " + ", ".join(sorted({f'{c.lane}/{c.scale}/{c.workload}' for c in cells}))[:90]) + if est: + tot = sum(e[3] for e in est) + miss = sorted({m for e in est for m in e[5]}) + print(f" ESTIMATE {chain}: {tot:.1f} h for {a.reps} repetition(s), summed over the stages above (setup excluded; not a measurement)" + + (f"; {len(miss)} unpriced: {miss[:3]}" if miss else "")) + + +def dry_order(a, chain, modes): + """The run order without writing any script (no pins needed): [(id, pass, mode, cells)].""" + prefix = a.id_prefix_jvm if chain == "jvm" else a.id_prefix_native + pieces = plan_pieces(modes) + order, idx = [(f"{prefix}00", 0, "wheel", [])], 0 + for pass_no in range(1, a.reps + 1): + for spec, mode in pieces: + idx += 1 + order.append((f"{prefix}{idx:02d}", pass_no, mode, [c for c in M.stage_cells(tuple(spec)) if not (mode == "build-only" and c.kind == "overlay")])) + return order + + +# ---------------------------------------------------------------------------------------------------------- selftest +def selftest(): + fails = [] + + def check(name, cond): + if not cond: + fails.append(name) + + modes = {tuple(cell_row(c, "x")[:7]): m for c, m in default_cells()} + check("22 default cells", len(modes) == 22) + check("2 overlays skipped", sum(1 for k, m in modes.items() if m == "skip") == 2 and all(k[6] == "overlay" for k, m in modes.items() if m == "skip")) + check("embedded arms only", all(embedded(k[3]) for k in modes)) + pieces = plan_pieces(modes) + jobs = [c for p, m in pieces for c in M.stage_cells(p) if c.kind == "lane"] + check("20 build-only lane jobs per pass", len(jobs) == 20 and all(m == "build-only" for _p, m in pieces)) + a = argparse.Namespace(reps=2, wheel_label="26.11.1.post1", engine_commit="a" * 40, harness_sha="b" * 40, tree_root="/home/x/post1-wt", after="none", + id_prefix_jvm="qPV", id_prefix_native="qPN", jvm_wheel_name="arcadedb_jvm-26.11.1.post1-py3-none-manylinux_2_28_x86_64.whl", + jvm_wheel_sha256="c" * 64, jvm_wheel_url="https://example.invalid/arcadedb_jvm-26.11.1.post1-py3-none-manylinux_2_28_x86_64.whl", + native_wheel_name="arcadedb-26.11.1.post1-py3-none-manylinux_2_28_x86_64.whl", native_wheel_sha256="d" * 64, + native_wheel_url="https://example.invalid/arcadedb-26.11.1.post1-py3-none-manylinux_2_28_x86_64.whl", + native_image_digest="sha256:" + "e" * 64) + d = tempfile.mkdtemp(prefix="post1sf-") + order = emit_chain(a, "jvm", modes, d, None) + check("jvm: wheel stage + 8 pieces x 2 passes", len(order) == 1 + len(pieces) * 2) + for sid, pass_no, mode, cells in order: + text = open(os.path.join(d, sid + ".sh"), encoding="utf-8").read() + rc = subprocess.run(["bash", "-n", os.path.join(d, sid + ".sh")], capture_output=True).returncode + check(f"bash -n {sid}", rc == 0) + if mode == "wheel": + check("wheel stage names the sha", "c" * 64 in text and "arcadedb_jvm-26.11.1.post1" in text) + continue + m = re.search(r'^BACKENDS="([^"]*)"', text, re.M) + check(f"{sid}: embedded arms only", bool(m) and all(embedded(b) for b in m.group(1).split())) + check(f"{sid}: build-only flag", 'BUILD_ONLY_FLAG="--build-only"' in text and "$BUILD_ONLY_FLAG" in text) + check(f"{sid}: post1 rows file", "_2611_post1.jsonl" in text and 'RF="runs_page_${PIN}_2611.jsonl"' not in text) + check(f"{sid}: post1 rep state", "-post1.tsv" in text) + check(f"{sid}: wheel label exported", "BENCH_WHEEL_LABEL=26.11.1.post1" in text and "BENCH_WHEEL_SHA256=" + "c" * 64 in text) + check(f"{sid}: no overlay in a build-only stage", "run_overlay \"" not in text) + check(f"{sid}: temp gate and rep gate kept", "temp_gate " in text and "rep_gate " in text) + check(f"{sid}: worktree, no pull", "git pull" not in text and ("REPO=/home/x/post1-wt/jvm" in text) and "turbo_gate" in text) + check(f"{sid}: pass {pass_no}", f"PASS={pass_no} " in text or f"PASS={pass_no}\n" in text) + dr = os.path.join(d, "run_post1.sh") + with open(dr, "w", encoding="utf-8") as fh: + fh.write(DRIVER) + check("driver: bash -n", subprocess.run(["bash", "-n", dr], capture_output=True).returncode == 0) + check("driver: reads a tab", "IFS=$'\\t'" in DRIVER) + import engine_pin + check("engine_pin.jar_pin", engine_pin.jar_pin("26.11.1.post1") == "26.11.1" and engine_pin.jar_pin("26.11.1") == "26.11.1" and engine_pin.jar_pin("26.11.1.dev0") == "26.11.1.dev0") + # a cell list with one full cell and a skipped lane cell is split exactly + m2 = dict(modes) + k1 = next(k for k in m2 if k[0] == "l1tpc" and k[2] == "tpch1" and k[1] == "oltp" and k[4] == "strict") + m2[k1] = "skip" + k2 = next(k for k in m2 if k[0] == "l2" and k[2] == "sf1" and k[1] == "oltp" and k[4] == "relaxed") + m2[k2] = "full" + p2 = plan_pieces(m2) + got = collections.Counter() + for p, m in p2: + for c in M.stage_cells(p): + if not (m == "build-only" and c.kind == "overlay"): + got[(tuple(cell_row(c, "x")[:7]), m)] += 1 + check("split: no skipped cell runs", all(k != k1 for (k, _m) in got)) + check("split: the full cell runs full, once", got[(k2, "full")] == 1) + check("split: every other lane cell build-only once", sum(1 for (k, m), n in got.items() if m == "build-only" and n == 1 and k[6] == "lane") == 18) + # a drifted anchor is refused, not silently skipped + try: + post1_patch("#!/bin/bash\nnothing\n", "26.11.1.post1", "x.whl", "0" * 64, True, "jvm") + check("drift refused", False) + except SystemExit: + pass + # the cost table: a priced and an unpriced cell + hrs = {("jvm", c.lane, c.workload, c.scale, c.backend, c.cls, c.arm or "-", c.kind): (1.0, 0.25) for c, _m in default_cells()} + est = estimate(order, "jvm", 2, hrs) + check("estimate: build-only priced at h_build", abs(sum(e[3] for e in est) - 0.25 * 20 * 2) < 1e-9 and all(not e[5] for e in est)) + for f in fails: + print("FAIL", f) + print(f"make_post1_rerun_stages selftest: {'FAILED' if fails else 'ok'} ({len(fails)} failure(s))") + return 1 if fails else 0 + + +def build_parser(): + ap = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) + ap.add_argument("--selftest", action="store_true") + ap.add_argument("--write-cells", metavar="FILE", help="write the default cell list and stop") + ap.add_argument("--cells", metavar="FILE", help="the cell list (default: every embedded cell, build-only)") + ap.add_argument("--dry-run", action="store_true", help="print the stage list and the estimate; write nothing") + ap.add_argument("--hours", metavar="TSV", help="per-cell hours for the estimate (chain lane workload scale backend class arm kind h_full h_build)") + ap.add_argument("--chains", default="jvm,native") + ap.add_argument("--reps", type=int, default=3) + ap.add_argument("--out") + ap.add_argument("--after", default="none") + ap.add_argument("--engine-commit", default="") + ap.add_argument("--wheel-label", default="", help="the release the wheels carry, e.g. 26.11.1.post1") + for ch in CHAINS: + ap.add_argument(f"--{ch}-wheel-name", default="") + ap.add_argument(f"--{ch}-wheel-sha256", default="") + ap.add_argument(f"--{ch}-wheel-url", default="") + ap.add_argument("--native-image-digest", default="") + ap.add_argument("--tree-root", default="", help="directory of the pinned worktrees (/jvm, /native); without it the stages keep git pull") + ap.add_argument("--harness-sha", default="") + ap.add_argument("--id-prefix-jvm", default=DEFAULT_PREFIX["jvm"]) + ap.add_argument("--id-prefix-native", default=DEFAULT_PREFIX["native"]) + return ap + + +def main(argv=None): + a = build_parser().parse_args(argv) + if a.selftest: + return selftest() + if a.write_cells: + write_cells(a.write_cells) + print(f"wrote {a.write_cells}") + return 0 + a.chains = [c for c in a.chains.split(",") if c] + if any(c not in CHAINS for c in a.chains) or not 1 <= a.reps <= 5: + raise SystemExit("--chains is jvm and/or native; --reps is 1 to 5") + modes = read_cells(a.cells) if a.cells else {tuple(cell_row(c, "x")[:7]): m for c, m in default_cells()} + hours = load_hours(a.hours) if a.hours else None + if a.dry_run: + print_dry_run(a, modes, hours) + return 0 + if not (a.out and a.engine_commit and a.wheel_label and re.fullmatch(r"[0-9a-f]{40}", a.engine_commit)): + raise SystemExit("REFUSING: --out, --wheel-label and the full 40-character --engine-commit are required") + if a.tree_root and not re.fullmatch(r"[0-9a-f]{40}", a.harness_sha): + raise SystemExit("REFUSING: --tree-root needs the full 40-character --harness-sha the worktrees are at") + after = M._after_arg(a.after) + allorder = [] + for chain in a.chains: # one chain after the other, each stage waiting for its predecessor's marker: one timed job at a time + d = os.path.join(a.out, chain) + order = emit_chain(a, chain, modes, d, after) + allorder.append((chain, order)) + after = order[-1][0] + with open(os.path.join(a.out, "ORDER.txt"), "w", encoding="utf-8") as fh: + for chain, order in allorder: + for sid, pass_no, mode, cells in order: + fh.write(f"{chain}/{sid}\t{pass_no}\t{mode}\t{len(cells)}\n") + with open(os.path.join(a.out, "run_post1.sh"), "w", encoding="utf-8") as fh: + fh.write(DRIVER) + os.chmod(os.path.join(a.out, "run_post1.sh"), 0o755) + print(f"wrote {sum(len(o) for _c, o in allorder)} stage script(s) under {a.out}; run order in ORDER.txt, driver run_post1.sh (needs the go file)") + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/benchmarks/experiments/post1_rows.py b/benchmarks/experiments/post1_rows.py new file mode 100644 index 00000000000..0f7344c5d0d --- /dev/null +++ b/benchmarks/experiments/post1_rows.py @@ -0,0 +1,403 @@ +#!/usr/bin/env python3 +"""Rows of a wheels-only re-measure (26.11.1.post1): build-only mode, the wheel label, and the per-cell choice of ONE wheel. + +WHY (DECISIONS #292 to #299, CAMPAIGN section 7 row 93, humemai/arcadedb-embedded-python#581). A wheels-only post1 changes the Python and +native glue of the ArcadeDB bindings, not the engine. The paper cells whose figures it can move are rerun on it, and the user's rule is that +the post1 rows OVERRIDE the 26.11.1 rows of the cells that are rerun, with ALL repetitions of a cell from ONE wheel. This module makes that a +property of the data instead of a thing somebody remembers: + + * BUILD-ONLY ROWS. `runner.py --build-only` (env BENCH_BUILD_ONLY=1) makes a lane stop right after `build()` and write a row stamped + `row_kind = "build_only"`. The lane's own `main()` calls `finish_build_only()` at the one place where the load and the index are done and + no timed workload has started. The row holds the build figures and nothing else, so it is NOT a cell and can never stand in for one. + * THE WHEEL LABEL. A row already says `engine_version` and `engine_backend` ("native" or "jvm"). What it did not say is WHICH FILE: the + candidate JVM wheel of the post1 glue A/B says 26.11.1 in its metadata. A stage that opts in (exports BENCH_WHEEL_LABEL and + BENCH_WHEEL_SHA256) makes the runner stamp `wheel_label`, `wheel_name` and `wheel_sha256` on its rows. Rows without them (every 26.11.1 + row already on the mini) are labelled by their `engine_version`. + * THE CHOICE. `resolve()` takes rows from the live pass and from the rerun and returns, for every in-process ArcadeDB cell, the rows of one + wheel: the newest wheel that has rows for the cell, and only when that set is complete at the required repetitions. A cell that would mix + wheels, or whose newer wheel is short of repetitions, is a PROBLEM (the caller exits nonzero), never a silent fall back. + +WHAT A BUILD-ONLY ROW MAY REPLACE, and what it must not. It replaces, in the 26.11.1 row of the same cell and repetition, only BUILD_FIELDS: +the load and index seconds and the facts about the load transport. Every timed workload figure of the cell stays at its 26.11.1 value, and +the memory, disk and cpu figures of the build window are carried beside them as `build_window_*`, never over them (the full row's figures +cover the whole container life, the build-only row's only the build). The replacement is legitimate only for glue changes that do not touch a +query path (DECISIONS #299: #318, #324, #325; not #321 or #323): whether a cell qualifies is the owner's call, written in the cell list. + +DEFAULTS ARE TODAY'S BEHAVIOUR. With none of the new environment variables set, nothing here changes a row, a lane or a stage. + +Usage: + python3 post1_rows.py --selftest + python3 post1_rows.py resolve --reps 3 --out resolved.jsonl LIVE.jsonl POST1.jsonl # prints the decision per cell, nonzero on a problem + python3 post1_rows.py strip --label 26.11.1.post1 results/runs.jsonl # rollback: delete the post1 rows by wheel label +""" +from __future__ import annotations + +import argparse +import hashlib +import json +import os +import re +import sys + +BUILD_ONLY_ENV = "BENCH_BUILD_ONLY" +WHEEL_LABEL_ENV = "BENCH_WHEEL_LABEL" +WHEEL_SHA256_ENV = "BENCH_WHEEL_SHA256" +WHEEL_NAME_ENV = "BENCH_WHEEL_NAME" + +ROW_KIND = "row_kind" +KIND_CELL = "cell" # the value of a row that has no row_kind: every row written before this module +KIND_BUILD_ONLY = "build_only" + +# The ONLY figures a build-only row may put into a full row (see the module docstring). index_before_load says how to read index_s. +BUILD_FIELDS = ("build_s", "ingest_s", "index_s", "index_before_load", "setup_s", "gav_build_s", + "lineitem_buckets", "async_writers", "async_sync", "load_call_rows", "columnar_insert") +# The build window's resource figures: copied under this prefix, never over the full row's own. +WINDOW_PREFIX = "build_window_" +WINDOW_KEY = re.compile(r"^(peak_|mem_|rss|cpu_usec|cpu_|disk_|io_|oom_killed|heap$|jvm_|gc_)") +# What a row must not carry over from the build-only row into the merged one under its own name. +PROVENANCE = ("build_figures_wheel", "build_figures_wheel_sha256", "build_figures_run_id", "build_figures_ts_utc", + "build_figures_replaced", "timed_ts_utc", "timed_wheel") + + +# ----------------------------------------------------------------------------------------------- the lane side +def build_only(): + """True when the cell was asked to stop after build() (runner.py --build-only).""" + return os.environ.get(BUILD_ONLY_ENV, "") == "1" + + +def finish_build_only(out, out_path, adapter=None): + """Called by a lane's main() right after build() and its stamps, in place of the workload. Marks the row, writes it exactly as the lane + writes its own, and ends the process. Does not return.""" + out[ROW_KIND] = KIND_BUILD_ONLY + out["build_only_stopped_after"] = "build" + gav = getattr(adapter, "gav_build_s", None) + if gav is not None and "gav_build_s" not in out: + out["gav_build_s"] = gav + try: + if adapter is not None and hasattr(adapter, "close"): + adapter.close() # untimed and unrecorded: the full cell's close_s belongs to the full cell + except Exception as e: # noqa: BLE001 + out["build_only_close_error"] = f"{type(e).__name__}: {e}"[:200] + with open(out_path, "w") as f: + json.dump(out, f) + print("RESULT " + json.dumps(out), flush=True) + os._exit(0) + + +# --------------------------------------------------------------------------------------------- the runner side +_SHA_CACHE: dict = {} + + +def sha256_file(path): + """sha256 of a file, cached per path (the runner stamps every row of a stage, the wheel is read once).""" + if path not in _SHA_CACHE: + h = hashlib.sha256() + with open(path, "rb") as fh: + for chunk in iter(lambda: fh.read(1 << 20), b""): + h.update(chunk) + _SHA_CACHE[path] = h.hexdigest() + return _SHA_CACHE[path] + + +def stamp_row(row, env=None): + """Runner side, once per row, just before the row is written. Adds nothing unless the stage opted in, so a row of the live pass is byte for + byte what it was. Returns the row.""" + env = os.environ if env is None else env + if env.get(BUILD_ONLY_ENV, "") == "1": + row.setdefault(ROW_KIND, KIND_BUILD_ONLY) # a cell killed before the lane wrote its row still says what it was + label = env.get(WHEEL_LABEL_ENV, "").strip() + if label: + row["wheel_label"] = label + if env.get(WHEEL_NAME_ENV, "").strip(): + row["wheel_name"] = env[WHEEL_NAME_ENV].strip() + if env.get(WHEEL_SHA256_ENV, "").strip(): + row["wheel_sha256"] = env[WHEEL_SHA256_ENV].strip().lower() + return row + + +# ------------------------------------------------------------------------------------------------ the data side +def kind_of(r): + return r.get(ROW_KIND) or KIND_CELL + + +def is_arcade_in_process(r): + b = str(r.get("backend") or "") + return b.startswith("arcadedb") and "server" not in b and "http_client" not in b and "imgdefaults" not in b + + +def build_of(r): + """The build of an in-process ArcadeDB row: its engine_backend, "jvm" when it names none (every row before 26.11.1).""" + return r.get("engine_backend") if r.get("engine_backend") in ("native", "jvm") else "jvm" + + +def wheel_of(r): + """The label of the wheel a row ran on: wheel_label when the stage stamped one, else the engine_version. Version strings that name a + server ("server:26.11.1") do not occur on in-process rows; the digits-and-suffix part is what is compared.""" + lab = str(r.get("wheel_label") or r.get("engine_version") or "").strip() + m = re.search(r"\d+\.\d+\.\d+(?:\.(?:dev|post)\d+)?", lab) + return m.group(0) if m else lab + + +def version_key(label): + """Orders wheel labels: 26.11.1 < 26.11.1.post1 < 26.11.1.post2; a .devN sorts before its release.""" + m = re.fullmatch(r"(\d+)\.(\d+)\.(\d+)(?:\.(dev|post)(\d+))?", label) + if not m: + return (0, 0, 0, -2, 0) + a, b, c, kind, n = m.groups() + rank = {None: 0, "post": 1, "dev": -1}[kind] + return (int(a), int(b), int(c), rank, int(n or 0)) + + +def cell_key(r): + """What identifies a cell apart from its repetition and its wheel: the canonical key of merge_campaign plus the durability class, the + GAV twin marker and the build (native and JVM are two arms of one release).""" + return (r.get("lane"), r.get("scale"), r.get("n_docs"), r.get("workload"), r.get("backend"), + r.get("durability_class") or "relaxed", r.get("backend_arm") or "", r.get("gav") is not False, build_of(r)) + + +def clean(r): + return not r.get("error") and r.get("rc") in (0, None) + + +def _newest_by_rep(rows): + best = {} + for r in rows: + rep = r.get("rep") + if rep not in best or str(r.get("ts_utc")) > str(best[rep].get("ts_utc")): + best[rep] = r + return best + + +def _label(key): + return f"{key[0]}/{key[1]}/{key[3]}/{key[4]}/{key[5]}" + (f"/{key[6]}" if key[6] else "") + f"/{key[8]}" + + +def merge_build_only(base, bo, label_bo): + """The 26.11.1 (or other) full row `base` with the build figures of the build-only row `bo`. Nothing else of the full row moves.""" + out = dict(base) + replaced = [] + for k in BUILD_FIELDS: + if k in bo: + out[k] = bo[k] + replaced.append(k) + for k, v in bo.items(): + if WINDOW_KEY.match(k) and k not in BUILD_FIELDS and not isinstance(v, (dict, list)): + out[WINDOW_PREFIX + k] = v + out["timed_ts_utc"] = base.get("ts_utc") + out["timed_wheel"] = wheel_of(base) + out["build_figures_wheel"] = label_bo + out["build_figures_wheel_sha256"] = bo.get("wheel_sha256") + out["build_figures_run_id"] = bo.get("run_id") + out["build_figures_ts_utc"] = bo.get("ts_utc") + out["build_figures_replaced"] = ",".join(replaced) + out["ts_utc"] = max(str(base.get("ts_utc")), str(bo.get("ts_utc"))) # the canonical rule is newest-wins; the timed figures keep timed_ts_utc + return out + + +def resolve(rows, reps=3): + """(resolved rows, decisions, problems). + + Per in-process ArcadeDB cell: collect the CLEAN rows by wheel label and kind. The newest label that has any row for the cell must be + complete at `reps` repetitions (1..reps), else it is a problem. A full-cell set of a newer wheel replaces the older wheel's rows whole; + a build-only set of a newer wheel replaces the build figures of the older wheel's complete full set, repetition by repetition; a build-only + set with no complete full set underneath is a problem, because it has no cell to belong to. Rows of anything else (comparators, served arms, + failed cells) pass through untouched, and a build-only row never passes through on its own.""" + problems, decisions, out = [], [], [] + groups = {} + for r in rows: + if not is_arcade_in_process(r): + if kind_of(r) == KIND_BUILD_ONLY: + problems.append(f"{r.get('lane')}/{r.get('backend')}: a build-only row of a backend this tooling does not handle") + else: + out.append(r) + continue + groups.setdefault(cell_key(r), []).append(r) + want = list(range(1, reps + 1)) + for key in sorted(groups, key=lambda k: tuple(str(x) for x in k)): + rs = groups[key] + tag = _label(key) + failed = [r for r in rs if not clean(r)] + good = [r for r in rs if clean(r)] + by = {} + for r in good: + by.setdefault((wheel_of(r), kind_of(r)), []).append(r) + labels = sorted({w for w, _k in by}, key=version_key) + out.extend(failed) # failed cells land as what they are, as before (the freeze skips them); they are never a repetition here + if not labels: + continue + newest = labels[-1] + full_new = _newest_by_rep(by.get((newest, KIND_CELL), [])) + bo_new = _newest_by_rep(by.get((newest, KIND_BUILD_ONLY), [])) + have_full = [x for x in want if x in full_new] + have_bo = [x for x in want if x in bo_new] + older = [w for w in labels if w != newest] + if full_new and bo_new: + problems.append(f"{tag}: wheel {newest} has both full rows and build-only rows; one kind per cell and wheel") + continue + if full_new: + if have_full != want: + problems.append(f"{tag}: wheel {newest} has reps {sorted(full_new)} of the {reps} required " + f"(older wheels with rows: {older or 'none'}); a cell never mixes wheels, so finish {newest} or drop its rows") + continue + out.extend(full_new[x] for x in want) + decisions.append(f"{tag}: full cell, wheel {newest}, reps {want}" + (f" (replaces {', '.join(older)})" if older else "")) + continue + if bo_new: + if have_bo != want: + problems.append(f"{tag}: wheel {newest} build-only rows have reps {sorted(bo_new)} of the {reps} required") + continue + if not older: + problems.append(f"{tag}: build-only rows of {newest} and no full cell of an older wheel to put them into") + continue + base_label = older[-1] + base = _newest_by_rep(by.get((base_label, KIND_CELL), [])) + if [x for x in want if x in base] != want: + problems.append(f"{tag}: the full rows of {base_label} have reps {sorted(base)} of the {reps} required, so the build-only rows of " + f"{newest} have no complete cell to be merged into") + continue + out.extend(merge_build_only(base[x], bo_new[x], newest) for x in want) + decisions.append(f"{tag}: timed figures {base_label}, build figures {newest} (build-only), reps {want}") + continue + problems.append(f"{tag}: no usable row at the newest wheel {newest}") + return out, decisions, problems + + +def load_jsonl(path): + rows = [] + with open(path, encoding="utf-8") as fh: + for ln in fh: + ln = ln.strip() + if ln: + rows.append(json.loads(ln)) + return rows + + +# ---------------------------------------------------------------------------------------------------- selftest +def _row(rep, ts, label="26.11.1", kind=None, build="native", backend="arcadedb_embedded", **kw): + r = {"lane": "l1tpc", "scale": "tpch1", "workload": "oltp", "backend": backend, "rep": rep, "ts_utc": ts, "engine_version": label, + "engine_backend": build, "rc": 0, "durability_class": "relaxed", "build_s": 100.0 + rep, "ingest_s": 90.0, "index_s": 10.0, + "crud_read_p50_ms": 1.5, "peak_rss_mb": 900} + if kind == KIND_BUILD_ONLY: + r = {k: v for k, v in r.items() if k != "crud_read_p50_ms"} + r[ROW_KIND] = KIND_BUILD_ONLY + r["peak_rss_mb"] = 400 + r.update(kw) + return r + + +def selftest(): + fails = [] + + def check(name, cond): + if not cond: + fails.append(name) + + # 1. a build-only row is never a cell: no live row, no rerun + live = [_row(i, f"2026-11-0{i}") for i in (1, 2, 3)] + bo = [_row(i, f"2026-11-1{i}", label="26.11.1.post1", kind=KIND_BUILD_ONLY, build_s=50.0 + i, wheel_label="26.11.1.post1") for i in (1, 2, 3)] + res, dec, prob = resolve(live + bo, 3) + check("merge: no problem", not prob) + check("merge: 3 rows", len(res) == 3) + check("merge: build figure from post1", [r["build_s"] for r in res] == [51.0, 52.0, 53.0]) + check("merge: timed figure from 26.11.1", all(r["crud_read_p50_ms"] == 1.5 for r in res)) + check("merge: window memory kept apart", all(r["peak_rss_mb"] == 900 and r["build_window_peak_rss_mb"] == 400 for r in res)) + check("merge: provenance", all(r["build_figures_wheel"] == "26.11.1.post1" and r["timed_wheel"] == "26.11.1" for r in res)) + check("merge: not a build-only row", all(kind_of(r) == KIND_CELL for r in res)) + check("merge: newest wins the canonical rule", all(r["ts_utc"] > r["timed_ts_utc"] for r in res)) + # 2. build-only with too few reps + _, _, prob = resolve(live + bo[:2], 3) + check("bo short: problem", len(prob) == 1 and "reps [1, 2] of the 3" in prob[0]) + # 3. a build-only row with nothing underneath + _, _, prob = resolve(bo, 3) + check("bo alone: problem", len(prob) == 1 and "no full cell" in prob[0]) + # 4. a post1 full cell replaces the live rows whole; an incomplete one is a problem, never a mix + full = [_row(i, f"2026-11-1{i}", label="26.11.1.post1", wheel_label="26.11.1.post1") for i in (1, 2, 3)] + res, dec, prob = resolve(live + full, 3) + check("full: replaces", not prob and [r["engine_version"] for r in res] == ["26.11.1.post1"] * 3) + _, _, prob = resolve(live + full[:2], 3) + check("full short: problem, no mix", len(prob) == 1 and "never mixes wheels" in prob[0]) + # 5. the rerun at n=5 against a live set of 3 reps + _, _, prob = resolve(live, 5) + check("live short of n: problem", len(prob) == 1) + # 6. two builds are two cells; comparators and served arms pass through; a failed row is not a rep + other = [_row(i, f"2026-11-0{i}", build="jvm") for i in (1, 2, 3)] + comp = [{"lane": "l1tpc", "backend": "duckdb_embedded", "rep": 1, "ts_utc": "2026-11-01", "engine_version": "1.4.0", "rc": 0}] + srv = [{"lane": "l1tpc", "backend": "arcadedb_server", "rep": 1, "ts_utc": "2026-11-01", "engine_version": "server:26.11.1", "rc": 0}] + res, _, prob = resolve(live + bo + other + comp + srv, 3) + check("builds apart", not prob and len(res) == 3 + 3 + 2) + bad = _row(1, "2026-11-20", error="timeout_after_7200s", rc=1) + res, _, prob = resolve(live + [bad], 3) + check("failed row does not count", not prob and [r["rep"] for r in res if is_arcade_in_process(r) and clean(r)] == [1, 2, 3] and bad in res) + # 7. a build-only row of a backend we do not handle is refused + _, _, prob = resolve([{"lane": "l1tpc", "backend": "duckdb_embedded", ROW_KIND: KIND_BUILD_ONLY, "rep": 1}], 3) + check("foreign build-only", len(prob) == 1) + # 8. the wheel label: stamped only on opt-in, and the candidate JVM wheel (metadata 26.11.1) is told apart by it + r0 = stamp_row({"x": 1}, env={}) + check("stamp: inert by default", r0 == {"x": 1}) + r1 = stamp_row({"engine_version": "26.11.1"}, env={WHEEL_LABEL_ENV: "26.11.1.post1", WHEEL_SHA256_ENV: "AB" * 32, WHEEL_NAME_ENV: "a.whl", BUILD_ONLY_ENV: "1"}) + check("stamp: label", r1["wheel_label"] == "26.11.1.post1" and r1["wheel_sha256"] == "ab" * 32 and r1[ROW_KIND] == KIND_BUILD_ONLY) + check("wheel_of: label wins over metadata", wheel_of(r1) == "26.11.1.post1") + check("wheel_of: engine_version fallback", wheel_of({"engine_version": "26.11.1"}) == "26.11.1") + check("version order", version_key("26.11.1") < version_key("26.11.1.post1") < version_key("26.11.1.post2") and version_key("26.11.1.dev0") < version_key("26.11.1")) + # 10. rollback by wheel label removes exactly the post1-derived rows + import tempfile + tf = tempfile.NamedTemporaryFile("w", suffix=".jsonl", delete=False) + tf.write("".join(json.dumps(r) + "\n" for r in res + comp + full)) + tf.close() + main(["strip", "--label", "26.11.1.post1", tf.name]) + left = load_jsonl(tf.name) + check("strip: post1 rows gone, others kept", comp[0] in left and not any(r.get("wheel_label") == "26.11.1.post1" or r.get("build_figures_wheel") == "26.11.1.post1" for r in left)) + os.unlink(tf.name) + # 9. a cell with a full and a build-only set of one wheel is ambiguous + _, _, prob = resolve(live + full + bo, 3) + check("both kinds one wheel", len(prob) == 1) + for f in fails: + print("FAIL", f) + print(f"post1_rows selftest: {'FAILED' if fails else 'ok'} ({len(fails)} failure(s))") + return 1 if fails else 0 + + +def main(argv=None): + ap = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) + sub = ap.add_subparsers(dest="cmd") + sub.add_parser("selftest") + rp = sub.add_parser("resolve") + rp.add_argument("--reps", type=int, default=3) + rp.add_argument("--out", required=True) + rp.add_argument("files", nargs="+") + sp = sub.add_parser("strip", help="ROLLBACK: drop every row that carries a wheel label (a full post1 row, or a cell merged from build-only rows)") + sp.add_argument("--label", required=True) + sp.add_argument("file") + ap.add_argument("--selftest", action="store_true") + a = ap.parse_args(argv) + if a.selftest or a.cmd == "selftest": + return selftest() + if a.cmd == "resolve": + rows = [r for f in a.files for r in load_jsonl(f)] + res, dec, prob = resolve(rows, a.reps) + for d in dec: + print(" " + d) + for p in prob: + print(" PROBLEM " + p) + if prob: + print(f"REFUSING: {len(prob)} cell(s) cannot be resolved; nothing written") + return 1 + with open(a.out + ".new", "w", encoding="utf-8") as fh: + fh.write("".join(json.dumps(r) + "\n" for r in res)) + os.replace(a.out + ".new", a.out) + print(f"{len(res)} row(s) written to {a.out}") + return 0 + if a.cmd == "strip": + rows = load_jsonl(a.file) + keep = [r for r in rows if r.get("wheel_label") != a.label and r.get("build_figures_wheel") != a.label] + with open(a.file + ".new", "w", encoding="utf-8") as fh: + fh.write("".join(json.dumps(r) + "\n" for r in keep)) + os.replace(a.file + ".new", a.file) + print(f"{len(rows) - len(keep)} row(s) of wheel {a.label} removed from {a.file}, {len(keep)} kept; land the live rows file again to restore the replaced rows") + return 0 + ap.print_help() + return 2 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/benchmarks/experiments/runner.py b/benchmarks/experiments/runner.py index d51d2700929..4b710d125cc 100644 --- a/benchmarks/experiments/runner.py +++ b/benchmarks/experiments/runner.py @@ -46,6 +46,7 @@ import bench_common # noqa: E402 (INSTRUMENT, stamped on every row) import memory_probe # noqa: E402 (memory per phase, instrument 2026-11 only) import vendor_memory # noqa: E402 (Neo4j's and Elasticsearch's own memory rules, DECISIONS #202) +import post1_rows # noqa: E402 (build-only mode and the wheel label of a wheels-only rerun; inert unless the stage opts in) DATA = os.path.abspath(os.environ.get("BENCH_DATA", os.path.join(HERE, "data"))) RESULTS = os.path.join(HERE, "results") RAW = os.path.join(RESULTS, "raw") @@ -3554,7 +3555,9 @@ def run_cell(job, rep, scale, cpuset, tier, net_name): # the sparse lane's warm-up count (row 81, DECISIONS #177); the lane reads it in the container "BENCH_SPARSE_WARMUP", # the served ArcadeDB time-series arms read query answers as NDJSON, OFF unless set (row 85) - "BENCH_ARCADEDB_TS_NDJSON"): + "BENCH_ARCADEDB_TS_NDJSON", + # build-only mode (post1_rows.py): the lane stops after build(); forwarded only when the launcher set it + "BENCH_BUILD_ONLY"): if os.environ.get(_k): bench_env += ["-e", f"{_k}={os.environ[_k]}"] @@ -4264,9 +4267,20 @@ def main(): "rep), so an N=1 smoke at a paper scale silently " "supersedes rep 1 of a finished cell and the table " "then prints one smoke row beside four real ones.") + ap.add_argument("--build-only", action="store_true", + help="stop each cell after build() and record only the load and index figures, with row_kind=build_only (post1_rows.py). " + "Needs a results file named like runs_page__2611_post.jsonl: a build-only row is not a cell and must never " + "sit in the pass's own rows file") args = ap.parse_args() if os.sep in args.results_file or args.results_file.startswith("."): ap.error("--results-file is a bare filename under results/") + if args.build_only: + if args.driver: + ap.error("--build-only and --driver exclude each other: an overlay driver times queries, there is nothing to stop after") + if not re.search(r"_post\d+\.jsonl$", args.results_file) and os.environ.get("BENCH_ALLOW_BUILD_ONLY_ANY_FILE") != "1": + ap.error("--build-only writes build-only rows, which are not cells: the results file must end in _post.jsonl " + "(runs_page__2611_post1.jsonl), never the pass's own rows file") + os.environ["BENCH_BUILD_ONLY"] = "1" if args.mem: # Paired with --heap: raising the heap without raising the cap puts the @@ -4469,6 +4483,7 @@ def worker(shard): row = run_cell(job, rep, args.scale, shard, args.tier, net_name) _hold.stamp(row) precell_gate_stamp(row) + post1_rows.stamp_row(row) # adds nothing unless BENCH_WHEEL_LABEL / BENCH_BUILD_ONLY is set (post1_rows.py) except Exception: with cv: raised.append(f"{job['lane']}_{job['backend']}_r{rep}") diff --git a/benchmarks/experiments/test_post1_rerun.py b/benchmarks/experiments/test_post1_rerun.py new file mode 100644 index 00000000000..bca90e78d90 --- /dev/null +++ b/benchmarks/experiments/test_post1_rerun.py @@ -0,0 +1,39 @@ +"""Selftests of the post1 rerun tooling (post1_rows.py, make_post1_rerun_stages.py). Pure Python: no docker, no JVM, no timing. +Run as `uv run --no-project python test_post1_rerun.py`, or under pytest.""" +import os +import sys + +sys.path.insert(0, os.path.dirname(os.path.abspath(__file__))) + + +def test_post1_rows_selftest(): + import post1_rows + assert post1_rows.selftest() == 0 + + +def test_stage_generator_selftest(): + import make_post1_rerun_stages + assert make_post1_rerun_stages.selftest() == 0 + + +def test_build_only_is_off_by_default_in_the_lanes_and_the_runner(): + here = os.path.dirname(os.path.abspath(__file__)) + for lane in ("l1_tpc.py", "l2_graph.py", "e2_hybrid.py", "l3d_dense.py"): + src = open(os.path.join(here, lane), encoding="utf-8").read() + assert src.count('os.environ.get("BENCH_BUILD_ONLY") == "1"') == 1, lane + assert src.count("post1_rows.finish_build_only(") == 1, lane + runner = open(os.path.join(here, "runner.py"), encoding="utf-8").read() + assert 'os.environ["BENCH_BUILD_ONLY"] = "1"' in runner and runner.count("if args.build_only:") == 1 + + +if __name__ == "__main__": + rc = 0 + for name, fn in sorted(globals().items()): + if name.startswith("test_") and callable(fn): + try: + fn() + print("ok ", name) + except AssertionError as e: + print("FAIL", name, e) + rc = 1 + sys.exit(rc)