From d3d1af6b4a17a33704d9bc42ab8ea81ab0f0a1b3 Mon Sep 17 00:00:00 2001 From: Taewoon Kim Date: Thu, 29 Oct 2026 19:17:24 +0900 Subject: [PATCH 01/23] benchmarks: import arcadedb (native and arcadedb-jvm), fall back to arcadedb_embedded 26.11.1 ships `arcadedb` (native) and `arcadedb-jvm`, both with import name `arcadedb`; neither has an `arcadedb_embedded` module. One compat module (`_arcadedb_import.py`) imports `arcadedb` first and falls back to `arcadedb_embedded`, and the lanes (l1_tpc, l1_tabular, l2_graph, l3_sparse, l3d_dense, l4_tsbs, l5_lifecycle, e2_hybrid, e3_recovery, e4_decomp and the e4 probe) call it instead of importing the old name. The engine_version stamp (bench_common.run_conditions and the lane adapters) now reads the installed distribution through the same module, new names first, so a new install no longer stamps "unknown (PackageNotFoundError)". The l5 cold-start subprocess resolves the import name outside its timed window, so its two import lines are unchanged statements on the old wheels. No measured code path, timing, cell definition, backend key or comparator code changes. Backend keys such as `arcadedb_embedded` stay row keys. Closes #553 Co-Authored-By: Claude Opus 5.5 --- benchmarks/experiments/_arcadedb_import.py | 102 ++++++++++++++++++ benchmarks/experiments/bench_common.py | 4 +- .../experiments/deployment_decomp_probe.py | 5 +- benchmarks/experiments/e2_hybrid.py | 6 +- benchmarks/experiments/e3_recovery.py | 8 +- benchmarks/experiments/e4_decomp.py | 4 +- benchmarks/experiments/l1_tabular.py | 6 +- benchmarks/experiments/l1_tpc.py | 6 +- benchmarks/experiments/l2_graph.py | 6 +- benchmarks/experiments/l3_sparse.py | 9 +- benchmarks/experiments/l3d_dense.py | 6 +- benchmarks/experiments/l4_tsbs.py | 3 +- benchmarks/experiments/l5_lifecycle.py | 18 ++-- .../experiments/test_arcadedb_import.py | 91 ++++++++++++++++ benchmarks/experiments/test_e4_client_axis.py | 4 + benchmarks/experiments/test_lean_http_live.py | 6 +- 16 files changed, 250 insertions(+), 34 deletions(-) create mode 100644 benchmarks/experiments/_arcadedb_import.py create mode 100644 benchmarks/experiments/test_arcadedb_import.py diff --git a/benchmarks/experiments/_arcadedb_import.py b/benchmarks/experiments/_arcadedb_import.py new file mode 100644 index 00000000000..a6093d0e8c3 --- /dev/null +++ b/benchmarks/experiments/_arcadedb_import.py @@ -0,0 +1,102 @@ +"""The one place the harness decides which ArcadeDB Python package it imports. + +Since 26.11.1 the Python bindings ship as two distributions that both install the import name `arcadedb`: +`arcadedb` (the native build) and `arcadedb-jvm` (the Java engine through JPype). Before that the one +distribution was `arcadedb-embedded`, import name `arcadedb_embedded`. The lanes used to write +`import arcadedb_embedded as arcadedb` in every ArcadeDB arm; they now call `load()` here instead: + + import _arcadedb_import + arcadedb = _arcadedb_import.load() + +`load()` imports `arcadedb` first and falls back to `arcadedb_embedded`, so the same harness runs +against the new wheels and against the `arcadedb_embedded` wheels the images carry today. An +`arcadedb_embedded` install resolves to the old package unless `arcadedb` is installed too (the +26.11.1 `arcadedb-embedded` shim re-exports `arcadedb`, so there both names give the same module). + +Everything here is lazy: importing this file imports nothing from ArcadeDB, because a lane module is +also imported by cells that never touch ArcadeDB (DuckDB, SQLite, ...), on hosts without the wheel. + +What differs between the packages, and what this module hides: + + * module name: `arcadedb` vs `arcadedb_embedded` (module_name()). + * distribution name for the version stamp: `arcadedb` / `arcadedb-jvm` / `arcadedb-embedded` + (dist_version(); `importlib.metadata.version("arcadedb-embedded")` is a PackageNotFoundError on a new install). + * which engine runs: `arcadedb.__backend__` is "native" or "jvm"; the old package has no such + attribute and is always "jvm" (backend()). + * `.jvm`: importable in both new wheels, but the native wheel ships no JARs, so + `start_jvm()` raises ArcadeDBError ("No JAR files found") there (jvm_module()). The cold-start + subprocess in l5_lifecycle.py calls it, so its `jvm_start_ms` / `import_ms` / `first_open_ms` + columns are null on the native build (checked 2026-10-29 with arcadedb 26.11.1.dev0). + * JVM-only APIs (`get_java_database`, `jpype`, Java arrays): they exist only on the JVM build and + the native build has no JPype (l3_sparse's embedded arm imports it and fails there). A lane arm + that needs one must check backend() first; this module does not paper over it. +""" +import importlib +import importlib.metadata + +# Import names, in order. The first one that is the real package wins. +MODULES = ("arcadedb", "arcadedb_embedded") +# Distribution names for the version stamp, same order idea: new first. +DISTS = ("arcadedb", "arcadedb-jvm", "arcadedb-embedded") + +_loaded = None + + +def _is_engine(mod): + # `arcadedb` could also resolve as an empty namespace directory on sys.path; require the API. + return hasattr(mod, "create_database") and hasattr(mod, "DatabaseFactory") + + +def load(): + """Return the ArcadeDB Python module: `arcadedb`, else `arcadedb_embedded`. + + Raises ImportError (naming both) when neither is installed. + """ + global _loaded + if _loaded is not None: + return _loaded + errors = [] + for name in MODULES: + try: + mod = importlib.import_module(name) + except ImportError as e: + errors.append(f"{name}: {e}") + continue + if _is_engine(mod): + _loaded = mod + return mod + errors.append(f"{name}: imported but is not the ArcadeDB package ({getattr(mod, '__file__', None)})") + raise ImportError("no ArcadeDB Python package: install arcadedb or arcadedb-jvm (or the older arcadedb-embedded); " + + "; ".join(errors)) + + +def module_name(): + """The import name `load()` resolved to ("arcadedb" or "arcadedb_embedded"). For code strings run in a subprocess.""" + return load().__name__ + + +def backend(): + """"native" or "jvm": the engine this process runs. The old `arcadedb_embedded` package is always "jvm".""" + mod = load() + return getattr(mod, "__backend__", None) or getattr(mod, "_DISTRIBUTION_BACKEND", None) or "jvm" + + +def jvm_module(): + """`.jvm` (start_jvm, shutdown_jvm, jar_fingerprint ...). Importing the submodule does not start a JVM.""" + return importlib.import_module(module_name() + ".jvm") + + +def dist_version(): + """The installed distribution's version, read from package metadata without importing the engine. + + Same contract as the old `importlib.metadata.version("arcadedb-embedded")`: returns a version string, + or raises PackageNotFoundError when none of the distributions is installed (callers format that into + "unknown (PackageNotFoundError)" exactly as before). + """ + last = None + for dist in DISTS: + try: + return importlib.metadata.version(dist) + except importlib.metadata.PackageNotFoundError as e: + last = e + raise last diff --git a/benchmarks/experiments/bench_common.py b/benchmarks/experiments/bench_common.py index 61a24b3d7ae..521c22c95b1 100644 --- a/benchmarks/experiments/bench_common.py +++ b/benchmarks/experiments/bench_common.py @@ -908,8 +908,8 @@ def _read(path): "ts_utc": time.strftime("%Y-%m-%dT%H:%M:%S+00:00", time.gmtime()), } try: - from importlib.metadata import version as _v - out["engine_version"] = _v("arcadedb-embedded") + import _arcadedb_import + out["engine_version"] = _arcadedb_import.dist_version() except Exception as e: out["engine_version"] = f"unknown ({e.__class__.__name__})" # THE JPYPE THIS PROCESS RAN (CAMPAIGN 7 row 73): empty unless the process imported it. diff --git a/benchmarks/experiments/deployment_decomp_probe.py b/benchmarks/experiments/deployment_decomp_probe.py index 65b3fe58870..5f5de46185f 100644 --- a/benchmarks/experiments/deployment_decomp_probe.py +++ b/benchmarks/experiments/deployment_decomp_probe.py @@ -324,7 +324,8 @@ def main() -> int: args = ap.parse_args() import requests - import arcadedb_embedded as arcadedb + import _arcadedb_import + arcadedb = _arcadedb_import.load() results: dict[str, dict] = {} meta = { @@ -354,7 +355,7 @@ def main() -> int: # has to be pinned on the JVM before the server exists. Same -Xms=-Xmx # policy as every other lane: fixed heap for latency parity, with the # working set reported separately from cgroup anon. - from arcadedb_embedded.jvm import start_jvm + start_jvm = _arcadedb_import.jvm_module().start_jvm start_jvm(heap_size=args.heap, jvm_args=[f"-Xms{args.heap}"]) # Arms 1 and 2 share this process, so they share the JVM, heap, GC, engine diff --git a/benchmarks/experiments/e2_hybrid.py b/benchmarks/experiments/e2_hybrid.py index afcff8881ba..0f5131252da 100644 --- a/benchmarks/experiments/e2_hybrid.py +++ b/benchmarks/experiments/e2_hybrid.py @@ -236,7 +236,8 @@ def index_readback(self): TXN_SCOPE = "whole" def __init__(self): - import arcadedb_embedded as arcadedb + import _arcadedb_import + arcadedb = _arcadedb_import.load() self._a = arcadedb heap = os.environ.get("ARCADEDB_HEAP", "4g") self.db = arcadedb.create_database( @@ -247,8 +248,7 @@ def __init__(self): "jvm_args": bench_common.arcade_jvm_args()}) # THE WHEEL'S version, not its name (#156). A row stamped # "arcadedb-embedded" cannot be re-measured by anyone including us. - from importlib.metadata import version as _v - self.version = _v("arcadedb-embedded") + self.version = _arcadedb_import.dist_version() self.durability = bench_common.arcade_durability_readback() def build(self, vecs, edges): diff --git a/benchmarks/experiments/e3_recovery.py b/benchmarks/experiments/e3_recovery.py index 2148aa5783c..d565c08c57a 100644 --- a/benchmarks/experiments/e3_recovery.py +++ b/benchmarks/experiments/e3_recovery.py @@ -20,7 +20,9 @@ import sys import time -import arcadedb_embedded as arcadedb +import _arcadedb_import + +arcadedb = _arcadedb_import.load() DB = "/data-local/e3_arcade" ROWS_PER_TX = 10 @@ -70,8 +72,8 @@ def verify(): # numbers are right, but nothing can re-derive them. Fixed here so the # freeze re-run leaves evidence. try: - from importlib.metadata import version as _pkgver - out["engine_version"] = _pkgver("arcadedb-embedded") + import _arcadedb_import as _ai + out["engine_version"] = _ai.dist_version() except Exception as e: out["engine_version"] = f"unknown ({e.__class__.__name__})" outp = os.environ.get("PROBE_OUT", "") diff --git a/benchmarks/experiments/e4_decomp.py b/benchmarks/experiments/e4_decomp.py index 206c0fe8914..5f730b5a95a 100644 --- a/benchmarks/experiments/e4_decomp.py +++ b/benchmarks/experiments/e4_decomp.py @@ -107,8 +107,8 @@ def main(): meta = payload.get("meta", {}) # The runner's row: what the lane measured, in fields a reader of runs.jsonl # expects; the full artifact stays in the file. - from importlib.metadata import version as _v - payload["engine_version"] = _v("arcadedb-embedded") + import _arcadedb_import + payload["engine_version"] = _arcadedb_import.dist_version() # THE DURABILITY THE ARM RAN AT, read out of the engine rather than # asserted (#81, #90). Every 2026-10 row must record it -- F10 fails a row # that does not -- and this lane recorded nothing, which nobody noticed diff --git a/benchmarks/experiments/l1_tabular.py b/benchmarks/experiments/l1_tabular.py index dd090a38bff..168d48cbed5 100644 --- a/benchmarks/experiments/l1_tabular.py +++ b/benchmarks/experiments/l1_tabular.py @@ -504,7 +504,8 @@ def index_readback(self): lambda: self.db.query("sql", "SELECT FROM schema:indexes").to_list()) def connect(self): - import arcadedb_embedded as arcadedb + import _arcadedb_import + arcadedb = _arcadedb_import.load() import bench_common # this retired lane imports nothing else from it heap = os.environ.get("ARCADEDB_HEAP", "4g") # -Xms is left unset; heap_size sets -Xmx from the cell's memory budget (the bindings' documented practice, PROTOCOL section 7). @@ -538,7 +539,8 @@ def reopen(self): that reopened without it would be a different operating point wearing the same label. """ - import arcadedb_embedded as arcadedb + import _arcadedb_import + arcadedb = _arcadedb_import.load() import bench_common # this retired lane imports nothing else from it heap = os.environ.get("ARCADEDB_HEAP", "4g") self.db = arcadedb.open_database( diff --git a/benchmarks/experiments/l1_tpc.py b/benchmarks/experiments/l1_tpc.py index 9159f7f64ed..130762b5bfd 100644 --- a/benchmarks/experiments/l1_tpc.py +++ b/benchmarks/experiments/l1_tpc.py @@ -1056,7 +1056,8 @@ def index_readback(self): durability = bench_common.DURABILITY_ARCADEDB def connect(self): - import arcadedb_embedded as arcadedb + import _arcadedb_import + arcadedb = _arcadedb_import.load() self._a = arcadedb heap = os.environ.get("ARCADEDB_HEAP", "4g") self.db = arcadedb.create_database("/tmp/tpc_arcade", @@ -1067,8 +1068,7 @@ def connect(self): # than a default someone remembered. jvm_kwargs={"heap_size": heap, "jvm_args": bench_common.arcade_jvm_args()}) - from importlib.metadata import version as _pv - self.version = _pv("arcadedb-embedded") + self.version = _arcadedb_import.dist_version() # ASKED, not asserted (#81's standard, doubled by #90's second class). self.durability = bench_common.arcade_durability_readback() qr = bench_common.arcade_query_ram_readback() # {} unless the query heap budget override is on (DECISIONS #175) diff --git a/benchmarks/experiments/l2_graph.py b/benchmarks/experiments/l2_graph.py index e6c955d7cca..9e4eed3ca57 100644 --- a/benchmarks/experiments/l2_graph.py +++ b/benchmarks/experiments/l2_graph.py @@ -198,7 +198,8 @@ def index_readback(self): lambda: self.db.query("sql", "SELECT FROM schema:indexes").to_list()) def connect(self): - import arcadedb_embedded as arcadedb + import _arcadedb_import + arcadedb = _arcadedb_import.load() heap = os.environ.get("ARCADEDB_HEAP", "4g") # -Xms is left unset; heap_size sets -Xmx from the cell's memory budget (the bindings' documented practice, PROTOCOL section 7). self.db = arcadedb.create_database( @@ -228,7 +229,8 @@ def reopen(self): JVM engine and on none of the others, so it has to be measured separately rather than folded in. """ - import arcadedb_embedded as arcadedb + import _arcadedb_import + arcadedb = _arcadedb_import.load() heap = os.environ.get("ARCADEDB_HEAP", "4g") self.db = arcadedb.open_database( "/tmp/l2_arcade", diff --git a/benchmarks/experiments/l3_sparse.py b/benchmarks/experiments/l3_sparse.py index 2949136e46c..01f800b0d3e 100644 --- a/benchmarks/experiments/l3_sparse.py +++ b/benchmarks/experiments/l3_sparse.py @@ -175,7 +175,8 @@ def _index_metadata(self): return _json.dumps(meta) def connect(self): - import arcadedb_embedded as arcadedb + import _arcadedb_import + arcadedb = _arcadedb_import.load() heap = os.environ.get("ARCADEDB_HEAP", "4g") # -Xms is left unset; heap_size sets -Xmx from the cell's memory budget (the bindings' documented practice, PROTOCOL section 7). jvm_args = "" @@ -190,8 +191,7 @@ def connect(self): self.db = arcadedb.create_database( "/tmp/l3_arcade", jvm_kwargs={"heap_size": heap, "jvm_args": jvm_args or None}) - from importlib.metadata import version as _pv - self.version = _pv("arcadedb-embedded") + self.version = _arcadedb_import.dist_version() self.db.command("sql", "CREATE DOCUMENT TYPE Doc") self.db.command("sql", "CREATE PROPERTY Doc.id LONG") self.db.command("sql", "CREATE PROPERTY Doc.tokens ARRAY_OF_INTEGERS") @@ -212,7 +212,8 @@ def reopen(self): the point, since it is charged to every JVM engine and to none of the others. """ - import arcadedb_embedded as arcadedb + import _arcadedb_import + arcadedb = _arcadedb_import.load() heap = os.environ.get("ARCADEDB_HEAP", "4g") self.db = arcadedb.open_database( "/tmp/l3_arcade", diff --git a/benchmarks/experiments/l3d_dense.py b/benchmarks/experiments/l3d_dense.py index 5d06b46e72c..2f6597b01a9 100644 --- a/benchmarks/experiments/l3d_dense.py +++ b/benchmarks/experiments/l3d_dense.py @@ -584,7 +584,8 @@ def quantization(self): return canonical_quant_label(os.environ.get("BENCH_DENSE_QUANT", "")) def connect(self): - import arcadedb_embedded as arcadedb + import _arcadedb_import + arcadedb = _arcadedb_import.load() self._a = arcadedb heap = os.environ.get("ARCADEDB_HEAP", "4g") # ARCADEDB_EXTRA_JVM_ARGS: space-separated extras (e.g. an @@ -649,8 +650,7 @@ def connect(self): "/tmp/l3d_arcade", jvm_kwargs={"heap_size": heap, "jvm_args": extra.strip() or None}) - from importlib.metadata import version as _pv - self.version = _pv("arcadedb-embedded") + self.version = _arcadedb_import.dist_version() def build(self, vecs): db = self.db diff --git a/benchmarks/experiments/l4_tsbs.py b/benchmarks/experiments/l4_tsbs.py index de5fe19ceff..1955dc002f4 100644 --- a/benchmarks/experiments/l4_tsbs.py +++ b/benchmarks/experiments/l4_tsbs.py @@ -274,7 +274,8 @@ class ArcadeTS(BulkRead): name = "arcadedb_ts_doc" def connect(self): - import arcadedb_embedded as arcadedb + import _arcadedb_import + arcadedb = _arcadedb_import.load() self._arcadedb = arcadedb heap = os.environ.get("ARCADEDB_HEAP", "6g") self.db = arcadedb.create_database("/tmp/l4_arcade", diff --git a/benchmarks/experiments/l5_lifecycle.py b/benchmarks/experiments/l5_lifecycle.py index 3089cd3f01e..3ffac29d9e6 100644 --- a/benchmarks/experiments/l5_lifecycle.py +++ b/benchmarks/experiments/l5_lifecycle.py @@ -116,7 +116,8 @@ def build(situation, n, heap): Returns the build session's close time in ms. """ - import arcadedb_embedded as arcadedb + import _arcadedb_import + arcadedb = _arcadedb_import.load() _rm(DB) db = arcadedb.create_database( DB, jvm_kwargs={"heap_size": heap, "jvm_args": None}) @@ -474,7 +475,8 @@ def _drop(db, situation): def cycle(situation, mode, cold=False): """One open/close cycle. Returns (open_ms, close_ms).""" - import arcadedb_embedded as arcadedb + import _arcadedb_import + arcadedb = _arcadedb_import.load() if cold: # Verified eviction: evict() raises if pages survive, so a cold number # cannot be reported for a database that was still cached. @@ -640,11 +642,15 @@ def main(): # # What a cold caller actually pays is the whole process: interpreter start, import, JPype's JVM # boot, then the open. That is what this subprocess times, from its own first line. + # The import name is resolved HERE, outside the timed window (@MOD@ below), so the two import lines + # are the same statements as before (`arcadedb_embedded` on the old wheels, `arcadedb` on the new + # ones) and import_ms still times nothing else. + import _arcadedb_import _boot = subprocess.run( - [sys.executable, "-c", + [sys.executable, "-c", ( "import time; _t0 = time.perf_counter()\n" - "from arcadedb_embedded.jvm import start_jvm\n" - "import arcadedb_embedded as a\n" + "from @MOD@.jvm import start_jvm\n" + "import @MOD@ as a\n" "_ti = time.perf_counter()\n" "start_jvm()\n" "_tj = time.perf_counter()\n" @@ -653,7 +659,7 @@ def main(): "db.close()\n" "print('%%.3f %%.3f %%.3f %%.3f' %% ((_ti-_t0)*1000, (_tj-_ti)*1000, (_to-_tj)*1000, " "(time.perf_counter()-_t0)*1000))" - % DB], + % DB).replace("@MOD@", _arcadedb_import.module_name())], capture_output=True, text=True, timeout=900) if _boot.returncode == 0 and _boot.stdout.strip(): import_ms, jvm_start_ms, first_open_ms, cold_proc_ms = ( diff --git a/benchmarks/experiments/test_arcadedb_import.py b/benchmarks/experiments/test_arcadedb_import.py new file mode 100644 index 00000000000..e07ff236cd4 --- /dev/null +++ b/benchmarks/experiments/test_arcadedb_import.py @@ -0,0 +1,91 @@ +"""_arcadedb_import: one import path for the `arcadedb` (26.11.1+) and `arcadedb_embedded` (older) packages. + +The harness must run on the wheels the images carry today (`arcadedb_embedded`) and on the new ones (`arcadedb`, `arcadedb-jvm`), so +the order (new first, old as the fallback), the failure when neither exists, the native/jvm answer and the version lookup are pinned here. +Run: python -m pytest -q test_arcadedb_import.py +""" +import importlib.metadata +import sys +import types +from pathlib import Path + +import pytest + +sys.path.insert(0, str(Path(__file__).resolve().parent)) +import _arcadedb_import as AI # noqa: E402 + + +def _fake(name, **attrs): + m = types.ModuleType(name) + m.create_database = lambda *a, **k: None + m.DatabaseFactory = object + for k, v in attrs.items(): + setattr(m, k, v) + m.jvm = types.ModuleType(name + ".jvm") + return m + + +@pytest.fixture +def reset(monkeypatch): + monkeypatch.setattr(AI, "_loaded", None) + + def install(**mods): + for n in ("arcadedb", "arcadedb.jvm", "arcadedb_embedded", "arcadedb_embedded.jvm"): + monkeypatch.setitem(sys.modules, n, None) # None in sys.modules makes the import raise ImportError + for n, m in mods.items(): + monkeypatch.setitem(sys.modules, n, m) + monkeypatch.setitem(sys.modules, n + ".jvm", m.jvm) + return install + + +def test_new_name_wins_when_both_exist(reset): + new, old = _fake("arcadedb", __backend__="jvm"), _fake("arcadedb_embedded") + reset(arcadedb=new, arcadedb_embedded=old) + assert AI.load() is new and AI.module_name() == "arcadedb" + + +def test_falls_back_to_the_old_package(reset): + old = _fake("arcadedb_embedded") + reset(arcadedb_embedded=old) + assert AI.load() is old and AI.module_name() == "arcadedb_embedded" + assert AI.backend() == "jvm" # the old package has no __backend__ and is always the JVM build + assert AI.jvm_module() is old.jvm + + +def test_a_namespace_directory_called_arcadedb_is_not_the_engine(reset): + empty = types.ModuleType("arcadedb") # what an empty `arcadedb/` directory on sys.path imports as + empty.jvm = types.ModuleType("arcadedb.jvm") + old = _fake("arcadedb_embedded") + reset(arcadedb_embedded=old) + sys.modules["arcadedb"] = empty + assert AI.load() is old + + +def test_neither_installed_says_what_to_install(reset): + reset() + with pytest.raises(ImportError, match="arcadedb-jvm"): + AI.load() + + +@pytest.mark.parametrize("backend", ["native", "jvm"]) +def test_backend_is_what_the_new_package_says_it_runs(reset, backend): + reset(arcadedb=_fake("arcadedb", __backend__=backend, _DISTRIBUTION_BACKEND="jvm")) + assert AI.backend() == backend + + +def test_dist_version_tries_the_new_names_first_and_keeps_the_old_failure(monkeypatch): + have = {"arcadedb-jvm": "26.11.1.dev0"} + + def version(dist): + if dist in have: + return have[dist] + raise importlib.metadata.PackageNotFoundError(dist) + + monkeypatch.setattr(importlib.metadata, "version", version) + assert AI.dist_version() == "26.11.1.dev0" + have.clear() + have["arcadedb-embedded"] = "26.10.2" + assert AI.dist_version() == "26.10.2" + have.clear() + with pytest.raises(importlib.metadata.PackageNotFoundError): # callers format it as "unknown (PackageNotFoundError)" + AI.dist_version() diff --git a/benchmarks/experiments/test_e4_client_axis.py b/benchmarks/experiments/test_e4_client_axis.py index 65e20e84c01..91eaac5dc9a 100644 --- a/benchmarks/experiments/test_e4_client_axis.py +++ b/benchmarks/experiments/test_e4_client_axis.py @@ -234,6 +234,10 @@ def run_probe(monkeypatch, tmp_path, probe): fake.jvm = jvm monkeypatch.setitem(sys.modules, "arcadedb_embedded", fake) monkeypatch.setitem(sys.modules, "arcadedb_embedded.jvm", jvm) + # the probe resolves the package through _arcadedb_import, which prefers `arcadedb` when a new wheel is installed: + # pin the stub there so the test never reaches a real engine + import _arcadedb_import + monkeypatch.setattr(_arcadedb_import, "_loaded", fake) monkeypatch.setattr(probe, "SIZES", [1, 10, 100]) monkeypatch.setattr(probe, "ROWS", 300) monkeypatch.setattr(probe, "REPS", 4) diff --git a/benchmarks/experiments/test_lean_http_live.py b/benchmarks/experiments/test_lean_http_live.py index 6a77aa11a50..d8dc3204527 100644 --- a/benchmarks/experiments/test_lean_http_live.py +++ b/benchmarks/experiments/test_lean_http_live.py @@ -403,7 +403,11 @@ def test_l5_lifecycle_server_read_identical(env, workload, tmp_path): def test_e4_decomposition_measures_both_clients_and_every_path_returns_the_same_rows(legacy, tmp_path): """The e4 lane (e4_decomp.py, with a real in-process engine and a real server) at a small corpus, once with requests importable and once with it blocked, which is the e4 cell's own condition (its image has none, so the probe's client is the urllib shim).""" - pytest.importorskip("arcadedb_embedded") + import _arcadedb_import + try: + _arcadedb_import.load() + except ImportError: + pytest.skip("no ArcadeDB Python package (arcadedb or arcadedb_embedded)") host, port = _host_port() _admin("drop database deploy_decomp", ok_missing=True) out = tmp_path / f"e4-{legacy}.json" From 1921aa94bf62bcface9c47b4c295050306ffab56 Mon Sep 17 00:00:00 2001 From: Taewoon Kim Date: Fri, 30 Oct 2026 02:36:07 +0900 Subject: [PATCH 02/23] benchmarks: native paths for l3_sparse and l5, engine_backend row field, pin and stage tooling on the new package names l3_sparse embedded arm: one code path for both packages. Ingest through Database.insert_many with the generators' lists, settle through SQL COMPACT INDEX (the statement the served arm already ran), query parameters as plain Python lists (the JVM build converts any numpy array to float[], which the indices argument refuses). No JPype, no get_java_database. l5_lifecycle: the cold-start subprocess is built per backend (_cold_start_code): the native build times the library load and isolate creation as engine_load_ms where the JVM build times start_jvm(); jvm_start_ms stays JVM-only and is null on native. A second fresh process (_cold_query_code) adds cold_first_query_ms and cold_query_process_ms on both builds. BENCH_LC_DB moves the database directory for a laptop smoke (default unchanged). Rows: engine_backend ("native"/"jvm", arcadedb.__backend__) in bench_common.run_conditions and in every lane that sets engine_version by hand; null when the process never loaded ArcadeDB. Named engine_backend because `backend` is already the arm name. Tooling: engine_pin reads arcadedb / arcadedb-jvm / arcadedb-embedded (native engine version from _build_info), wheel_guard and queue_lint accept the new wheel names, Dockerfile.bench installs JPype only for JVM wheels, build_images.sh, verify_pair_c25.sh, make_2611_stages.py and jvm_payload_check.py follow. smoke_backend_rows.py checks a smoke row and the stored array types. Co-Authored-By: Claude Sonnet 5.5 --- benchmarks/experiments/Dockerfile.bench | 13 +- benchmarks/experiments/PAGE-SPEC.md | 2 +- benchmarks/experiments/READING-RESULTS.md | 2 + benchmarks/experiments/_arcadedb_import.py | 18 ++- benchmarks/experiments/bench_common.py | 13 ++ benchmarks/experiments/build_images.sh | 5 +- benchmarks/experiments/e2_hybrid.py | 1 + benchmarks/experiments/e3_recovery.py | 2 + benchmarks/experiments/e4_decomp.py | 1 + benchmarks/experiments/engine_pin.py | 54 ++++++-- benchmarks/experiments/jvm_payload_check.py | 2 +- benchmarks/experiments/l1_tabular.py | 2 + benchmarks/experiments/l1_tpc.py | 1 + benchmarks/experiments/l2_graph.py | 1 + benchmarks/experiments/l3_sparse.py | 71 ++++------ benchmarks/experiments/l3d_dense.py | 1 + benchmarks/experiments/l5_lifecycle.py | 124 ++++++++++++++--- benchmarks/experiments/l6_restart.py | 1 + benchmarks/experiments/make_2611_stages.py | 11 +- benchmarks/experiments/queue_lint.py | 4 +- benchmarks/experiments/smoke_backend_rows.py | 125 ++++++++++++++++++ .../experiments/stage_drafts_2611.sha256 | 76 +++++------ .../experiments/test_arcadedb_import.py | 8 ++ benchmarks/experiments/test_engine_pin.py | 33 ++++- benchmarks/experiments/test_jpype_pin.py | 13 ++ benchmarks/experiments/test_queue_lint.py | 7 + benchmarks/experiments/verify_pair_c25.sh | 11 +- benchmarks/experiments/wheel_guard.py | 17 ++- 28 files changed, 480 insertions(+), 139 deletions(-) create mode 100644 benchmarks/experiments/smoke_backend_rows.py diff --git a/benchmarks/experiments/Dockerfile.bench b/benchmarks/experiments/Dockerfile.bench index 4ed1a3fb3be..597e352a461 100644 --- a/benchmarks/experiments/Dockerfile.bench +++ b/benchmarks/experiments/Dockerfile.bench @@ -1,6 +1,6 @@ # Generic benchmark image. Pin the package set via build ARG so each backend # (arcadedb, duckdb, ladybug, chroma, faiss, ...) gets its own pinned image later. -# arcadedb-embedded bundles its own JRE, so no Java install is needed. +# arcadedb-jvm (and the older arcadedb-embedded) bundle their own JRE, so no Java install is needed; the native `arcadedb` wheel has no JVM. FROM python:3.12-slim ARG PIP_PACKAGES="" @@ -12,16 +12,21 @@ ARG PIP_PACKAGES="" # default is bench_common.JPYPE_PIN; `make_2611_stages.py --check` refuses a # default that differs, and every stage reads the version back out of the image. ARG JPYPE_VERSION=1.7.1 -# Local wheels (e.g. a not-yet-released arcadedb-embedded) placed in +# NO JPYPE IN A NATIVE IMAGE (26.11.1): the `arcadedb` wheel is the native build, no JVM and no JPype, and its rows must say so +# (bench_common.jpype_fields reads the JPype the process imported). JPype is installed with `arcadedb-jvm`, `arcadedb_jvm-*.whl`, +# `arcadedb-embedded` and `arcadedb_embedded-*.whl`; a wheel named `arcadedb-...` (the native one) gets none. +# Local wheels (e.g. a not-yet-released arcadedb-jvm) placed in # docker/wheels/ are installed first; the dir ships empty (.gitkeep). COPY docker-wheels/ /tmp/wheels/ RUN set -e; \ if ls /tmp/wheels/*.whl >/dev/null 2>&1; then \ - pip install --no-cache-dir /tmp/wheels/*.whl "jpype1==${JPYPE_VERSION}"; \ + _jp=""; \ + for _w in /tmp/wheels/*.whl; do case "$(basename "$_w")" in arcadedb-[0-9]*) ;; *) _jp="jpype1==${JPYPE_VERSION}";; esac; done; \ + pip install --no-cache-dir /tmp/wheels/*.whl ${_jp}; \ fi; \ if [ -n "${PIP_PACKAGES}" ]; then \ _jp=""; \ - case " ${PIP_PACKAGES} " in *" arcadedb-embedded"*) _jp="jpype1==${JPYPE_VERSION}";; esac; \ + case " ${PIP_PACKAGES} " in *" arcadedb-embedded"*|*" arcadedb-jvm"*) _jp="jpype1==${JPYPE_VERSION}";; esac; \ pip install --no-cache-dir ${PIP_PACKAGES} ${_jp}; \ fi; \ if python -c "import jpype" 2>/dev/null; then \ diff --git a/benchmarks/experiments/PAGE-SPEC.md b/benchmarks/experiments/PAGE-SPEC.md index fa94276d95b..1a6b3b19808 100644 --- a/benchmarks/experiments/PAGE-SPEC.md +++ b/benchmarks/experiments/PAGE-SPEC.md @@ -154,7 +154,7 @@ Sizes: `e2` and `e2atom` two, 50k and 500k products (the larger so retrieval rec **Time the actions, not just the ends.** Every scenario reports `open + action + close = session`, and the page quotes the session: timing `open` and `close` alone misses where a query-triggered rebuild lands. The lane's page columns are the cold-start decomposition (4 columns), then per scenario: open ms, action ms, close ms, session ms. -**Cold start is four numbers, not one.** Measured in a fresh subprocess per situation: `import_ms` (interpreter and module import), `jvm_start_ms` (`start_jvm()` alone), `first_open_ms` (first `open_database()`, JVM already up) and `cold_process_ms` (what a CLI actually waits for). A page that quotes a warm open without the cold process beside it is off by orders of magnitude for anything CLI-shaped, and embedded use is mostly CLI-shaped. +**Cold start is four numbers, not one.** Measured in a fresh subprocess per situation: `import_ms` (interpreter and module import), `jvm_start_ms` (`start_jvm()` alone), `first_open_ms` (first `open_database()`, JVM already up) and `cold_process_ms` (what a CLI actually waits for). A page that quotes a warm open without the cold process beside it is off by orders of magnitude for anything CLI-shaped, and embedded use is mostly CLI-shaped. On the native `arcadedb` build there is no JVM: `jvm_start_ms` is null and `engine_load_ms` (the engine library load and isolate creation; `start_jvm()` on the JVM build) is the same step, so compare `import_ms + engine_load_ms` across the two builds. Two columns come from a second fresh process: `cold_first_query_ms` (the situation's first read after the first open) and `cold_query_process_ms` (process start through the end of that read). The definitions are in the docstring of `l5_lifecycle.py`. ### Rendering rules, every table diff --git a/benchmarks/experiments/READING-RESULTS.md b/benchmarks/experiments/READING-RESULTS.md index 4d82c240b85..3c9ba14bc69 100644 --- a/benchmarks/experiments/READING-RESULTS.md +++ b/benchmarks/experiments/READING-RESULTS.md @@ -26,6 +26,8 @@ The single rule, from which most of the rest follows: **`engine_version` is deliberately null for non-ArcadeDB rows in L4.** Use `backend_version`. `run_conditions()` stamps `engine_version` from the installed wheel, which is right for the ArcadeDB row and wrong for DuckDB and QuestDB, so `l4_tsbs.py` asks each backend for its own version instead. A wrong version that passes a provenance audit is worse than none. +**`engine_backend` says which ArcadeDB build produced a row: `native` (the `arcadedb` wheel) or `jvm` (`arcadedb-jvm`, the older `arcadedb_embedded`).** `engine_version` is the same string for both wheels of one release, and `backend` is the arm name, so neither can tell them apart. It is read from `arcadedb.__backend__` by `bench_common.engine_backend()` and is null where the process never loaded ArcadeDB (every comparator, and a served ArcadeDB arm, whose engine runs in a container). + **Arms are selected by flag, not by filename.** `claims_check.ts_arm()` takes `primitive=` and `numpy_cols=` because the published row is one specific arm. Assert the flags rather than globbing the directory. **A comparator row's version and digest are read from the row** (`server_image`, `engine_version`); `runner.BACKENDS` describes future rows only (BUGS.md F32). diff --git a/benchmarks/experiments/_arcadedb_import.py b/benchmarks/experiments/_arcadedb_import.py index a6093d0e8c3..4cbec597417 100644 --- a/benchmarks/experiments/_arcadedb_import.py +++ b/benchmarks/experiments/_arcadedb_import.py @@ -25,11 +25,12 @@ attribute and is always "jvm" (backend()). * `.jvm`: importable in both new wheels, but the native wheel ships no JARs, so `start_jvm()` raises ArcadeDBError ("No JAR files found") there (jvm_module()). The cold-start - subprocess in l5_lifecycle.py calls it, so its `jvm_start_ms` / `import_ms` / `first_open_ms` - columns are null on the native build (checked 2026-10-29 with arcadedb 26.11.1.dev0). + subprocess in l5_lifecycle.py therefore starts the native library instead (see its docstring). * JVM-only APIs (`get_java_database`, `jpype`, Java arrays): they exist only on the JVM build and - the native build has no JPype (l3_sparse's embedded arm imports it and fails there). A lane arm - that needs one must check backend() first; this module does not paper over it. + the native build has no JPype. No lane uses them any more (l3_sparse's embedded arm went through + insert_many, plain Python lists and SQL COMPACT INDEX); a new arm that needs one must check + backend() first, and this module does not paper over it. + * loaded_backend(): backend() once load() has run, else None; it is what the `engine_backend` row field reads. """ import importlib import importlib.metadata @@ -81,6 +82,15 @@ def backend(): return getattr(mod, "__backend__", None) or getattr(mod, "_DISTRIBUTION_BACKEND", None) or "jvm" +def loaded_backend(): + """`backend()` when this process has loaded the package, else None (a comparator arm, a served arm's driver). + + Never imports the engine. This is what the row stamp `engine_backend` reads, so a row of a process that never touched ArcadeDB + carries an explicit None, not a guess. + """ + return backend() if _loaded is not None else None + + def jvm_module(): """`.jvm` (start_jvm, shutdown_jvm, jar_fingerprint ...). Importing the submodule does not start a JVM.""" return importlib.import_module(module_name() + ".jvm") diff --git a/benchmarks/experiments/bench_common.py b/benchmarks/experiments/bench_common.py index 521c22c95b1..4141187d2d2 100644 --- a/benchmarks/experiments/bench_common.py +++ b/benchmarks/experiments/bench_common.py @@ -786,6 +786,15 @@ def _host_identity(): return name or "unknown" +def engine_backend(): + """"native" or "jvm" when this process loaded the ArcadeDB package, else None. Lanes that set engine_version by hand stamp it too.""" + try: + import _arcadedb_import + return _arcadedb_import.loaded_backend() + except Exception: + return None + + def run_conditions(**extra): """The conditions this process is actually running under, read not asserted. @@ -914,6 +923,10 @@ def _read(path): out["engine_version"] = f"unknown ({e.__class__.__name__})" # THE JPYPE THIS PROCESS RAN (CAMPAIGN 7 row 73): empty unless the process imported it. out.update(jpype_fields()) + # WHICH ENGINE BUILD THIS PROCESS RAN, "native" or "jvm" (arcadedb.__backend__). engine_version is the same string for the + # `arcadedb` and `arcadedb-jvm` wheels of one release, so the version alone cannot tell the two arms apart. NOT named `backend`: + # that field is already the arm name (`arcadedb_sparse_embedded`, ...). + out["engine_backend"] = engine_backend() out.update(extra) return out diff --git a/benchmarks/experiments/build_images.sh b/benchmarks/experiments/build_images.sh index 361bfa3d18b..e141738419d 100755 --- a/benchmarks/experiments/build_images.sh +++ b/benchmarks/experiments/build_images.sh @@ -34,7 +34,7 @@ fi _want_arcadedb=1 if [ $# -gt 0 ]; then _want_arcadedb=0; for _t in "$@"; do [ "$_t" = "arcadedb" ] && _want_arcadedb=1; done; fi if [ "$_want_arcadedb" = 1 ] && [ -z "${ARCADEDB_WHEEL:-}" ]; then - echo "REFUSING: the arcadedb image needs ARCADEDB_WHEEL=; there is no PyPI fallback." >&2 + echo "REFUSING: the arcadedb image needs ARCADEDB_WHEEL=; there is no PyPI fallback." >&2 echo " Name the targets (e.g. './build_images.sh duckdb client dense') to build the images that take no wheel." >&2 exit 1 fi @@ -108,7 +108,8 @@ LADYBUG_EXTENSIONS="vector=23da57e805fed208f1eacb4b011cb0c2e917ae098fa6853763e44 # test only that. Same for the wheel path, whose basename carries the version. _pin="" if [ -n "${ARCADEDB_WHEEL:-}" ]; then - # arcadedb_embedded-26.8.1.dev20-py3-none-linux_x86_64.whl -> 26.8.1.dev20 + # arcadedb-26.11.1.dev0-py3-none-manylinux_2_28_x86_64.whl, arcadedb_jvm-26.11.1.dev0-... and the older + # arcadedb_embedded-26.8.1.dev20-py3-none-linux_x86_64.whl all carry the version in the second field -> 26.8.1.dev20 _pin=$(basename "$ARCADEDB_WHEEL" | cut -d- -f2) fi case "$_pin" in diff --git a/benchmarks/experiments/e2_hybrid.py b/benchmarks/experiments/e2_hybrid.py index 0f5131252da..ecc250adb2f 100644 --- a/benchmarks/experiments/e2_hybrid.py +++ b/benchmarks/experiments/e2_hybrid.py @@ -2329,6 +2329,7 @@ def main(): with _beat.phase("connect", backend=args.backend): b = BACKENDS[args.backend]() out["engine_version"] = b.version + out["engine_backend"] = bench_common.engine_backend() out["instrument"] = bench_common.INSTRUMENT # The lane's search breadth (graph-index arms); IVF and LanceDB calibrate by effect and say so. An engine with a lower maximum # (pgvector, 1000) reports what it APPLIED in vector_ef and the lane's value in vector_ef_lane, so a clamp is never silent. diff --git a/benchmarks/experiments/e3_recovery.py b/benchmarks/experiments/e3_recovery.py index d565c08c57a..e564d0494c0 100644 --- a/benchmarks/experiments/e3_recovery.py +++ b/benchmarks/experiments/e3_recovery.py @@ -76,6 +76,8 @@ def verify(): out["engine_version"] = _ai.dist_version() except Exception as e: out["engine_version"] = f"unknown ({e.__class__.__name__})" + import bench_common + out["engine_backend"] = bench_common.engine_backend() outp = os.environ.get("PROBE_OUT", "") if outp: with open(outp, "w") as f: diff --git a/benchmarks/experiments/e4_decomp.py b/benchmarks/experiments/e4_decomp.py index 5f730b5a95a..132a4eb2da3 100644 --- a/benchmarks/experiments/e4_decomp.py +++ b/benchmarks/experiments/e4_decomp.py @@ -109,6 +109,7 @@ def main(): # expects; the full artifact stays in the file. import _arcadedb_import payload["engine_version"] = _arcadedb_import.dist_version() + payload["engine_backend"] = _arcadedb_import.loaded_backend() # THE DURABILITY THE ARM RAN AT, read out of the engine rather than # asserted (#81, #90). Every 2026-10 row must record it -- F10 fails a row # that does not -- and this lane recorded nothing, which nobody noticed diff --git a/benchmarks/experiments/engine_pin.py b/benchmarks/experiments/engine_pin.py index d258eee5fbd..87c9c51d057 100644 --- a/benchmarks/experiments/engine_pin.py +++ b/benchmarks/experiments/engine_pin.py @@ -8,7 +8,9 @@ THE RULE. The stage pins one ArcadeDB version by exporting `ARCADEDB_WHEEL` (the pinned wheel file, whose name carries the version). Before the first cell, for every ArcadeDB arm in the batch, `runner.py` reads the version out of what will run and refuses to start unless it is the pin: - embedded the image's installed `arcadedb-embedded` version AND the engine jar the wheel ships (`arcadedb-engine-.jar`); + embedded the image's installed ArcadeDB distribution version AND the engine the wheel ships: `arcadedb-jvm` (JVM build) or the older + `arcadedb-embedded` carry `arcadedb-engine-.jar`; `arcadedb` (the native build, no JVM, no jars) carries the engine + inside libarcadedb.so and names its version in `_build_info.ENGINE_VERSION`, read here as the same jar name; served the engine jar of the server image (`/home/arcadedb/lib/arcadedb-engine-.jar`), which is the engine the server loads. Nothing is started to ask, so the check costs two short `docker run`s per image and measures nothing. @@ -29,16 +31,34 @@ FLOATING_OK = ("client", "duckdb", "dense", "pg-age", "mongo-search", "composed") SERVER_LIB = "/home/arcadedb/lib" _JAR_RE = re.compile(r"^arcadedb-engine-(.+)\.jar$") -_EMBEDDED_PROBE = ( - "import importlib.metadata as m, importlib.util as u, os; " - "s = u.find_spec('arcadedb_embedded'); " - "print(m.version('arcadedb-embedded')); " - "print(' '.join(sorted(os.listdir(os.path.join(os.path.dirname(s.origin), 'jars')))))" -) +_EMBEDDED_PROBE = """ +import importlib.metadata as m, importlib.util as u, os +s = next((x for x in (u.find_spec(n) for n in ('arcadedb', 'arcadedb_embedded')) if x and x.origin), None) +dist = None +for n in ('arcadedb', 'arcadedb-jvm', 'arcadedb-embedded'): + try: + v = m.version(n) + dist = n + break + except m.PackageNotFoundError: + pass +print(v) +print(dist) +pkg = os.path.dirname(s.origin) +jars = os.path.join(pkg, 'jars') +if os.path.isdir(jars): + print(' '.join(sorted(os.listdir(jars)))) +else: + from importlib import import_module + print('arcadedb-engine-' + import_module(os.path.basename(pkg) + '._build_info').ENGINE_VERSION + '.jar') +""" +# The native distribution is `arcadedb`, the JVM one `arcadedb-jvm`; the old `arcadedb-embedded` is a JVM build. +BACKEND_OF = {"arcadedb": "native", "arcadedb-jvm": "jvm", "arcadedb-embedded": "jvm"} def pin_from_wheel(path): - """The version a wheel file name carries (arcadedb_embedded-26.10.2-cp312-...whl -> 26.10.2), or "" when it carries none.""" + """The version a wheel file name carries (arcadedb_embedded-26.10.2-cp312-...whl, arcadedb_jvm-26.11.1.dev0-py3-...whl or + arcadedb-26.11.1.dev0-py3-...whl -> the version), or "" when it carries none.""" parts = os.path.basename(path or "").split("-") return parts[1] if len(parts) > 2 and re.fullmatch(VERSION_RE, parts[1]) else "" @@ -91,14 +111,22 @@ def _engine_jar_versions(names): def read_embedded(image, run=None): - """({"wheel": ..., "engine_jar": [...]}, None) read out of a bench image by running python in it, or (None, why).""" + """({"wheel": ..., "dist": ..., "backend": ..., "engine_jar": [...]}, None) read out of a bench image by running python in it, or (None, why). + + Three lines from the probe (version, distribution name, engine jar names). Two lines (version, jars) are still read, for a probe + or a test double that predates the new package names: the distribution is then the old `arcadedb-embedded`. + """ out, err = _docker(["--entrypoint", "python3", image, "-c", _EMBEDDED_PROBE], run) if err: return None, err lines = out.strip().splitlines() - if len(lines) < 2: - return None, f"the probe in {image} printed {out.strip()!r}, not a wheel version and a jar list" - return {"wheel": lines[0].strip(), "engine_jar": _engine_jar_versions(lines[1].split())}, None + if len(lines) == 2: + lines = [lines[0], "arcadedb-embedded", lines[1]] + if len(lines) < 3: + return None, f"the probe in {image} printed {out.strip()!r}, not a wheel version, a distribution name and a jar list" + dist = lines[1].strip() + return {"wheel": lines[0].strip(), "dist": dist, "backend": BACKEND_OF.get(dist), + "engine_jar": _engine_jar_versions(lines[2].split())}, None def read_served(image, run=None): @@ -122,7 +150,7 @@ def verify(pin, embedded_images=(), served_images=(), run=None): problems.append(f"{image}: {err}") continue if got["wheel"] != pin: - problems.append(f"{image} has arcadedb-embedded {got['wheel']}, the stage pins {pin}") + problems.append(f"{image} has {got['dist']} {got['wheel']}, the stage pins {pin}") if got["engine_jar"] != [pin]: problems.append(f"{image} ships engine jar {','.join(got['engine_jar']) or '(none)'}, the stage pins {pin}") for image in sorted(set(served_images)): diff --git a/benchmarks/experiments/jvm_payload_check.py b/benchmarks/experiments/jvm_payload_check.py index 3c7afd4ded5..a8ce2bb41d7 100644 --- a/benchmarks/experiments/jvm_payload_check.py +++ b/benchmarks/experiments/jvm_payload_check.py @@ -182,7 +182,7 @@ def check_file(path): # where a list legitimately becomes a Java array -- flagging it is circular, # and its one internal `vector.tolist()` feeds a normalisation loop rather # than the JVM directly. - if os.path.basename(path) == "vector.py" and "arcadedb_embedded" in path: + if os.path.basename(path) == "vector.py" and ("arcadedb_embedded" in path or os.sep + "arcadedb" + os.sep in path): return [] try: src = open(path, encoding="utf-8").read() diff --git a/benchmarks/experiments/l1_tabular.py b/benchmarks/experiments/l1_tabular.py index 168d48cbed5..4eec6ba3bb6 100644 --- a/benchmarks/experiments/l1_tabular.py +++ b/benchmarks/experiments/l1_tabular.py @@ -706,6 +706,8 @@ def main(): b.connect() out["connect_s"] = round(time.perf_counter() - t0, 3) out["engine_version"] = getattr(b, "version", "?") + import bench_common as _bc + out["engine_backend"] = _bc.engine_backend() out.update(getattr(b, "row_extra", None) or {}) # F6 provenance: what the engine's pool was actually sized to, and what the # cpuset allowed. A cell where these differ is oversubscribed; recording it diff --git a/benchmarks/experiments/l1_tpc.py b/benchmarks/experiments/l1_tpc.py index 130762b5bfd..45eb68bc9d4 100644 --- a/benchmarks/experiments/l1_tpc.py +++ b/benchmarks/experiments/l1_tpc.py @@ -1955,6 +1955,7 @@ def main(): with _beat.phase("connect", backend=args.backend): b.connect() out["engine_version"] = b.version + out["engine_backend"] = bench_common.engine_backend() t0 = time.perf_counter() with _beat.phase("build", n=len(li)): b.build(li, part) diff --git a/benchmarks/experiments/l2_graph.py b/benchmarks/experiments/l2_graph.py index 9e4eed3ca57..891debb47fe 100644 --- a/benchmarks/experiments/l2_graph.py +++ b/benchmarks/experiments/l2_graph.py @@ -3504,6 +3504,7 @@ def gen_edges(n, *a, **kw): # noqa: F811 - deliberate shadow, counted ad.connect() out["connect_s"] = round(time.perf_counter() - t0, 3) out["engine_version"] = ad.version + out["engine_backend"] = bench_common.engine_backend() # What a served engine reported about itself at connect: client library # version, thread pool, memory limit, timeouts, persistence settings # (Memgraph, FalkorDB, DuckPGQ). Read from the server, not restated from diff --git a/benchmarks/experiments/l3_sparse.py b/benchmarks/experiments/l3_sparse.py index 01f800b0d3e..26b57a53adf 100644 --- a/benchmarks/experiments/l3_sparse.py +++ b/benchmarks/experiments/l3_sparse.py @@ -96,6 +96,8 @@ def recall_at_k(result_ids, gt_row): # the same ids in the same order as the whole-record form (50 of 50 queries, the pin and main, laptop). SPARSE_RESULT = "id" ARCADE_SPARSE_SEARCH_SQL = "SELECT id, score FROM (SELECT expand(`vector.sparseNeighbors`({a})))" +# The settle step, one statement for every ArcadeDB arm (embedded on either package, served): engine #5144. +COMPACT_SQL = "COMPACT INDEX `Doc[tokens,weights]`" class Base: @@ -221,44 +223,31 @@ def reopen(self): self.idx_name = "Doc[tokens,weights]" def build(self, n_docs): - # Native document API with primitive arrays — the embedded analog of - # Qdrant/Milvus binary clients (1.8x faster than SQL INSERT batches; - # the server adapter keeps SQL-over-HTTP, its native remote surface). - import jpype - JInt, JFloat = jpype.JArray(jpype.JInt), jpype.JArray(jpype.JFloat) - jdb = self.db.get_java_database() - jdb.begin() - open_n = 0 - for i, idx, vals in gen_docs(n_docs): - d = jdb.newDocument("Doc") - d.set("id", i) - d.set("tokens", JInt(idx)) - d.set("weights", JFloat(vals)) - d.save() - open_n += 1 - if open_n == INGEST_BATCH: - jdb.commit() - jdb.begin() - open_n = 0 - jdb.commit() + # ONE PATH FOR BOTH PACKAGES (the native `arcadedb` build has no JVM, no JPype and no get_java_database(), so the + # old jdb.newDocument + JArray ingest could not run there). The documents go through the Python API's bulk insert, + # `Database.insert_many`: one batch per commit (INGEST_BATCH), the token and weight lists as the generators yield + # them (Python lists of int and float). The declared property types turn them into ARRAY_OF_INTEGERS and + # ARRAY_OF_FLOATS (the smoke reads the stored types back); the weights are stored as float32 either way, so the + # ingested values equal the ground truth's, as in the Java-array path this replaced. + self.db.insert_many( + "Doc", + ({"id": i, "tokens": idx, "weights": vals} for i, idx, vals in gen_docs(n_docs)), + commit_every=INGEST_BATCH) def post_build(self): - # ArcadeDB's settle step (fairness parity with ES forcemerge / Milvus - # flush+load / Qdrant green-wait): blocking LSM segment compaction via - # the Java API — embedded-only, no SQL/HTTP trigger exists. Measured at - # 1M docs: ~2s, query p50 9.5ms -> 7.0ms (control run: no change). - jdb = self.db.get_java_database() - for idx in jdb.getSchema().getIndexes(): - if "SparseVector" in str(idx.getClass().getSimpleName()): - idx.compact() + # ArcadeDB's settle step (fairness parity with ES forcemerge / Milvus flush+load / Qdrant green-wait): blocking LSM + # segment compaction, the SQL statement the served arm already runs (engine #5144), so embedded, native and server + # settle the same way. Measured at 1M docs with the old Java call (idx.compact()): ~2s, query p50 9.5ms -> 7.0ms + # (control run: no change). + self.db.command("sql", COMPACT_SQL) def search(self, idx, vals, k): - import jpype - ji = jpype.JArray(jpype.JInt)(idx) - jv = jpype.JArray(jpype.JFloat)(vals) + # Python lists, as the generators yield them: both packages convert a list to the int[] / float[] the function + # reads. NOT numpy arrays: the JVM build's argument conversion turns every ndarray into float[], which the + # indices argument refuses (measured on 26.11.1.dev0: "Array must be int[], long[], Integer[]..."). rows = self.db.query( "sql", ARCADE_SPARSE_SEARCH_SQL.format(a="?, ?, ?, ?"), - self.idx_name, ji, jv, k).to_json_list() + self.idx_name, idx, vals, k).to_json_list() return [int(r["id"]) for r in rows] # ids, as every comparator returns them def resolve(self, ids): @@ -279,13 +268,11 @@ class ArcadeEmbeddedFP32(ArcadeEmbedded): class ArcadeEmbeddedNoCompact(ArcadeEmbedded): - """Settle-step ablation; de-confounds the deployment axis. + """Settle-step ablation: the embedded arm without COMPACT INDEX. - The server cannot trigger compaction (no SQL/HTTP path, engine #5144), so - "embedded (compacted) vs server (as-shipped)" mixes transport cost with - settle-step cost. Skipping compaction here gives the decomposition: - embedded vs embedded_nocompact -> settle-step cost - embedded_nocompact vs server -> pure transport cost + The served arm now runs the same COMPACT INDEX, so embedded vs server is the + transport cost and embedded vs embedded_nocompact is the settle-step cost + (time spent compacting, and the query latency it buys). """ name = "arcadedb_sparse_embedded_nocompact" @@ -323,10 +310,10 @@ def connect(self): def post_build(self): # Settle step, now client-reachable over HTTP/SQL via COMPACT INDEX - # (engine #5144, in 26.8.x). This gives the server the same settle the - # embedded adapter gets via idx.compact(), so the deployment axis - # isolates transport, not the settle asymmetry. Synchronous. - self._cmd("sql", "COMPACT INDEX `Doc[tokens,weights]`") + # (engine #5144, in 26.8.x), the same statement the embedded adapter runs, + # so the deployment axis isolates transport, not the settle asymmetry. + # Synchronous. + self._cmd("sql", COMPACT_SQL) def _cmd(self, language, command, params=None): payload = {"language": language, "command": command} diff --git a/benchmarks/experiments/l3d_dense.py b/benchmarks/experiments/l3d_dense.py index 2f6597b01a9..2c51b19eed0 100644 --- a/benchmarks/experiments/l3d_dense.py +++ b/benchmarks/experiments/l3d_dense.py @@ -3261,6 +3261,7 @@ def main(): b.connect() out["connect_s"] = round(time.perf_counter() - t0, 3) out["engine_version"] = b.version + out["engine_backend"] = bench_common.engine_backend() t0 = time.perf_counter() with _beat.phase("build", n=len(train)): diff --git a/benchmarks/experiments/l5_lifecycle.py b/benchmarks/experiments/l5_lifecycle.py index 3ffac29d9e6..4e909b3f604 100644 --- a/benchmarks/experiments/l5_lifecycle.py +++ b/benchmarks/experiments/l5_lifecycle.py @@ -36,6 +36,24 @@ rows by up to 617x, because a session that never touches the index leaves it in a state close then has to resolve. That is not a neutral protocol, it is one specific case (#5747), so all three are measured and reported separately. + +THE COLD ROW ON BOTH BUILDS (26.11.1: `arcadedb-jvm` runs on a JVM through JPype, `arcadedb` is the native image with no +JVM). The cold-start columns come from two fresh subprocesses (_cold_start_code, _cold_query_code), against a warm page cache +(cold_process_cache_state), and mean the same on both builds: + + import_ms interpreter start + `import arcadedb` + engine_load_ms the build's own start step: start_jvm() on the JVM build, the engine library load + isolate creation + (arcadedb._native.lib._load(), what the first database call triggers) on native + jvm_start_ms = engine_load_ms on the JVM build, null on native (there is no JVM; kept for the old rows) + first_open_ms the first open_database, with the engine already started + cold_process_ms the whole process through close(): what a CLI that opens and closes waits for + cold_first_query_ms the situation's first read (the `read` mode's _read) in a fresh process after the first open; + null for `empty`, which has no read + cold_query_process_ms process start through the end of that read: open, engine start and one question + engine_backend "native" or "jvm", on every ArcadeDB row (arcadedb.__backend__) + +Library load = import_ms + engine_load_ms on either build. cold_open_ms / cold_close_ms (a verified page-cache eviction, then a +cycle in the lane's own process) are measured identically on both builds and are the only cold-cache numbers. """ import argparse import json @@ -48,7 +66,8 @@ import bench_common import pagecache -DB = "/lcdb/lc" # a HOST directory bind-mounted here; see _assert_fs +# a HOST directory bind-mounted here; see _assert_fs. BENCH_LC_DB moves it for a laptop smoke (the default is the runner's mount, unchanged). +DB = os.environ.get("BENCH_LC_DB", "/lcdb/lc") DIM = 64 ITERS = int(os.environ.get("BENCH_LC_ITERS", "3")) # Which scenarios to run. At very large scales the write modes trigger a full @@ -585,6 +604,66 @@ def measure_stale(situation, mode): return st.median(o), st.median(c), st.median(w) +def _cold_start_code(native, db): + """The source of the cold-start subprocess: process start, import, engine load, first open, close. Prints four numbers (ms). + + JVM build (and the old arcadedb_embedded wheel), byte for byte what this lane has always timed: + import_ms interpreter start + `import @MOD@` (the JPype module included) + engine_load_ms start_jvm() alone, no database (also written as jvm_start_ms) + first_open_ms first open_database, JVM already up + cold_process_ms the whole process through close() + Native build (`arcadedb`, no JVM, no JPype). The same four steps, each where it means the same thing: + import_ms interpreter start + `import arcadedb` (a ctypes module; the engine library is not loaded yet) + engine_load_ms the engine library load + isolate creation (`arcadedb._native.lib._load()`, the step the first database + call triggers); the native counterpart of start_jvm(), so jvm_start_ms is null + first_open_ms first open_database, library already loaded (the same meaning as on the JVM) + cold_process_ms the whole process through close() (the same meaning) + "Library load" in the cold row is therefore import_ms + engine_load_ms on native and import_ms + jvm_start_ms on the JVM build. + """ + if native: + head = ("import time; _t0 = time.perf_counter()\n" + "import @MOD@ as a\n" + "_ti = time.perf_counter()\n" + "from @MOD@._native import lib as _lib; _lib._load()\n") + else: + head = ("import time; _t0 = time.perf_counter()\n" + "from @MOD@.jvm import start_jvm\n" + "import @MOD@ as a\n" + "_ti = time.perf_counter()\n" + "start_jvm()\n") + return head + ( + "_tj = time.perf_counter()\n" + "db = a.open_database(%r)\n" + "_to = time.perf_counter()\n" + "db.close()\n" + "print('%%.3f %%.3f %%.3f %%.3f' %% ((_ti-_t0)*1000, (_tj-_ti)*1000, (_to-_tj)*1000, " + "(time.perf_counter()-_t0)*1000))" % db) + + +def _cold_query_code(situation, db, here): + """The source of the cold-query subprocess: the first query of `situation` in a fresh process. Prints two numbers (ms). + + Same on both builds. A fresh process opens the built database (page cache warm, as for cold_process_ms) and runs the + situation's read exactly as the `read` mode does (`_read`: the vector search, the sparse search, the counts ...), once: + cold_first_query_ms that one read, after the first open (null for `empty`, which has no read) + cold_query_process_ms process start (after this lane's own stdlib helper imports) through the end of that read: + what a script that opens a database and asks one question waits for + The open and the engine start are inside cold_query_process_ms (they are not separate columns here; the cold-start process + reports them), and the close is not: a close after a query is the `read` mode's number. + """ + return ( + "import sys; sys.path.insert(0, %r)\n" + "import l5_lifecycle as _L\n" + "import time; _t0 = time.perf_counter()\n" + "import @MOD@ as a\n" + "db = a.open_database(%r)\n" + "_tq = time.perf_counter()\n" + "_L._read(db, %r)\n" + "_te = time.perf_counter()\n" + "db.close()\n" + "print('%%.3f %%.3f' %% ((_te-_tq)*1000, (_te-_t0)*1000))" % (here, db, situation)) + + def main(): ap = argparse.ArgumentParser() ap.add_argument("--backend", required=True) @@ -646,23 +725,13 @@ def main(): # are the same statements as before (`arcadedb_embedded` on the old wheels, `arcadedb` on the new # ones) and import_ms still times nothing else. import _arcadedb_import + _native = _arcadedb_import.backend() == "native" _boot = subprocess.run( - [sys.executable, "-c", ( - "import time; _t0 = time.perf_counter()\n" - "from @MOD@.jvm import start_jvm\n" - "import @MOD@ as a\n" - "_ti = time.perf_counter()\n" - "start_jvm()\n" - "_tj = time.perf_counter()\n" - "db = a.open_database(%r)\n" - "_to = time.perf_counter()\n" - "db.close()\n" - "print('%%.3f %%.3f %%.3f %%.3f' %% ((_ti-_t0)*1000, (_tj-_ti)*1000, (_to-_tj)*1000, " - "(time.perf_counter()-_t0)*1000))" - % DB).replace("@MOD@", _arcadedb_import.module_name())], + [sys.executable, "-c", _cold_start_code(_native, DB) + .replace("@MOD@", _arcadedb_import.module_name())], capture_output=True, text=True, timeout=900) if _boot.returncode == 0 and _boot.stdout.strip(): - import_ms, jvm_start_ms, first_open_ms, cold_proc_ms = ( + import_ms, engine_load_ms, first_open_ms, cold_proc_ms = ( float(x) for x in _boot.stdout.strip().split()[-4:]) else: # NOT -1.0. A sentinel that is a float goes into a median, a mean and a @@ -673,7 +742,25 @@ def main(): sys.stderr.write( "cold-start subprocess failed (rc=%s); cold columns will be null\n%s\n" % (_boot.returncode, (_boot.stderr or "")[-2000:])) - import_ms = jvm_start_ms = first_open_ms = cold_proc_ms = None + import_ms = engine_load_ms = first_open_ms = cold_proc_ms = None + # jvm_start_ms keeps its meaning (start_jvm() alone) and exists only where there is a JVM. The native build has none, so the + # column is null there and engine_load_ms (below) carries the build's own start step. + jvm_start_ms = None if _native else engine_load_ms + # The first query in a FRESH process (see _cold_query_code), a second subprocess so the one above keeps measuring + # open-then-close without a query, as it always did. + _q = subprocess.run( + [sys.executable, "-c", _cold_query_code(args.workload, DB, os.path.dirname(os.path.abspath(__file__))) + .replace("@MOD@", _arcadedb_import.module_name())], + capture_output=True, text=True, timeout=900) + if _q.returncode == 0 and _q.stdout.strip(): + cold_first_query_ms, cold_query_process_ms = (float(x) for x in _q.stdout.strip().split()[-2:]) + else: + sys.stderr.write( + "cold-query subprocess failed (rc=%s); its columns will be null\n%s\n" + % (_q.returncode, (_q.stderr or "")[-2000:])) + cold_first_query_ms = cold_query_process_ms = None + if args.workload == "empty": + cold_first_query_ms = None # the situation has no read to time (_read returns at once) out = bench_common.run_conditions( lane="lifecycle", backend=args.backend, workload=args.workload, @@ -705,7 +792,10 @@ def main(): # and first_open_ms is ArcadeDB's own initialisation with the JVM already up. _r3 = lambda x: None if x is None else round(x, 3) out["import_ms"] = _r3(import_ms) # interpreter + module import - out["jvm_start_ms"] = _r3(jvm_start_ms) # start_jvm() alone, no database + out["jvm_start_ms"] = _r3(jvm_start_ms) # start_jvm() alone, no database; null on the native build + out["engine_load_ms"] = _r3(engine_load_ms) # the build's own start step: start_jvm() (jvm) or library load + isolate (native) + out["cold_first_query_ms"] = _r3(cold_first_query_ms) # first query of the situation, fresh process, after the first open + out["cold_query_process_ms"] = _r3(cold_query_process_ms) # that process: start through the end of the first query out["first_open_ms"] = _r3(first_open_ms) # first open, JVM already up out["cold_process_ms"] = _r3(cold_proc_ms) # what a CLI actually waits for # WHICH CACHE STATE THESE FOUR DESCRIBE. cold_open_ms above is measured after diff --git a/benchmarks/experiments/l6_restart.py b/benchmarks/experiments/l6_restart.py index afac2c05d22..bed2022117a 100644 --- a/benchmarks/experiments/l6_restart.py +++ b/benchmarks/experiments/l6_restart.py @@ -794,6 +794,7 @@ def main(): _rx.update(_ad.readbacks()) # the dense adapters' hook, after their build timer out.update({k: v for k, v in _rx.items() if k in _ov.STAMP_FIELDS}) out["engine_version"] = eng.version + out["engine_backend"] = bench_common.engine_backend() bench_common.stamp_durability(out, eng.durability) out["instrument"] = bench_common.INSTRUMENT out["restart_ping"] = eng.ping_desc diff --git a/benchmarks/experiments/make_2611_stages.py b/benchmarks/experiments/make_2611_stages.py index a2b9af20546..640bf8db37f 100644 --- a/benchmarks/experiments/make_2611_stages.py +++ b/benchmarks/experiments/make_2611_stages.py @@ -66,7 +66,7 @@ ARCADEDB_WHEEL=... ARCADEDB_SERVER_IMAGE=... ARCADEDB_ENGINE_COMMIT=<40 hex> \\ python3 make_2611_stages.py --after qR15 --status-file STATUS.txt --id-prefix qS --out DIR [--engines a,b] [--lanes graph,dense] [--profile paper-2611|full] [--second-list (full only)] python3 make_2611_stages.py --arcadedb-release 26.11.1 --engine-commit <40 hex> --image-digest sha256:<64 hex> \\ - --wheel-name arcadedb_embedded-26.11.1-.whl --wheel-sha256 <64 hex> --wheel-url \\ + --wheel-name arcadedb_jvm-26.11.1-.whl --wheel-sha256 <64 hex> --wheel-url \\ --after --status-file STATUS.txt --id-prefix qT --out DIR python3 make_2611_stages.py --check --drafts DIR [--status-file STATUS.txt] # also: the drafts' first wait is not already in STATUS.txt """ @@ -1119,7 +1119,10 @@ def seed(m): # The pair check below is the REPO copy, not ~/verify_pair_c25.sh: that # one is a stale copy from 2026-08-30 and would miss its own fourth # check. A script pulled with the tree cannot drift from it. -IV=$(docker run --rm --entrypoint python3 "dbbench:arcadedb-$WV" -c 'import importlib.metadata as m; print(m.version("arcadedb-embedded"))' 2>/dev/null | tr -dc "0-9a-zA-Z.-") +IV=$(docker run --rm --entrypoint python3 "dbbench:arcadedb-$WV" -c 'import importlib.metadata as m +for n in ("arcadedb", "arcadedb-jvm", "arcadedb-embedded"): + try: print(m.version(n)); break + except m.PackageNotFoundError: pass' 2>/dev/null | tr -dc "0-9a-zA-Z.-") [ "$IV" = "$WV" ] || {{ say "$ID ABORT: dbbench:arcadedb carries wheel $IV, dist has $WV"; exit 1; }} say "$ID: dbbench:arcadedb carries wheel $IV, matching dist" IJ=$(docker run --rm --entrypoint python3 "dbbench:arcadedb-$WV" -c 'import jpype; print(jpype.__version__)' 2>/dev/null | tr -dc "0-9a-zA-Z.-") @@ -1726,9 +1729,9 @@ def release_pins(a): raise SystemExit("REFUSING: --image-digest must be sha256:<64 hex>, the official image by digest") if not re.fullmatch(r"[0-9a-f]{64}", a.wheel_sha256): raise SystemExit("REFUSING: --wheel-sha256 must be 64 hex characters") - if not re.fullmatch(rf"arcadedb_embedded-{re.escape(a.arcadedb_release)}-[A-Za-z0-9_.]+-[A-Za-z0-9_.]+-[A-Za-z0-9_.]+\.whl", + if not re.fullmatch(rf"arcadedb(?:_embedded|_jvm)?-{re.escape(a.arcadedb_release)}-[A-Za-z0-9_.]+-[A-Za-z0-9_.]+-[A-Za-z0-9_.]+\.whl", a.wheel_name): - raise SystemExit(f"REFUSING: --wheel-name {a.wheel_name} is not a {a.arcadedb_release} arcadedb_embedded wheel") + raise SystemExit(f"REFUSING: --wheel-name {a.wheel_name} is not a {a.arcadedb_release} arcadedb, arcadedb_jvm or arcadedb_embedded wheel") if not a.wheel_url.startswith("https://") or not a.wheel_url.endswith(a.wheel_name): raise SystemExit("REFUSING: --wheel-url must be an https URL ending in the wheel's file name") return {"ARCADEDB_WHEEL": os.path.join("bindings", "python", "dist", a.wheel_name), diff --git a/benchmarks/experiments/queue_lint.py b/benchmarks/experiments/queue_lint.py index afbb603326d..2cf04381ee2 100644 --- a/benchmarks/experiments/queue_lint.py +++ b/benchmarks/experiments/queue_lint.py @@ -128,7 +128,9 @@ def wheel_is_prerelease(body): build_images.sh line and rejected all fourteen stages of the 26.10.1 chain the first time they were linted against the official wheel. """ - m = re.search(r"arcadedb_embedded-([^-\s\"']+)-", body) + # arcadedb_embedded-- (old), arcadedb_jvm-- and arcadedb-- (26.11.1+); the version starts with a digit, so + # `arcadedb-engine-` in a jar name is not a wheel. + m = re.search(r"arcadedb(?:_embedded|_jvm)?-([0-9][^-\s\"']*)-", body) if not m: return None return bool(_PRE_TOKENS.search(m.group(1))) diff --git a/benchmarks/experiments/smoke_backend_rows.py b/benchmarks/experiments/smoke_backend_rows.py new file mode 100644 index 00000000000..1f16f5c52bc --- /dev/null +++ b/benchmarks/experiments/smoke_backend_rows.py @@ -0,0 +1,125 @@ +#!/usr/bin/env python3 +"""Smoke check of the `engine_backend` stamp and of the two lanes that run on the native build (l3 sparse, l5 lifecycle). + +Run it with the SAME wheel that produced the row (`uv run --no-project --python 3.12 --with --with numpy python +smoke_backend_rows.py l3|l5 ROW.json native|jvm`): it asserts the row, and for l3 it reopens the lane's database and checks what +the Python-API ingest stored. Exits nonzero on any failed assertion, so a queue line chained with `&&` fails loudly. + + l3 the row says engine_backend, every document was ingested (n_docs_ingested == n_docs), the stored property types are + ARRAY_OF_INTEGERS / ARRAY_OF_FLOATS (read back through SQL), the stored arrays equal the generator's (tokens exactly, weights + to float32), and a top-10 sparse search with plain Python lists as parameters reaches recall >= 0.9 against a brute-force dot + product over the generator's own corpus (the default INT8 posting quantization keeps it below 1). The lane's database + (/tmp/l3_arcade) is removed at the end, so the queue line must start with `test ! -e /tmp/l3_arcade`. + l5 the row says engine_backend and carries the cold row: import_ms, engine_load_ms, first_open_ms, cold_process_ms, + cold_first_query_ms, cold_query_process_ms all positive; jvm_start_ms is null on native and equals engine_load_ms on the JVM. +""" +import json +import os +import shutil +import sys + +HERE = os.path.dirname(os.path.abspath(__file__)) +sys.path.insert(0, HERE) + +L3_DB = "/tmp/l3_arcade" # nosec B108 - the lane's own fixed scratch path (l3_sparse.ArcadeEmbedded.connect) + + +def fail(msg): + print("SMOKE FAIL: " + msg, flush=True) + sys.exit(1) + + +def ok(msg): + print("ok " + msg, flush=True) + + +def check_l3(row, expected): + if row.get("engine_backend") != expected: + fail(f"engine_backend {row.get('engine_backend')!r}, expected {expected!r}") + ok(f"engine_backend = {expected}") + if row.get("n_docs_ingested") != row.get("n_docs"): + fail(f"n_docs_ingested {row.get('n_docs_ingested')} != n_docs {row.get('n_docs')}") + ok(f"ingested {row['n_docs_ingested']} documents") + import sparse_common as sc + import _arcadedb_import + if _arcadedb_import.backend() != expected: + fail(f"this process runs the {_arcadedb_import.backend()} build, the row says {expected}") + arcadedb = _arcadedb_import.load() + n = row["n_docs"] + docs = list(sc.gen_docs(n)) + db = arcadedb.open_database(L3_DB) + try: + for prop, want in (("tokens", "ARRAY_OF_INTEGERS"), ("weights", "ARRAY_OF_FLOATS")): + got = db.query("sql", f"SELECT {prop}.type() AS t FROM Doc WHERE id = 7").to_json_list()[0]["t"] + if got != want: + fail(f"stored type of Doc.{prop} is {got}, expected {want}") + ok(f"Doc.{prop} stored as {got}") + cnt = db.query("sql", "SELECT count(*) AS c FROM Doc").to_json_list()[0]["c"] + if int(cnt) != n: + fail(f"count(*) = {cnt}, expected {n}") + import numpy as np + for i in (0, 7, n // 2, n - 1): + r = db.query("sql", "SELECT tokens, weights FROM Doc WHERE id = ?", i).to_json_list()[0] + _, idx, vals = docs[i] + if list(r["tokens"]) != list(idx): + fail(f"tokens of doc {i} differ from the generator's") + if not np.array_equal(np.asarray(r["weights"], np.float32), np.asarray(vals, np.float32)): + fail(f"weights of doc {i} differ from the generator's beyond float32 rounding") + ok("stored tokens equal the generator's, weights equal at float32") + queries = sc.gen_queries(50) + sql = "SELECT id, score FROM (SELECT expand(`vector.sparseNeighbors`(?, ?, ?, ?)))" + hits = total = 0 + for qi, qv in queries: + q = dict(zip(qi, qv)) + scores = [(sum(q.get(t, 0.0) * w for t, w in zip(idx, vals)), i) for i, idx, vals in docs] + truth = {i for s, i in sorted(scores, key=lambda x: (-x[0], x[1]))[:10] if s > 0} + rows = db.query("sql", sql, "Doc[tokens,weights]", qi, qv, 10).to_json_list() + got = {int(r["id"]) for r in rows} + hits += len(got & truth) + total += len(truth) + recall = hits / total + if recall < 0.9: + fail(f"recall@10 against brute force is {recall:.3f} (lists as parameters), expected >= 0.9") + ok(f"recall@10 vs brute force {recall:.3f} over {len(queries)} queries (list parameters)") + if "build_s" in row: + ok(f"row build_s {row['build_s']}") + finally: + db.close() + shutil.rmtree(L3_DB, ignore_errors=True) + + +def check_l5(row, expected): + if row.get("engine_backend") != expected: + fail(f"engine_backend {row.get('engine_backend')!r}, expected {expected!r}") + ok(f"engine_backend = {expected}") + for k in ("import_ms", "engine_load_ms", "first_open_ms", "cold_process_ms", "cold_query_process_ms"): + if not isinstance(row.get(k), (int, float)) or row[k] <= 0: + fail(f"{k} = {row.get(k)!r}, expected a positive number") + if row.get("workload") != "empty" and not (isinstance(row.get("cold_first_query_ms"), (int, float)) and row["cold_first_query_ms"] > 0): + fail(f"cold_first_query_ms = {row.get('cold_first_query_ms')!r}, expected a positive number") + if expected == "native": + if row.get("jvm_start_ms") is not None: + fail(f"jvm_start_ms = {row['jvm_start_ms']!r} on native, expected null") + elif row.get("jvm_start_ms") != row.get("engine_load_ms"): + fail(f"jvm_start_ms {row.get('jvm_start_ms')!r} != engine_load_ms {row.get('engine_load_ms')!r} on the JVM build") + ok("cold row: " + ", ".join(f"{k}={row.get(k)}" for k in ("import_ms", "jvm_start_ms", "engine_load_ms", "first_open_ms", + "cold_process_ms", "cold_first_query_ms", "cold_query_process_ms"))) + for k in ("cold_open_ms", "cold_close_ms", "clean_open_ms", "read_session_ms"): + if not isinstance(row.get(k), (int, float)): + fail(f"{k} missing from the row") + ok("warm and evicted-cache columns present") + + +def main(argv): + if len(argv) != 4 or argv[1] not in ("l3", "l5") or argv[3] not in ("native", "jvm"): + print(__doc__) + return 2 + with open(argv[2]) as f: + row = json.load(f) + {"l3": check_l3, "l5": check_l5}[argv[1]](row, argv[3]) + print("SMOKE PASS", argv[1], argv[3], flush=True) + return 0 + + +if __name__ == "__main__": + sys.exit(main(sys.argv)) diff --git a/benchmarks/experiments/stage_drafts_2611.sha256 b/benchmarks/experiments/stage_drafts_2611.sha256 index 9bf61482b93..ceb52ac7b80 100644 --- a/benchmarks/experiments/stage_drafts_2611.sha256 +++ b/benchmarks/experiments/stage_drafts_2611.sha256 @@ -1,42 +1,42 @@ 4c69fbd9e12a9112e73a13340e2683711a7fd8db927a53261275f7f5d117ca31 arcadedb-release/qV00.sh -df5e42f41212cb6b68183074336b304a0d8f21c4c14bf29a65a4b114ea50ed4c arcadedb-release/qV01.sh -264a943206b36c2bd8e7a63def915b666a01b0bee6e01a9b1a6aacfc6d57f04b arcadedb-release/qV02.sh -063e7df3f2be7a528b484986f1d92cb1e1d6221d793555931e6038f8fb24fc31 arcadedb-release/qV03.sh -5bf911d637bbee3b3fbca1cff436a0c228c68b56e0486bde6fc8e9942dfe5f6d arcadedb-release/qV04.sh -059912310409c4b6b4584de7382f7d519b891aaa9a5e0c63df9a2fd3ac7187e5 arcadedb-release/qV05.sh -60ae19a28cbf97fc36dbd4ade97c80bbc83ae8aacaa2898586435fd4bfad4780 arcadedb-release/qV06.sh -521749d2db5f9ebad9b011f19e01c98d01d09ea0e82c96e916fbcf81011b93b3 arcadedb-release/qV07.sh -974298c78ac5e5cc90acdd572a450477f60c13b15d63db145caaf905337b5672 arcadedb-release/qV08.sh -b33b3fbd56bf0ebbf4e028f3b2ffceda7b5a4fccc006ea4fac12b7e8f4f8b911 arcadedb-release/qV09.sh -84b7d52822e8339b29a5f0d87f8913f30259b854ee0ed05bc71c771af58f397d arcadedb-release/qV10.sh -585e39d3eba2bcfcade39f9059fe36be82b10e696ed74b2a3164301a73ce4c08 arcadedb-release/qV11.sh -0c61e00da2d357fe355b0542cadf38839c2035fc24b65bc675ea3b96fd445b15 arcadedb-release/qV12.sh -08cccff639fdca580da34006d5c6bbebc3ed1029a2a15aa1b0dae6ae2570df78 arcadedb-release/qV13.sh -2ccc1ccf197a21accaade013c1123b7a14ce1e5bb59f93a8b16ca36c13bfa6b6 arcadedb-release/qV14.sh -5ae6843a40178071c1760464a6c11050cf4f529bcc754f1d2fe1ff84a1d638be arcadedb-release/qV15.sh -44f0d227abd372b376457a676b51abaafa095ffa107c617080fc6c68ba503faf arcadedb-release/qV16.sh -a2ea7fee39a52760454c7e68dc48c73ab3dd6eed566aa922f8400c9126447657 arcadedb-release/qV17.sh -9532354bc1d0d4ae48153f67defda6815b140d74cd753b75e6b0bc3a02494219 arcadedb-release/qV18.sh -e167400657ffc131afbb57a4827e9562071ed7ae281e6f40a26fc7ea674c54cb arcadedb-release/qV19.sh -7e2adf0676e7fa67e028c20e5b401d713e1ac2d28416bdb7207bf29f6efb7cdc arcadedb-release/qV20.sh -9d212fe845a10601778215e284b9da89ebc60377cbfaf462eb98765fb134ce71 arcadedb-release/qV21.sh -137ff53b2eeb1363487589c37c0efbf234135fb0d8d3580f4fd13c93ce38d27b arcadedb-release/qV22.sh -7cc8c102751a49d0b595c9dd40e4b149b61b0fc8cbe262734f2310b12ece553a arcadedb-release/qV23.sh -3852bcf3d387e27bf4929bdb927bcbe6c07e45a0d6413454c85ea15c59583e8f arcadedb-release/qV24.sh -8c2c0a166eaec26c1b8ba988c292a7e28471ca6e74294f348bbcad752a243c21 arcadedb-release/qV25.sh -925387e21ba0a7e7f861f867b3ce71a9d59e656bf52764f2907c181ffcdc355a arcadedb-release/qV26.sh -9f19f13c9a744c989b90ea3ba7d53978e37b65ac7878c91fc3e6da56d9fbbe02 arcadedb-release/qV27.sh -3fbc13d42e1f618a8f1a3341476823685ffacfd6431318e9bb2f91fab0162bf7 arcadedb-release/qV28.sh -bfbfd9fa8d12441f1f5fe54547faa4ba1de0c5237077b513a28f42cd7e59cd04 arcadedb-release/qV29.sh -1875ed1bd55530b2e2ba436c4fcc3e9427349786eec3c249d28ca78bfea86736 arcadedb-release/qV30.sh -dc14086ff2e4f19b75be492ed8383de486125ae0481e44a7ea8df72a3f14fa57 arcadedb-release/qV31.sh -000acd27d98c403a4b0ed9fdfe15bf4cc28cb04fa4bd7a2100d9fe8cb094336a arcadedb-release/qV32.sh -687c8ba2d35797c7a58083451bc4f7e8982e014fb6929e80b07483c6b3c52fe9 arcadedb-release/qV33.sh -aa41770c3f940229bc9f1fc8a453c9782579f5cfe0eb729964703aab6b7ddcd7 arcadedb-release/qV34.sh -53192f053507883e4ca51e583240d19a1e7ccfd71f4282a271a6f1664d3b78f4 arcadedb-release/qV35.sh -0cf0f3076e6da4167233ab514c0f428e1e917842e2b120f944a35c58d71093d8 arcadedb-release/qV36.sh -c28a919ef36663e4de5697c7e072dc8f6d4be5d78f185a164455fe3ae59efcfe arcadedb-release/qV37.sh -7915c4a03f7d6af9eb7b350360051cd59e42d78e976282077a90d532ab4cb1b1 arcadedb-release/qV38.sh +a168325cab96fe46d9b5a8ef6c4bad92acdd107e00fc8335ad15874f785fe548 arcadedb-release/qV01.sh +683d137c511786c505da1752d291814decdc5f892b4de58424a8e57c3b54f9bc arcadedb-release/qV02.sh +b6406f96385b5c9171db7aab27f6b512af96cdc150dc029ee0d19cd55800c4b9 arcadedb-release/qV03.sh +3a1af0073d71b348a28d660ab5e17ba70125d539e5d3457dbc30c8199c002059 arcadedb-release/qV04.sh +826096159e118964c7000a2ba8485d51a78066abd4c4651e5a949b5d1178546f arcadedb-release/qV05.sh +9a244a35c85f1d001482af845f074f0887ec04e8d51c5620d326dbbeef98b0b0 arcadedb-release/qV06.sh +3a3e421d65b60a4371d91a973a7d27002fa799a867e7b54a55d21a8b30662745 arcadedb-release/qV07.sh +4fade8d29ba24c4cdc7c56f7718ad9929bc108b7335c9466e8b97c30de3c54d3 arcadedb-release/qV08.sh +799cdfd95037882cb25b25d11b97adf898b284e48ede3bcb493afcb762849c7f arcadedb-release/qV09.sh +9755f0c9af6673778025e5ad7384ffb994d75e1287bf4bf1c9c7570e10ec9564 arcadedb-release/qV10.sh +d4e94034e9aa1dfe388113ad786a5bcaecba216e6306384901a28e883343e414 arcadedb-release/qV11.sh +b2053a1b2c15f40959b943f1adbe2db1833e353465589f2809657e0722b19a7f arcadedb-release/qV12.sh +676ea6f7183d518cda1fc9048d5b6888497590ff03d1825d60fa9b6c90820092 arcadedb-release/qV13.sh +54cefaa2248430143aafea606b211abf82cef7230e6b0becee6a7b34a9f67cf9 arcadedb-release/qV14.sh +7ef382f657a9ce20966dbc9781163873c5577aaed60c7c300e3e7e2ab5702ec7 arcadedb-release/qV15.sh +ba32dd19e84a04c71fe94720884ce7e43eae9505a2d3fe56f2ad6544e3f577d6 arcadedb-release/qV16.sh +a71c482b5bffe1d0faf15eb14c99b10442b1a02e1723605bfa9860910c90926d arcadedb-release/qV17.sh +eca51330cb01aa21498547c00cf31c8350bb35716098360b5f36ca3ee8b314ca arcadedb-release/qV18.sh +07c7f32a72ae392c9ec8e60ec8ee3580f899fbde17b33f271db786ee7419ea1c arcadedb-release/qV19.sh +dadc9d0677a8d0ab7ae4a85240356b645f44700b072f4fd749a3e8cdbd242965 arcadedb-release/qV20.sh +9e7909ae10422bbb94d8f1abeada8ee5af4d15e308e35a61140f500b7efa8034 arcadedb-release/qV21.sh +90deeb9696e59b52d238c1c2e64c6a301494c23b2574e7d6d623885629d35ade arcadedb-release/qV22.sh +066f37aa8aaa85ca7371369abf875e95b58cb6921f05c62576dd66bc5ff71f22 arcadedb-release/qV23.sh +2bfa57078d37f5cb3cbd5cb5690ed949be0f0d26ad066a9e701292437dfc520c arcadedb-release/qV24.sh +2124facae9ca44e6ef708215f1cd038915e743e0ba1cd532f3fea92d0ad5fa1e arcadedb-release/qV25.sh +bd0af3ff8ede9a92231d4e4211540bc857697870c4a980b0e85f9eb1b13ec425 arcadedb-release/qV26.sh +07424273f1d3ce02065ba1467923e1a2f93ccc71a6a4cf6641f97f9f17313ca3 arcadedb-release/qV27.sh +3aadf894847187bd20b730835bd4907bd875d62f882d83fae41b50e52e43d944 arcadedb-release/qV28.sh +216d90dc14e021b7e63ab8dc19f4dffb7b77bb0c02a1ffa8cbcfa3424cbefc0d arcadedb-release/qV29.sh +b208a5630458f98ccbea29a50560eb303db55a1426f6115cb6236bffae135611 arcadedb-release/qV30.sh +dd645fc4fb6f371a35fe64c1e3ac5ef964c1360e2fb10dee398c4a79f9ee2154 arcadedb-release/qV31.sh +c9c13c5ef9a0fc5b99831fce0461643943a9526419db8981eaf58af8f02c0aa6 arcadedb-release/qV32.sh +8c0e1b53173800097b2311eb3fcd4499bda5a96de2965910249aef3cce2b3a1c arcadedb-release/qV33.sh +498ad98759ddbcff598f6d9461b60c9a06223bca2445d8eddb54fce982867eb2 arcadedb-release/qV34.sh +9cf8a83c1fd3694a590342f2ba439518c004e9182f52bf579bbece02039ca9a6 arcadedb-release/qV35.sh +090aeec74126feaa72e55a3d9644ba784116d36e0e824a2fc0868ffba9007d0d arcadedb-release/qV36.sh +f1c22c68cf374ee38175f79707168d676b966f35e597b4505d0acdced98a9be9 arcadedb-release/qV37.sh +53ff2148d50bfee0e80f675aa04ddf51f00d57c7f131b0b3aa3a0530fda4ed14 arcadedb-release/qV38.sh e275d1d43aa0aa37f3706a4c76c6f48f4332823b9b6a58caf0f14f585d9fd871 comparators-no-tiers/qS01.sh 644cdc3ede592f651eed80c87fd067388464a1a0e9c5ddc748e25a65123a3944 comparators-no-tiers/qS02.sh 09e8c2015781560aecaac76aed575f128999629f420ed979347b00e5e260e057 comparators-no-tiers/qS03.sh diff --git a/benchmarks/experiments/test_arcadedb_import.py b/benchmarks/experiments/test_arcadedb_import.py index e07ff236cd4..c7cbabaf402 100644 --- a/benchmarks/experiments/test_arcadedb_import.py +++ b/benchmarks/experiments/test_arcadedb_import.py @@ -52,6 +52,14 @@ def test_falls_back_to_the_old_package(reset): assert AI.jvm_module() is old.jvm +def test_loaded_backend_reads_none_until_the_package_is_loaded(reset): + native = _fake("arcadedb", __backend__="native") + reset(arcadedb=native) + assert AI.loaded_backend() is None # nothing imported by this harness yet: an explicit None on the row + AI.load() + assert AI.loaded_backend() == "native" + + def test_a_namespace_directory_called_arcadedb_is_not_the_engine(reset): empty = types.ModuleType("arcadedb") # what an empty `arcadedb/` directory on sys.path imports as empty.jvm = types.ModuleType("arcadedb.jvm") diff --git a/benchmarks/experiments/test_engine_pin.py b/benchmarks/experiments/test_engine_pin.py index 4e9396551a3..8fb1135b66a 100644 --- a/benchmarks/experiments/test_engine_pin.py +++ b/benchmarks/experiments/test_engine_pin.py @@ -40,6 +40,9 @@ def run(cmd, **kw): if spec[0] == "embedded": assert "python3" in cmd return SimpleNamespace(returncode=0, stdout=f"{spec[1]}\n{' '.join(spec[2])}\n", stderr="") + if spec[0] == "embedded3": # the probe's three lines: version, distribution, jars (26.11.1+ names) + assert "python3" in cmd + return SimpleNamespace(returncode=0, stdout=f"{spec[1]}\n{spec[2]}\n{' '.join(spec[3])}\n", stderr="") assert "sh" in cmd return SimpleNamespace(returncode=0, stdout="\n".join(spec[1]) + "\n", stderr="") @@ -95,10 +98,38 @@ def test_matching_images_pass_and_the_readback_names_what_was_read(): run = fake_docker({EMB: GOOD_EMB, SRV: GOOD_SRV}) readback, problems = E.verify(PIN, [EMB], [SRV], run=run) assert problems == [] - assert readback[EMB] == {"wheel": PIN, "engine_jar": [PIN]} and readback[SRV] == {"engine_jar": [PIN]} + # a two-line probe answer is the old `arcadedb-embedded`, a JVM build + assert readback[EMB] == {"wheel": PIN, "dist": "arcadedb-embedded", "backend": "jvm", "engine_jar": [PIN]} + assert readback[SRV] == {"engine_jar": [PIN]} assert len(run.calls) == 2 +def test_the_new_distributions_are_read_back_as_native_or_jvm(): + pin = "26.11.1.dev0" + native = ("embedded3", pin, "arcadedb", ["arcadedb-engine-26.11.1-SNAPSHOT.jar"]) # no jars dir: _build_info.ENGINE_VERSION + jvm = ("embedded3", pin, "arcadedb-jvm", ["arcadedb-engine-26.11.1-SNAPSHOT.jar", "arcadedb-network-26.11.1-SNAPSHOT.jar", "x.jar"]) + n_img, j_img = f"dbbench:arcadedb-{pin}", "dbbench:arcadedb-jvm-img" + run = fake_docker({n_img: native, j_img: jvm}) + readback, problems = E.verify(pin, [n_img], [], run=run) + assert problems == [] and readback[n_img]["backend"] == "native" and readback[n_img]["dist"] == "arcadedb" + # the other image's tag is not a versioned ArcadeDB tag, so the check_image_tag refusal is what is read here; the probe itself: + got, err = E.read_embedded(j_img, run) + assert err is None and got["backend"] == "jvm" and got["engine_jar"] == [pin] + # a native image whose engine is another version is refused with the distribution named + stale = ("embedded3", pin, "arcadedb", ["arcadedb-engine-26.11.0.jar"]) + _, problems = E.verify(pin, [n_img], [], run=fake_docker({n_img: stale})) + assert any("engine jar 26.11.0" in p for p in problems) + + +def test_the_probe_in_the_image_resolves_the_new_names_first(): + # not run here (it runs in the image): pin the order the probe reads, the same as _arcadedb_import.py + assert "('arcadedb', 'arcadedb_embedded')" in E._EMBEDDED_PROBE + assert "('arcadedb', 'arcadedb-jvm', 'arcadedb-embedded')" in E._EMBEDDED_PROBE + compile(E._EMBEDDED_PROBE, "probe", "exec") + assert E.pin_from_wheel("/x/arcadedb_jvm-26.11.1.dev0-py3-none-manylinux_2_28_x86_64.whl") == "26.11.1.dev0" + assert E.pin_from_wheel("/x/arcadedb-26.11.1.dev0-py3-none-manylinux_2_28_x86_64.whl") == "26.11.1.dev0" + + def test_the_incident_an_image_that_carries_the_retired_version_is_refused(): # the tag says 26.10.2 (a retag, a stale layer) but the image holds 26.10.1 old = ("embedded", "26.10.1", ["arcadedb-engine-26.10.1.jar"]) diff --git a/benchmarks/experiments/test_jpype_pin.py b/benchmarks/experiments/test_jpype_pin.py index 7fec195c6c9..c8f748afbf5 100644 --- a/benchmarks/experiments/test_jpype_pin.py +++ b/benchmarks/experiments/test_jpype_pin.py @@ -153,6 +153,19 @@ def test_wheel_guard_accepts_a_chunked_wheel_and_refuses_the_whole_input_one(tmp assert r.returncode == 1 and "cannot be opened" in r.stderr +def test_wheel_guard_reads_the_new_package_name(tmp_path): + """arcadedb (native) and arcadedb-jvm (26.11.1+) carry arcadedb/core.py, not arcadedb_embedded/core.py.""" + import zipfile + guard = [sys.executable, str(HERE / "wheel_guard.py")] + for name, marker, rc in (("arcadedb-26.11.1.dev0-py3-none-x.whl", "_INSERT_MANY_CHUNK = 1000\n", 0), + ("arcadedb_jvm-26.11.1.dev0-py3-none-x.whl", "_INSERT_MANY_CHUNK = 10000\n", 0), + ("arcadedb_jvm-26.10.1-py3-none-x.whl", "def insert_many(rows): ...\n", 1)): + p = tmp_path / name + with zipfile.ZipFile(p, "w") as z: + z.writestr("arcadedb/core.py", marker) + assert subprocess.run(guard + [str(p)], capture_output=True, text=True).returncode == rc, name + + def test_build_images_refuses_a_wheel_without_the_chunked_insert_many(tmp_path): old = _fake_wheel(tmp_path / "arcadedb_embedded-26.10.1-cp312-cp312-manylinux_2_34_x86_64.whl", chunked=False) for args in (["arcadedb"], ["duckdb"]): # the generic images carry the wheel too diff --git a/benchmarks/experiments/test_queue_lint.py b/benchmarks/experiments/test_queue_lint.py index 51ea64df5ec..8cb81eb1996 100644 --- a/benchmarks/experiments/test_queue_lint.py +++ b/benchmarks/experiments/test_queue_lint.py @@ -156,6 +156,13 @@ def probs(head, line): _check("pre-release wheel with BENCH_ALLOW_DEV=0 is flagged", len(probs(dev, "BENCH_ALLOW_DEV=0 ./build_images.sh client")), 1) _check("pre-release wheel with no assignment is flagged", len(probs(dev, "./build_images.sh client")), 1) + # the 26.11.1 package names: arcadedb (native) and arcadedb_jvm, a version that starts with a digit + for who in ("arcadedb", "arcadedb_jvm"): + dev2 = f'W=dist/{who}-26.11.1.dev0-py3-none-manylinux_2_28_x86_64.whl\n' + stable2 = f'W=dist/{who}-26.11.1-py3-none-manylinux_2_28_x86_64.whl\n' + _check(f"{who} pre-release wheel with BENCH_ALLOW_DEV=0 is flagged", len(probs(dev2, "BENCH_ALLOW_DEV=0 ./build_images.sh client")), 1) + _check(f"{who} stable wheel with BENCH_ALLOW_DEV=0 holds", probs(stable2, "BENCH_ALLOW_DEV=0 ./build_images.sh client"), []) + _check("a jar name is not a wheel", queue_lint.wheel_is_prerelease("ls arcadedb-engine-26.11.1-SNAPSHOT.jar"), None) _check("no wheel named: only 1 holds", (len(probs("", "BENCH_ALLOW_DEV=0 ./build_images.sh client")), len(probs("", "BENCH_ALLOW_DEV=1 ./build_images.sh client"))), (1, 0)) diff --git a/benchmarks/experiments/verify_pair_c25.sh b/benchmarks/experiments/verify_pair_c25.sh index b716a213d94..5d12639a9e6 100755 --- a/benchmarks/experiments/verify_pair_c25.sh +++ b/benchmarks/experiments/verify_pair_c25.sh @@ -57,13 +57,16 @@ print(next(l.split('=',1)[1].strip() for l in p.splitlines() if l.strip().starts IJ=$(docker run --rm --entrypoint sh "$IMG" -c 'java -version 2>&1' | head -1 | grep -oE '"[0-9]+' | tr -d '"') # The venv's JRE when this checkout has one (the bench host does); otherwise the JRE inside the # embedded image, which is the JVM the embedded arm actually runs (a worktree has no .venv). -VENV_JAVA="$REPO/.venv/lib/python3.12/site-packages/arcadedb_embedded/jre/bin/java" -if [ -x "$VENV_JAVA" ]; then +VENV_JAVA="" +for _p in arcadedb arcadedb_embedded; do # arcadedb-jvm (26.11.1+) installs `arcadedb`; the older wheel `arcadedb_embedded` + [ -x "$REPO/.venv/lib/python3.12/site-packages/$_p/jre/bin/java" ] && VENV_JAVA="$REPO/.venv/lib/python3.12/site-packages/$_p/jre/bin/java" +done +if [ -n "$VENV_JAVA" ]; then EJ=$("$VENV_JAVA" -version 2>&1 | head -1 | grep -oE '"[0-9]+' | tr -d '"') else EJ=$(docker run --rm --entrypoint python3 "$EMBEDDED_IMAGE" -c " import importlib.metadata as m, pathlib, subprocess -d = pathlib.Path(m.distribution('arcadedb-embedded').locate_file('arcadedb_embedded')) +d = next(pathlib.Path(m.distribution(n).locate_file(p)) for n, p in (('arcadedb-jvm', 'arcadedb'), ('arcadedb', 'arcadedb'), ('arcadedb-embedded', 'arcadedb_embedded')) if any(m.distributions(name=n))) print(subprocess.run([str(d / 'jre' / 'bin' / 'java'), '-version'], capture_output=True, text=True).stderr.splitlines()[0])" \ 2>/dev/null | grep -oE '"[0-9]+' | tr -d '"') fi @@ -81,7 +84,7 @@ fi if docker image inspect "$EMBEDDED_IMAGE" >/dev/null 2>&1; then BSHA=$(docker run --rm --entrypoint python3 "$EMBEDDED_IMAGE" -c " import importlib.metadata as m, pathlib, zipfile, io, sys -d = pathlib.Path(m.distribution('arcadedb-embedded').locate_file('arcadedb_embedded')) +d = next(pathlib.Path(m.distribution(n).locate_file(p)) for n, p in (('arcadedb-jvm', 'arcadedb'), ('arcadedb', 'arcadedb'), ('arcadedb-embedded', 'arcadedb_embedded')) if any(m.distributions(name=n))) j = next(x for x in (d / 'jars').glob('arcadedb-engine-*.jar')) z = zipfile.ZipFile(j) p = z.read('com/arcadedb/arcadedb.properties').decode() diff --git a/benchmarks/experiments/wheel_guard.py b/benchmarks/experiments/wheel_guard.py index a20e6409a5f..20e2d4d2259 100644 --- a/benchmarks/experiments/wheel_guard.py +++ b/benchmarks/experiments/wheel_guard.py @@ -3,7 +3,8 @@ The 26.10.1 wheel's insert_many turned one input into one JSON text and parsed it into one Java array, so a large time-series document load held the whole input in memory and the kernel killed l4/ts1000/arcadedb_ts_doc (2026-10-08). #294/#295 chunked it, -first shipped in 26.10.2: a chunked wheel has `_INSERT_MANY_CHUNK` in arcadedb_embedded/core.py. Rows measured on a wheel without +first shipped in 26.10.2: a chunked wheel has `_INSERT_MANY_CHUNK` in its core.py (`arcadedb/core.py` in the `arcadedb` and `arcadedb-jvm` +wheels, `arcadedb_embedded/core.py` in the older one). Rows measured on a wheel without it are a memory artefact, not the engine's number, so no benchmark image may carry one. Usage: wheel_guard.py WHEEL exit 0 when chunked, 1 with the reason and the fix when not (or when the file is no wheel). @@ -12,21 +13,23 @@ import zipfile MARKER = "_INSERT_MANY_CHUNK" -CORE = "arcadedb_embedded/core.py" -FIX = "use a 26.10.2 or newer arcadedb_embedded wheel (the first to ship the chunked insert_many, #294/#295)" +CORES = ("arcadedb/core.py", "arcadedb_embedded/core.py") # new package name first, as _arcadedb_import.py +FIX = "use a 26.10.2 or newer arcadedb, arcadedb-jvm or arcadedb_embedded wheel (the first to ship the chunked insert_many, #294/#295)" def check(wheel): """Return None when the wheel's core.py has the chunking marker, else the reason it is refused.""" try: with zipfile.ZipFile(wheel) as z: - if CORE not in z.namelist(): - return f"{wheel} has no {CORE}, so it is not an arcadedb_embedded wheel" - if MARKER in z.read(CORE).decode("utf-8", "replace"): + names = z.namelist() + core = next((c for c in CORES if c in names), None) + if core is None: + return f"{wheel} has none of {', '.join(CORES)}, so it is not an ArcadeDB bindings wheel" + if MARKER in z.read(core).decode("utf-8", "replace"): return None except (zipfile.BadZipFile, OSError) as e: return f"{wheel} cannot be opened as a wheel ({e})" - return f"{wheel} has no {MARKER} in {CORE}: its insert_many materialises the whole input (pre-26.10.2)" + return f"{wheel} has no {MARKER} in {core}: its insert_many materialises the whole input (pre-26.10.2)" def main(argv): From 6eac541c63a7c99a2f93e3b79897e10a43325fa4 Mon Sep 17 00:00:00 2001 From: Taewoon Kim Date: Fri, 30 Oct 2026 02:50:05 +0900 Subject: [PATCH 03/23] benchmarks: native and JVM builds of one release are two arms (identity, tags, stages, native settings) An ArcadeDB in-process arm is now (engine_version, engine_backend), rows without the field count as the JVM build (arm_identity.py). The freeze key (make_paper_tables, merge_campaign) carries the build, a freeze names one build (BENCH_ENGINE_BACKEND), a cell holding both is refused, the exporter refuses a pooled entry, the lifecycle table reads engine_load_ms where there is no JVM start, page_check names the new fields, version_consistency_check reads "(native)"/"(jvm)" as one release. Image tags carry the build (dbbench:arcadedb--native / -jvm; the unsuffixed tag still reads for arcadedb_embedded wheels); engine_pin, runner, build_images.sh, the stage template and verify_pair_c25.sh follow, and verify_pair says what it cannot check on a native wheel. One stage chain per wheel (the generator was already per wheel; the stage scripts now derive the tag suffix from it). On native: jvm_kwargs become ARCADEDB_NATIVE_ISOLATE_ARGS (-Xmx, -D settings, -Xms) before the isolate exists and the effective string is recorded as native_isolate_args; the durability readback asks the engine in SQL (schema:database), the same path on both builds. Co-Authored-By: Claude Sonnet 5.5 --- benchmarks/experiments/_arcadedb_import.py | 79 +++ benchmarks/experiments/arm_identity.py | 99 ++++ benchmarks/experiments/bench_common.py | 52 +- benchmarks/experiments/build_images.sh | 12 +- benchmarks/experiments/e2_hybrid.py | 2 +- benchmarks/experiments/engine_pin.py | 51 +- benchmarks/experiments/export_web.py | 15 +- benchmarks/experiments/l1_tpc.py | 2 +- benchmarks/experiments/l2_graph.py | 2 +- benchmarks/experiments/l5_lifecycle.py | 4 + benchmarks/experiments/make_2611_stages.py | 23 +- benchmarks/experiments/make_paper_tables.py | 23 +- benchmarks/experiments/merge_campaign.py | 10 +- benchmarks/experiments/page_check.py | 8 +- benchmarks/experiments/runner.py | 10 +- .../experiments/stage_drafts_2611.sha256 | 456 +++++++++--------- benchmarks/experiments/test_engine_pin.py | 2 +- benchmarks/experiments/test_jpype_pin.py | 13 +- .../experiments/test_native_jvm_arms.py | 251 ++++++++++ benchmarks/experiments/verify_pair_c25.sh | 25 +- .../experiments/version_consistency_check.py | 2 + 21 files changed, 866 insertions(+), 275 deletions(-) create mode 100644 benchmarks/experiments/arm_identity.py create mode 100644 benchmarks/experiments/test_native_jvm_arms.py diff --git a/benchmarks/experiments/_arcadedb_import.py b/benchmarks/experiments/_arcadedb_import.py index 4cbec597417..4350f92db69 100644 --- a/benchmarks/experiments/_arcadedb_import.py +++ b/benchmarks/experiments/_arcadedb_import.py @@ -34,6 +34,7 @@ """ import importlib import importlib.metadata +import os # Import names, in order. The first one that is the real package wins. MODULES = ("arcadedb", "arcadedb_embedded") @@ -42,6 +43,18 @@ _loaded = None +# THE NATIVE BUILD HAS NO JVM, SO NO `jvm_kwargs`: the native package ignores `heap_size` and `jvm_args` (with a warning) and reads its isolate +# options from this environment variable when the isolate is created (arcadedb._native.lib). The lanes still pass the JVM spelling, because that +# is the one code path for both packages, so on native `load()` wraps create_database/open_database and turns the same kwargs into isolate +# arguments before the first call: heap_size -> -Xmx, and the -D and -Xms/-Xmx entries of jvm_args as they are +# (the engine reads its settings, such as arcadedb.txWalFlush, from system properties; checked on the 26.10.2 native wheel: -Darcadedb.txWalFlush=1 +# in the isolate args shows value 1 in `SELECT settings FROM schema:database`). Other JVM flags (-XX:...) have no native meaning and are left out; +# they are recorded in `native_dropped`. An argument already present in the variable wins, as it does inside the package. +NATIVE_ENV = "ARCADEDB_NATIVE_ISOLATE_ARGS" +_PASS_PREFIXES = ("-D", "-Xms", "-Xmx") +native_dropped = [] +_isolate_created = False + def _is_engine(mod): # `arcadedb` could also resolve as an empty namespace directory on sys.path; require the API. @@ -65,6 +78,7 @@ def load(): continue if _is_engine(mod): _loaded = mod + _wrap_native(mod) return mod errors.append(f"{name}: imported but is not the ArcadeDB package ({getattr(mod, '__file__', None)})") raise ImportError("no ArcadeDB Python package: install arcadedb or arcadedb-jvm (or the older arcadedb-embedded); " @@ -110,3 +124,68 @@ def dist_version(): except importlib.metadata.PackageNotFoundError as e: last = e raise last + + +def _arg_key(tok): + """The part of an isolate argument that names the setting: `-Darcadedb.txWalFlush=1` -> `-Darcadedb.txWalFlush`, `-Xmx4g` -> `-Xmx`.""" + if tok.startswith("-D"): + return tok.split("=", 1)[0] + return tok[:4] if tok[:4] in ("-Xms", "-Xmx") else tok + + +def native_isolate_args(heap_size=None, jvm_args=None): + """The isolate arguments that mean what a JVM `heap_size` / `jvm_args` pair means; ([kept], [dropped]). Pure, no environment.""" + toks = [f"-Xmx{heap_size}"] if heap_size else [] + ja = jvm_args.split() if isinstance(jvm_args, str) else list(jvm_args or []) + dropped = [t for t in ja if not t.startswith(_PASS_PREFIXES)] + toks += [t for t in ja if t.startswith(_PASS_PREFIXES)] + return toks, dropped + + +def apply_native(jvm_kwargs=None, environ=None): + """Put the isolate arguments for `jvm_kwargs` into ARCADEDB_NATIVE_ISOLATE_ARGS and return the effective string. + + Merge rule: a setting already in the variable wins (the caller or the package's own default is not overridden), a new one is appended, + two entries for one setting in `jvm_kwargs` keep the first. Meaningful only before the isolate exists; see `_wrap_native`. + """ + env = os.environ if environ is None else environ + kw = jvm_kwargs or {} + new, dropped = native_isolate_args(kw.get("heap_size"), kw.get("jvm_args")) + have = env.get(NATIVE_ENV, "").split() + keys = {_arg_key(t) for t in have} + for t in new: + if _arg_key(t) not in keys: + have.append(t) + keys.add(_arg_key(t)) + for t in dropped: + if t not in native_dropped: + native_dropped.append(t) + env[NATIVE_ENV] = " ".join(have) + return env[NATIVE_ENV] + + +def effective_native_args(): + """The isolate arguments this process runs with ("" when none), or None when it is not a native process. What a row records.""" + return os.environ.get(NATIVE_ENV, "") if _loaded is not None and backend() == "native" else None + + +def _wrap_native(mod): + """On the native build, make `mod.create_database` / `mod.open_database` apply their `jvm_kwargs` as isolate arguments (see NATIVE_ENV).""" + if (getattr(mod, "__backend__", None) or getattr(mod, "_DISTRIBUTION_BACKEND", None) or "jvm") != "native": + return + for name in ("create_database", "open_database"): + fn = getattr(mod, name, None) + if fn is None or getattr(fn, "_bench_native_wrapped", False): + continue + + def wrapped(*a, __fn=fn, **kw): + global _isolate_created + if not _isolate_created: # the isolate takes its arguments once; a later call cannot change them (the JVM cannot restart either) + apply_native(kw.get("jvm_kwargs")) + _isolate_created = True + return __fn(*a, **kw) + + wrapped._bench_native_wrapped = True + wrapped.__name__ = getattr(fn, "__name__", name) + wrapped.__doc__ = getattr(fn, "__doc__", None) + setattr(mod, name, wrapped) diff --git a/benchmarks/experiments/arm_identity.py b/benchmarks/experiments/arm_identity.py new file mode 100644 index 00000000000..b63b13d2f31 --- /dev/null +++ b/benchmarks/experiments/arm_identity.py @@ -0,0 +1,99 @@ +"""What an ArcadeDB arm is, for every script that reads rows. + +Since 26.11.1 one release is measured on two builds of the same Python package API: `arcadedb` (native) and `arcadedb-jvm` (Java engine +through JPype). Both stamp the same `engine_version` and the same `backend` arm name (`arcadedb_sparse_embedded`, ...); only the row +field `engine_backend` ("native" or "jvm") tells them apart. A consumer that groups or dedupes on `engine_version` alone would pool or +overwrite the two builds, so the identity of an in-process ArcadeDB arm is (engine_version, build), and this module is the one +definition of `build`: + + * `engine_backend` is "native" or "jvm": that value. + * a row of an ArcadeDB in-process arm WITHOUT the field (every row before 26.11.1): "jvm", because that is what it ran. + * a served arm, the HTTP client container and every comparator: None. The served engine is a JVM server whichever Python package + drives it, and the field is null there on purpose, so it never selects or splits them. + +`select_build(rows, want)` is the one place a freeze picks a build: with BENCH_ENGINE_BACKEND=native|jvm the other build's in-process +ArcadeDB rows leave the freeze (they stay in runs.jsonl); without it, a cell that holds BOTH builds is refused by name, not pooled. +No import of ArcadeDB, no I/O: safe in every consumer, including the ones that read old rows. +""" +import os + +BUILDS = ("native", "jvm") +ENV = "BENCH_ENGINE_BACKEND" + +# Backend names that are ArcadeDB but not an in-process arm: the served engine and its client container. +_NOT_IN_PROCESS = ("server", "http_client", "imgdefaults") + + +def is_arcadedb(r): + return str(r.get("backend") or "").startswith("arcadedb") + + +def in_process(r): + b = str(r.get("backend") or "") + return b.startswith("arcadedb") and not any(t in b for t in _NOT_IN_PROCESS) + + +def build_of(r): + """"native", "jvm" or None for the row (see the module docstring).""" + eb = r.get("engine_backend") + if eb in BUILDS: + return eb + if in_process(r) and "engine_backend" not in r: + return "jvm" + return None + + +def arm_identity(r): + """(engine_version, build): what two rows must share to be one measurement of one engine.""" + return (str(r.get("engine_version") or ""), build_of(r)) + + +def identities(rows): + """The set of (engine_version, build) over the in-process ArcadeDB rows.""" + return {arm_identity(r) for r in rows if in_process(r)} + + +def wanted_build(env=None): + """The build a freeze was asked for ("native" or "jvm"), or None. A value that is neither is an error, not None.""" + v = (os.environ.get(ENV, "") if env is None else env).strip().lower() + if not v: + return None + if v not in BUILDS: + raise SystemExit(f"{ENV}={v!r}: expected one of {BUILDS}") + return v + + +def mixed_cells(rows): + """{(lane, scale, n_docs, backend): builds} for the cells whose in-process ArcadeDB rows come from more than one build.""" + seen = {} + for r in rows: + if in_process(r) and build_of(r): + seen.setdefault((r.get("lane"), r.get("scale"), r.get("n_docs"), r.get("backend")), set()).add(build_of(r)) + return {k: sorted(v) for k, v in seen.items() if len(v) > 1} + + +def select_build(rows, want=None): + """The rows a freeze may publish: one build of the in-process ArcadeDB arms, or a refusal that says why not. + + `want` None reads BENCH_ENGINE_BACKEND. With a build, rows of the other build are dropped (served arms and comparators carry no + build and stay). Without one, any cell holding both builds stops the freeze: a median over native and JVM rows names neither. + """ + want = wanted_build() if want is None else want + if want: + return [r for r in rows if not (in_process(r) and build_of(r) not in (None, want))] + mixed = mixed_cells(rows) + if mixed: + sample = ", ".join("/".join(str(x) for x in k) for k in sorted(mixed, key=str)[:3]) + raise SystemExit( + f"REFUSING: {len(mixed)} cell(s) hold rows of both ArcadeDB builds (native and jvm), e.g. {sample}. A table cell is one " + f"build; set {ENV}=native or {ENV}=jvm to freeze that build (the other build's rows stay in the log).") + return rows + + +def refuse_pooled_builds(rs, where): + """SystemExit when one entry's rows span two builds of ArcadeDB (the exporter's check, beside _one_version's).""" + ids = identities(rs) + if len({b for _v, b in ids}) > 1: + raise SystemExit(f"REFUSING: {where} pools ArcadeDB rows of different builds {sorted(ids, key=str)}; its median would " + f"belong to neither. Freeze one build ({ENV}) or split the arms.") + return rs diff --git a/benchmarks/experiments/bench_common.py b/benchmarks/experiments/bench_common.py index 4141187d2d2..19b1b33edd8 100644 --- a/benchmarks/experiments/bench_common.py +++ b/benchmarks/experiments/bench_common.py @@ -494,6 +494,11 @@ def arcade_query_ram_readback(): return {"arcadedb_query_max_heap_ram_mb": got, "arcadedb_query_max_heap_ram_source": "read from the engine (GlobalConfiguration.QUERY_MAX_HEAP_RAM)"} except Exception as e: # noqa: BLE001 + if engine_backend() == "native": + # arcadedb.queryMaxHeapRAM is SCOPE.JVM: schema:database does not list it, and the native build has no JPype to ask with. + return {"arcadedb_query_max_heap_ram_mb": n, + "arcadedb_query_max_heap_ram_source": "requested as the isolate argument -Darcadedb.queryMaxHeapRAM (a JVM-scope setting: " + "SELECT settings FROM schema:database does not list it, so the engine cannot be asked)"} return {"arcadedb_query_max_heap_ram_mb": n, "arcadedb_query_max_heap_ram_source": f"requested via the JVM argument (the engine could not be asked, {e.__class__.__name__})"} @@ -518,7 +523,30 @@ class separately (ArcadeData/arcadedb#8478): yes_nometadata is the executor's "durability_server_flags; the server exposes no read-back)") -def arcade_durability_readback(fallback_cls=None): +_ENGINE_SETTINGS = {} + + +def engine_database_settings(db): + """{key: value} of the database-scoped settings the ENGINE reports for an open database, asked in SQL so it is one code path on both + builds (the native package has no JPype): `SELECT settings FROM schema:database` lists every GlobalConfiguration entry of SCOPE.DATABASE + with the value in force for that database (checked at engine e353330c5c, FetchFromSchemaDatabaseStep, and on the native wheel). + JVM-scoped settings (arcadedb.queryMaxHeapRAM) are not in the list. + """ + row = db.query("sql", "SELECT settings FROM schema:database").first() + return {s["key"]: s["value"] for s in (row.get("settings") or [])} + + +def remember_engine_settings(db): + """Read the settings now and keep them, for a lane that closes its database before the row is stamped. Returns the dict; {} when it cannot be read.""" + try: + _ENGINE_SETTINGS.clear() + _ENGINE_SETTINGS.update(engine_database_settings(db)) + except Exception: # noqa: BLE001 + _ENGINE_SETTINGS.clear() + return dict(_ENGINE_SETTINGS) + + +def arcade_durability_readback(fallback_cls=None, db=None): """ASK THE ENGINE what txWalFlush it is running at, do not assert it. #81's standard for every durability string on a row is that it was read out @@ -530,12 +558,21 @@ def arcade_durability_readback(fallback_cls=None): Returns the string for the value the engine reports, or the asserted string with a note when the engine cannot be asked -- never silence. + + HOW IT ASKS (26.11.1, one path for both packages). txWalFlush is a SCOPE.DATABASE setting, so the engine answers in SQL + (`SELECT settings FROM schema:database`) on the JVM and on the native build alike: from `db` when the caller has the open database, else from + the snapshot `remember_engine_settings(db)` took, else, on the JVM build only, from GlobalConfiguration through JPype as before. The native + build has no JPype and so can only be asked through a database; a native caller that passes none gets the asserted string and its reason. """ want = at_class(DURABILITY_ARCADEDB, fallback_cls) try: - import jpype - gc = jpype.JClass("com.arcadedb.GlobalConfiguration") - value = int(gc.TX_WAL_FLUSH.getValue()) + settings = engine_database_settings(db) if db is not None else dict(_ENGINE_SETTINGS) + if "arcadedb.txWalFlush" in settings: + value = int(settings["arcadedb.txWalFlush"]) + else: + import jpype + gc = jpype.JClass("com.arcadedb.GlobalConfiguration") + value = int(gc.TX_WAL_FLUSH.getValue()) except Exception as e: # noqa: BLE001 return want + f" (asserted: the engine could not be asked, {e.__class__.__name__})" if value == 0: @@ -927,6 +964,13 @@ def _read(path): # `arcadedb` and `arcadedb-jvm` wheels of one release, so the version alone cannot tell the two arms apart. NOT named `backend`: # that field is already the arm name (`arcadedb_sparse_embedded`, ...). out["engine_backend"] = engine_backend() + # THE NATIVE HEAP AND SETTINGS, as the isolate received them (ARCADEDB_NATIVE_ISOLATE_ARGS, e.g. "-Xmx4g -Darcadedb.txWalFlush=0"): the + # native counterpart of the JVM's -Xmx/-D arguments, which the native package ignores. None on the JVM build and for every other arm. + try: + import _arcadedb_import + out["native_isolate_args"] = _arcadedb_import.effective_native_args() + except Exception: + out["native_isolate_args"] = None out.update(extra) return out diff --git a/benchmarks/experiments/build_images.sh b/benchmarks/experiments/build_images.sh index e141738419d..0f0c1d145c0 100755 --- a/benchmarks/experiments/build_images.sh +++ b/benchmarks/experiments/build_images.sh @@ -1,6 +1,6 @@ #!/usr/bin/env bash # Build the per-backend bench images. ARCADEDB_WHEEL= bakes a wheel into the arcadedb image, which is tagged with -# the wheel's version, dbbench:arcadedb-, and never with a floating dbbench:arcadedb (engine_pin.py, 2026-10-22). +# the wheel's version and build, dbbench:arcadedb--native or -jvm (the older arcadedb_embedded wheel: no suffix), and never with a floating dbbench:arcadedb (engine_pin.py, 2026-10-22). set -eu cd "$(dirname "$0")" @@ -140,10 +140,18 @@ echo "arcadedb pin: $_pin" # NO FLOATING ENGINE TAG (2026-10-22, engine_pin.py). dbbench:arcadedb pointed at whichever wheel was built last, and on the laptop it still # pointed at 26.10.1 after October was retired. The image that carries ArcadeDB is named for the version it carries, and runner.py refuses # any other tag for it. +# THE BUILD IS PART OF THE TAG (26.11.1, engine_pin.backend_from_wheel): one release ships a native wheel (arcadedb--...) and a JVM wheel +# (arcadedb_jvm--...), and `dbbench:arcadedb-` could hold only one. The tag is dbbench:arcadedb--native or -jvm; the older +# arcadedb_embedded wheels keep the unsuffixed tag, so the images and rows of earlier releases still resolve. +_bk="" +case "$(basename "${ARCADEDB_WHEEL:-}" | cut -d- -f1)" in + arcadedb) _bk=native ;; + arcadedb_jvm) _bk=jvm ;; +esac ARCADEDB_TAG="" if [ -n "$_pin" ]; then case "$_pin" in - [0-9]*.[0-9]*.[0-9]*) ARCADEDB_TAG="dbbench:arcadedb-$_pin" ;; + [0-9]*.[0-9]*.[0-9]*) ARCADEDB_TAG="dbbench:arcadedb-$_pin${_bk:+-$_bk}" ;; *) echo "REFUSING: the wheel name '$(basename "$ARCADEDB_WHEEL")' carries no x.y.z version, so the arcadedb image cannot be tagged" >&2; exit 1 ;; esac fi diff --git a/benchmarks/experiments/e2_hybrid.py b/benchmarks/experiments/e2_hybrid.py index ecc250adb2f..a7ff3eef569 100644 --- a/benchmarks/experiments/e2_hybrid.py +++ b/benchmarks/experiments/e2_hybrid.py @@ -249,7 +249,7 @@ def __init__(self): # THE WHEEL'S version, not its name (#156). A row stamped # "arcadedb-embedded" cannot be re-measured by anyone including us. self.version = _arcadedb_import.dist_version() - self.durability = bench_common.arcade_durability_readback() + self.durability = bench_common.arcade_durability_readback(db=self.db) def build(self, vecs, edges): db, a = self.db, self._a diff --git a/benchmarks/experiments/engine_pin.py b/benchmarks/experiments/engine_pin.py index 87c9c51d057..973c95d4b18 100644 --- a/benchmarks/experiments/engine_pin.py +++ b/benchmarks/experiments/engine_pin.py @@ -14,7 +14,9 @@ served the engine jar of the server image (`/home/arcadedb/lib/arcadedb-engine-.jar`), which is the engine the server loads. Nothing is started to ask, so the check costs two short `docker run`s per image and measures nothing. -NO FLOATING ENGINE TAGS. An image that carries ArcadeDB is tagged `dbbench:arcadedb-` (build_images.sh writes it) and the runner refuses +NO FLOATING ENGINE TAGS. An image that carries ArcadeDB is tagged `dbbench:arcadedb--native` or `-jvm` (build_images.sh writes it, the suffix +is the build of the pinned wheel: backend_from_wheel; the older unsuffixed `dbbench:arcadedb-` still reads, for the arcadedb_embedded wheels and +the rows made on them, and is refused for a native or JVM wheel) and the runner refuses any other tag for it, `dbbench:arcadedb` included: a floating tag points at whatever was built last, which is how a retired version stayed reachable. `dbbench:arcadedb` survives only as the logical build target name in runner.BACKENDS; the runner swaps it for the versioned tag from the pin, and a batch with an ArcadeDB arm and no pin is refused. The comparator images (client, duckdb, dense, pg-age, mongo-search, composed) keep @@ -63,27 +65,45 @@ def pin_from_wheel(path): return parts[1] if len(parts) > 2 and re.fullmatch(VERSION_RE, parts[1]) else "" -def versioned_image(version): - return f"{LOGICAL_IMAGE}-{version}" +def backend_from_wheel(path): + """"native" for arcadedb--*.whl, "jvm" for arcadedb_jvm--*.whl, "" for the older arcadedb_embedded wheel (or any other name). + + The build is part of the image tag since 26.11.1: one release has a native and a JVM wheel, and one tag cannot hold both. + """ + head = os.path.basename(path or "").split("-")[0].replace("_", "-") + return {"arcadedb": "native", "arcadedb-jvm": "jvm"}.get(head, "") + + +def versioned_image(version, backend=""): + """dbbench:arcadedb--native / -jvm; the older unsuffixed dbbench:arcadedb- when no build is named.""" + return f"{LOGICAL_IMAGE}-{version}" + (f"-{backend}" if backend else "") def build_target(image): """The build_images.sh target an image name stands for: dbbench:arcadedb-26.10.2 -> arcadedb, dbbench:duckdb -> duckdb.""" tag = image.split(":", 1)[1] - return TARGET if re.fullmatch(rf"{TARGET}(?:-{VERSION_RE})?", tag) else tag + return TARGET if re.fullmatch(rf"{TARGET}(?:-{VERSION_RE}(?:-(?:native|jvm))?)?", tag) else tag -def check_image_tag(image, pin=None): - """None when `image` may run, else why it is refused. Only `dbbench:*` names are judged here (digest-pinned third-party images are not ours).""" +def check_image_tag(image, pin=None, backend=""): + """None when `image` may run, else why it is refused. Only `dbbench:*` names are judged here (digest-pinned third-party images are not ours). + + `backend` ("native"/"jvm", from backend_from_wheel) is the build the stage pinned: the tag must then end in it. An unsuffixed tag is the + pre-26.11.1 form and is accepted only when no build is named (`backend` empty), so an old image can never stand in for a native or JVM wheel. + """ if not image.startswith("dbbench:"): return None tag = image.split(":", 1)[1] if tag.startswith(TARGET): - if not re.fullmatch(rf"{TARGET}-{VERSION_RE}", tag): + m = re.fullmatch(rf"{TARGET}-({VERSION_RE})(?:-(native|jvm))?", tag) + if not m: return (f"{image} is an ArcadeDB image tag without a version. A floating tag points at whatever was built last; " - f"the tag must be {LOGICAL_IMAGE}-") - if pin and tag != f"{TARGET}-{pin}": - return f"{image} names ArcadeDB {tag[len(TARGET) + 1:]}, the stage pins {pin}" + f"the tag must be {LOGICAL_IMAGE}--native or -jvm") + if pin and m.group(1) != pin: + return f"{image} names ArcadeDB {m.group(1)}, the stage pins {pin}" + if backend and m.group(2) != backend: + return (f"{image} is the {m.group(2) or 'unsuffixed (pre-26.11.1)'} image, the stage pins the {backend} build " + f"({versioned_image(pin or '', backend)})") return None if tag not in FLOATING_OK: return f"{image} is not a known benchmark image ({', '.join(FLOATING_OK)}, or {LOGICAL_IMAGE}-)" @@ -137,13 +157,16 @@ def read_served(image, run=None): return {"engine_jar": _engine_jar_versions(out.split())}, None -def verify(pin, embedded_images=(), served_images=(), run=None): - """(readback, problems): what each image reports, and one sentence per way it is not the pin. Empty `problems` means every image runs `pin`.""" +def verify(pin, embedded_images=(), served_images=(), run=None, backend=""): + """(readback, problems): what each image reports, and one sentence per way it is not the pin. Empty `problems` means every image runs `pin`. + + With `backend` ("native"/"jvm") an embedded image must also carry that build's distribution (arcadedb / arcadedb-jvm), read out of the image. + """ problems, readback = [], {} if not pin: return readback, ["no pinned version: ARCADEDB_WHEEL is unset or its file name carries no version"] for image in sorted(set(embedded_images)): - why = check_image_tag(image, pin) + why = check_image_tag(image, pin, backend) got, err = (None, why) if why else read_embedded(image, run) readback[image] = got if err: @@ -151,6 +174,8 @@ def verify(pin, embedded_images=(), served_images=(), run=None): continue if got["wheel"] != pin: problems.append(f"{image} has {got['dist']} {got['wheel']}, the stage pins {pin}") + if backend and got["backend"] != backend: + problems.append(f"{image} carries {got['dist']} ({got['backend']} build), the stage pins the {backend} build") if got["engine_jar"] != [pin]: problems.append(f"{image} ships engine jar {','.join(got['engine_jar']) or '(none)'}, the stage pins {pin}") for image in sorted(set(served_images)): diff --git a/benchmarks/experiments/export_web.py b/benchmarks/experiments/export_web.py index 77a50dd92fd..465f4dac8d6 100644 --- a/benchmarks/experiments/export_web.py +++ b/benchmarks/experiments/export_web.py @@ -46,6 +46,7 @@ from runner import BACKENDS, MEM_BY_SCALE, HEAP_BY_SCALE # noqa: E402 (path set above) import bench_common # noqa: E402 +import arm_identity as _AI # noqa: E402 # THE PQ ARMS CAN HAVE ROWS EVEN WHEN THEY ARE NOT MEASURING. runner registers them only under BENCH_DENSE_PQ=1 (a stage # must not start a PQ cell by accident), but a freeze that holds their rows is read here without that variable, and their @@ -281,6 +282,7 @@ def _one_version(rs, where): is indistinguishable from a correct row to every other gate -- the payload carries one string either way, so version_consistency_check sees nothing. """ + _AI.refuse_pooled_builds(rs, where) # (engine_version, build) is the arm: native and jvm rows of one release are two arms seen = {str(r.get("engine_version")) for r in rs if r.get("engine_version")} if len(seen) > 1: raise SystemExit( @@ -2934,7 +2936,11 @@ def _engine_of(r): by.setdefault((r.get("workload"), r.get("scale"), _srv, _engine_of(r)), []).append(r) entries = [] - _lc_columns = (["JVM start ms", "first open ms", "cold process ms"] + # The engine's own start step: start_jvm() on the JVM build (`jvm_start_ms`), which the native build has no counterpart of (null); + # there the same step is `engine_load_ms` (library load and isolate creation). A page is one build (select_build), so one column. + _start_field, _start_label = (("engine_load_ms", "engine load ms") + if any(_AI.build_of(r) == "native" for r in rows) else ("jvm_start_ms", "JVM start ms")) + _lc_columns = ([_start_label, "first open ms", "cold process ms"] + [LIFECYCLE_SCENARIO_LABELS[k] for k in LIFECYCLE_PAGE_SCENARIOS]) for (situation, scale, _srv, engine), rs in sorted(by.items()): if situation in LIFECYCLE_WITHHELD: @@ -2963,7 +2969,7 @@ def _engine_of(r): # session number: a 5 ms open inside a process that took half a second to # reach its first database call is not a 5 ms cost to anyone launching a # CLI. - for field, label in (("jvm_start_ms", "JVM start ms"), + for field, label in ((_start_field, _start_label), ("first_open_ms", "first open ms"), ("cold_process_ms", "cold process ms")): got = _agg(rs, field) @@ -2990,7 +2996,7 @@ def _engine_of(r): _trs = twin_by.get((situation, scale, _srv, engine)) if _trs and entry["metrics"]: twin_metrics = {} - for field, label in (("jvm_start_ms", "JVM start ms"), ("first_open_ms", "first open ms"), + for field, label in ((_start_field, _start_label), ("first_open_ms", "first open ms"), ("cold_process_ms", "cold process ms"), *[(f"{k}_session_ms", LIFECYCLE_SCENARIO_LABELS[k]) for k in LIFECYCLE_PAGE_SCENARIOS], (MEM_PEAK_FIELD, "peak memory GiB")): @@ -7118,6 +7124,9 @@ def main() -> int: print(f" cap rule: {len(_cut)} rows of {len(_over)} over-the-cap arm-cells withheld, shown censored instead") _gone = {id(r) for r in _cut} rows = [r for r in rows if id(r) not in _gone] + # ONE ARCADEDB BUILD PER PAGE (26.11.1): native and jvm rows of a release share engine_version and arm names, so a table cell + # holding both would print one median over two engines. BENCH_ENGINE_BACKEND picks the build; both with no pick is refused. + rows = _AI.select_build(rows) global _FROZEN_ROWS _FROZEN_ROWS = rows # WITHHELD: ArcadeDB rows whose engine_version identifies no build. diff --git a/benchmarks/experiments/l1_tpc.py b/benchmarks/experiments/l1_tpc.py index 45eb68bc9d4..4851322521f 100644 --- a/benchmarks/experiments/l1_tpc.py +++ b/benchmarks/experiments/l1_tpc.py @@ -1070,7 +1070,7 @@ def connect(self): "jvm_args": bench_common.arcade_jvm_args()}) self.version = _arcadedb_import.dist_version() # ASKED, not asserted (#81's standard, doubled by #90's second class). - self.durability = bench_common.arcade_durability_readback() + self.durability = bench_common.arcade_durability_readback(db=self.db) qr = bench_common.arcade_query_ram_readback() # {} unless the query heap budget override is on (DECISIONS #175) if qr: self.row_extra = {**(getattr(self, "row_extra", None) or {}), **qr} diff --git a/benchmarks/experiments/l2_graph.py b/benchmarks/experiments/l2_graph.py index 891debb47fe..27f2be06cc7 100644 --- a/benchmarks/experiments/l2_graph.py +++ b/benchmarks/experiments/l2_graph.py @@ -208,7 +208,7 @@ def connect(self): jvm_kwargs={"heap_size": heap, "jvm_args": bench_common.arcade_jvm_args()}) self.version = arcadedb.__version__ - self.durability = bench_common.arcade_durability_readback() + self.durability = bench_common.arcade_durability_readback(db=self.db) for ddl in ["CREATE VERTEX TYPE Person", "CREATE PROPERTY Person.id LONG", "CREATE PROPERTY Person.name STRING", diff --git a/benchmarks/experiments/l5_lifecycle.py b/benchmarks/experiments/l5_lifecycle.py index 4e909b3f604..3477e2d9d89 100644 --- a/benchmarks/experiments/l5_lifecycle.py +++ b/benchmarks/experiments/l5_lifecycle.py @@ -226,6 +226,10 @@ def build(situation, n, heap): # created the structure, and it is where a close-time rebuild doubles the # cost of building (#6489). A reopen cannot show it: by then the structure # is already on disk and clean. + # The engine's own settings, asked in SQL while the database is open (the same path on both builds; bench_common.arcade_durability_readback + # reads this snapshot when it stamps the row, long after this session closed). Not inside the timed close below. + import bench_common + bench_common.remember_engine_settings(db) _bt = time.perf_counter() db.close() return (time.perf_counter() - _bt) * 1000 diff --git a/benchmarks/experiments/make_2611_stages.py b/benchmarks/experiments/make_2611_stages.py index 640bf8db37f..c8309bf1e1e 100644 --- a/benchmarks/experiments/make_2611_stages.py +++ b/benchmarks/experiments/make_2611_stages.py @@ -21,6 +21,10 @@ only if time is left. PROFILES lists every drop and its reason, `--check` prints what was dropped and proves that nothing of it is scheduled (profile_problems; `--check --drafts` the same on written scripts), and `--profile full` is the whole subset as it was before the profile. + ONE CHAIN PER WHEEL. A release measured on both ArcadeDB builds is two generator runs, one per wheel (arcadedb- native, arcadedb_jvm- JVM), + each with its own --id-prefix and draft folder; the stage scripts derive the image tag suffix (-native / -jvm) from the wheel they are pinned to, + and a freeze names one build (BENCH_ENGINE_BACKEND=native|jvm, arm_identity.py). + The stage list is DERIVED from STAGES and runner.LANES, filtered by engine and lane, and `--check` proves the filtered list runs exactly the cells the unfiltered one would for those engines and lanes. Every registered arm is either in the subset or named in OUTSIDE, so a newly registered arm forces a decision here. @@ -1110,6 +1114,11 @@ def seed(m): ARCADE_PRE_TAIL = '''docker image inspect "$ARCADEDB_SERVER_IMAGE" >/dev/null 2>&1 || {{ say "$ID ABORT: no server image $ARCADEDB_SERVER_IMAGE"; exit 1; }} WV=$(basename "$W" | cut -d- -f2) +# THE BUILD THE WHEEL IS, and so the image tag's suffix (engine_pin.backend_from_wheel): arcadedb--*.whl is the native build, +# arcadedb_jvm--*.whl the JVM one, and the image is dbbench:arcadedb--native / -jvm. One chain is generated per wheel, so a release +# measured on both builds is two chains (two generator runs, two --id-prefix values, two draft folders) and never one stage with two engines. +WB="" +case "$(basename "$W" | cut -d- -f1)" in arcadedb) WB=-native ;; arcadedb_jvm) WB=-jvm ;; esac ''' ARCADE_PRE_BUILD = ARCADE_WHEEL_HEAD + ARCADE_WHEEL_COUNT + ARCADE_WHEEL_SHA_EXPORT + ARCADE_PRE_TAIL @@ -1119,15 +1128,19 @@ def seed(m): # The pair check below is the REPO copy, not ~/verify_pair_c25.sh: that # one is a stale copy from 2026-08-30 and would miss its own fourth # check. A script pulled with the tree cannot drift from it. -IV=$(docker run --rm --entrypoint python3 "dbbench:arcadedb-$WV" -c 'import importlib.metadata as m +IV=$(docker run --rm --entrypoint python3 "dbbench:arcadedb-$WV$WB" -c 'import importlib.metadata as m for n in ("arcadedb", "arcadedb-jvm", "arcadedb-embedded"): try: print(m.version(n)); break except m.PackageNotFoundError: pass' 2>/dev/null | tr -dc "0-9a-zA-Z.-") [ "$IV" = "$WV" ] || {{ say "$ID ABORT: dbbench:arcadedb carries wheel $IV, dist has $WV"; exit 1; }} say "$ID: dbbench:arcadedb carries wheel $IV, matching dist" -IJ=$(docker run --rm --entrypoint python3 "dbbench:arcadedb-$WV" -c 'import jpype; print(jpype.__version__)' 2>/dev/null | tr -dc "0-9a-zA-Z.-") -[ "$IJ" = "{jpype_version}" ] || {{ say "$ID ABORT: dbbench:arcadedb runs JPype '$IJ', the pin is {jpype_version} (Dockerfile.bench, ARG JPYPE_VERSION); rebuild dbbench:arcadedb, which installs the pin"; exit 1; }} -say "$ID: dbbench:arcadedb runs JPype $IJ, the pin" +if [ "$WB" = "-native" ]; then + say "$ID: dbbench:arcadedb$WB is the native build: no JVM, no JPype to pin" +else + IJ=$(docker run --rm --entrypoint python3 "dbbench:arcadedb-$WV$WB" -c 'import jpype; print(jpype.__version__)' 2>/dev/null | tr -dc "0-9a-zA-Z.-") + [ "$IJ" = "{jpype_version}" ] || {{ say "$ID ABORT: dbbench:arcadedb runs JPype '$IJ', the pin is {jpype_version} (Dockerfile.bench, ARG JPYPE_VERSION); rebuild dbbench:arcadedb, which installs the pin"; exit 1; }} + say "$ID: dbbench:arcadedb$WB runs JPype $IJ, the pin" +fi PAIR_IMAGE="$ARCADEDB_SERVER_IMAGE" ./verify_pair_c25.sh "$SHA" >> "$S" 2>&1 || {{ say "$ID ABORT: pair unverified at the pin"; exit 1; }} ''' @@ -1318,7 +1331,7 @@ def has_arcadedb_arm(backends): # server_not_ready. Check here, where the message can name the cause. for _img in {images}; do # the arcadedb image is tagged with the pinned version, never floating (engine_pin.py) - _tag="dbbench:$_img"; [ "$_img" = "arcadedb" ] && _tag="dbbench:arcadedb-$(basename "$ARCADEDB_WHEEL" | cut -d- -f2)" + _tag="dbbench:$_img"; [ "$_img" = "arcadedb" ] && _tag="dbbench:arcadedb-$(basename "$ARCADEDB_WHEEL" | cut -d- -f2)$(case "$(basename "$ARCADEDB_WHEEL" | cut -d- -f1)" in arcadedb) echo -native ;; arcadedb_jvm) echo -jvm ;; esac)" docker image inspect "$_tag" >/dev/null 2>&1 || {{ say "$ID ABORT: $_tag missing after build"; exit 1; }} done say "$ID: images present: {images}" diff --git a/benchmarks/experiments/make_paper_tables.py b/benchmarks/experiments/make_paper_tables.py index b7f41eb9f56..365a0abe9f4 100644 --- a/benchmarks/experiments/make_paper_tables.py +++ b/benchmarks/experiments/make_paper_tables.py @@ -88,6 +88,7 @@ def _commit_matches(row_commit): HERE = os.path.dirname(os.path.abspath(__file__)) import publish_instrument as _PI +import arm_identity as _AI RESULTS = str(_PI.results_dir()) # Every published cell is N=5. A row outside 1..5 is a probe, not a repetition; @@ -539,7 +540,11 @@ def load_canonical(apply_corpus=True, include_twins=False): # without the role the later of the two would replace the other on ts_utc. k = (r["lane"], r["scale"], r.get("n_docs"), r.get("workload"), r["backend"], r.get("gav") is not False, r["rep"], - r.get("durability_class") or "relaxed", r.get("arcadedb_http_client_role") == "sensitivity") + r.get("durability_class") or "relaxed", r.get("arcadedb_http_client_role") == "sensitivity", + # THE BUILD IS PART OF THE KEY (26.11.1): `arcadedb` (native) and `arcadedb-jvm` of one release stamp the same + # engine_version and the same backend name, so without it the later build would replace the other on ts_utc. + # Rows before the field are the JVM build; served arms and comparators carry None (arm_identity.build_of). + _AI.build_of(r)) if k not in best or r["ts_utc"] > best[k]["ts_utc"]: best[k] = r _write_withheld_recall() @@ -549,7 +554,9 @@ def load_canonical(apply_corpus=True, include_twins=False): # The paper tables, the gates and the claims read the headline rows. The twin rows go to the freeze (include_twins) so # the exporter can show them beside the headline, labelled, and are never pooled into a headline median. out = [r for r in best.values() if include_twins or r.get("arcadedb_http_client_role") != "sensitivity"] - return out + # ONE BUILD PER FREEZE: BENCH_ENGINE_BACKEND=native|jvm picks the build whose in-process ArcadeDB rows the tables, the exporter + # and the gates see; a cell that holds both builds with no choice made is refused here, never pooled downstream. + return _AI.select_build(out) def cells(rows, key): @@ -793,13 +800,21 @@ def _pinned_sparse_rows(arm="arcadedb_sparse_embedded"): # partial or mixed, and still refuses. if not out: return {} + # Both builds of one release at one pin: the freeze names the one it publishes (arm_identity.select_build), the other stays in the log. + _builds = {_AI.build_of(r) for t in out for r in out[t]} + if len(_builds) > 1: + _want = _AI.wanted_build() + if not _want: + sys.stderr.write(f"pinned sparse rows hold both builds {sorted(map(str, _builds))}; set {_AI.ENV}\n") + return None + out = {t: [r for r in rs if _AI.build_of(r) == _want] for t, rs in out.items()} if any(len(out.get(t, [])) < 5 for t in tiers): missing = [t for t in tiers if len(out.get(t, [])) < 5] sys.stderr.write(f"pinned sparse rows incomplete at {missing}\n") return None - versions = {str(r.get("engine_version")) for t in tiers for r in out[t]} + versions = {_AI.arm_identity(r) for t in tiers for r in out[t]} # (engine_version, build): one engine, one build if len(versions) != 1: - sys.stderr.write(f"pinned sparse rows span {sorted(versions)}\n") + sys.stderr.write(f"pinned sparse rows span {sorted(versions, key=str)}\n") return None return out diff --git a/benchmarks/experiments/merge_campaign.py b/benchmarks/experiments/merge_campaign.py index a0c71d30a19..978af9cfdc9 100644 --- a/benchmarks/experiments/merge_campaign.py +++ b/benchmarks/experiments/merge_campaign.py @@ -34,6 +34,7 @@ HERE = os.path.dirname(os.path.abspath(__file__)) sys.path.insert(0, HERE) from publish_instrument import results_dir # noqa: E402 +import arm_identity as _AI # noqa: E402 # The store moves with BENCH_RESULTS_DIR like every other reader of results/ (the freeze, the exporter, the gates), so a test # merges into a fixture directory and never into the working store. A publish never sets it. @@ -77,7 +78,7 @@ def key(r): against the real one below. """ return (r.get("lane"), r.get("scale"), r.get("n_docs"), r.get("workload"), - r.get("backend"), r.get("gav") is not False, r.get("rep")) + r.get("backend"), r.get("gav") is not False, r.get("rep"), _AI.build_of(r)) def _assert_key_matches_tables(): @@ -95,7 +96,7 @@ def _assert_key_matches_tables(): "workload": "search", "backend": "qdrant_sparse", "gav": None, "rep": 1} src = inspect.getsource(_M.load_canonical) - for field in ("n_docs", "scale", "workload", "backend", "rep"): + for field in ("n_docs", "scale", "workload", "backend", "rep", "build_of"): if field not in src: raise SystemExit( f"merge_campaign.key() includes {field!r} but " @@ -116,7 +117,10 @@ def canonicalise(rows): def version_of(r): - return str(r.get("engine_version") or r.get("wheel_version") or "?") + """The version a cell ran, with the build when it is not the JVM one ("26.11.1 native"): a release measured on both builds is two arms.""" + v = str(r.get("engine_version") or r.get("wheel_version") or "?") + b = _AI.build_of(r) + return f"{v} {b}" if b == "native" else v # The (lane, scale) pairs the paper actually prints. Everything else is an diff --git a/benchmarks/experiments/page_check.py b/benchmarks/experiments/page_check.py index d75d0b13828..76f5c5d9d86 100644 --- a/benchmarks/experiments/page_check.py +++ b/benchmarks/experiments/page_check.py @@ -779,6 +779,12 @@ def main() -> int: "which HTTP client an ArcadeDB served arm ran with (CAMPAIGN 7 row 72, lean_http.py): a condition of the served arms, read from " "the session that ran, never a column. The e4 decomposition's row lists the two clients it measured (`arcadedb_http_clients`); its table " "names the client in each HTTP column instead (export_web._e4_arms)"), + (r"^native_isolate_args$", + "the arguments the native engine's isolate was created with (heap -Xmx, -D settings), the native counterpart of the JVM arguments: a " + "condition of the cell read from the process (_arcadedb_import.effective_native_args), null on the JVM build, never a column"), + (r"^engine_backend$", + "which ArcadeDB build ran the cell, \"native\" or \"jvm\" (arcadedb.__backend__, bench_common.engine_backend): the identity of an arm beside " + "engine_version (arm_identity.py). export_web.select_build keeps one build per page and the cell's conditions name it, never a column"), (r"^jpype_version$", "which JPype the process that ran the cell had loaded (CAMPAIGN 7 row 73, bench_common.jpype_version): a condition of the embedded " "ArcadeDB arms, read from the process (`jpype.__version__`), empty for every arm that never imports it, never a column"), @@ -846,7 +852,7 @@ def main() -> int: "as conditions under the table, never as columns"), (r"^(settle_s|settle_s_lane|settle_s_adapter|engine_settle_s|gt_load_s|" r"recall_calc_s|query_gen_s|search_wall_s|phases_accounted_s|connect_s|" - r"import_ms|build_close_ms|close_s|mutate_n|mutate_queries|mutate_ran|" + r"import_ms|engine_load_ms|build_close_ms|close_s|mutate_n|mutate_queries|mutate_ran|" r"corpus_release_s)$", "harness bookkeeping around a timed phase: settling, loading ground " "truth, computing recall, the phase accounting, and the time-series " diff --git a/benchmarks/experiments/runner.py b/benchmarks/experiments/runner.py index 16a2fb642ba..fe55593e3a6 100644 --- a/benchmarks/experiments/runner.py +++ b/benchmarks/experiments/runner.py @@ -1803,6 +1803,8 @@ def _at_image_jvm_defaults(cfg): # logical build target; the image a cell runs is `dbbench:arcadedb-`, swapped in below, so no cell can start on a floating tag. import engine_pin # noqa: E402 _ENGINE_PIN = engine_pin.pin_from_wheel(_LOCAL_WHEEL) +# The build the wheel is ("native" / "jvm", "" for the older arcadedb_embedded wheel): part of the image tag and read back from the image. +_ENGINE_BACKEND = engine_pin.backend_from_wheel(_LOCAL_WHEEL) def _require_engine_commit(tier, backends): @@ -1888,7 +1890,7 @@ def _require_local_server_image(): if _ENGINE_PIN: for _cfg in BACKENDS.values(): if _cfg.get("image") == engine_pin.LOGICAL_IMAGE: - _cfg["image"] = engine_pin.versioned_image(_ENGINE_PIN) + _cfg["image"] = engine_pin.versioned_image(_ENGINE_PIN, _ENGINE_BACKEND) # Which backends run on a JVM, and therefore have a heap worth recording and @@ -2218,7 +2220,7 @@ def build_manifest(ts, args, workers, shards, jobs): # The image the embedded ArcadeDB arms run in (BACKENDS "image"): the only bench image whose wheel pulls JPype in. -ARCADEDB_BENCH_IMAGE = engine_pin.versioned_image(_ENGINE_PIN) if _ENGINE_PIN else engine_pin.LOGICAL_IMAGE +ARCADEDB_BENCH_IMAGE = engine_pin.versioned_image(_ENGINE_PIN, _ENGINE_BACKEND) if _ENGINE_PIN else engine_pin.LOGICAL_IMAGE def jpype_version_in_image(image, timeout=180): @@ -2269,10 +2271,10 @@ def _require_engine_pin(jobs): "(engine_pin.py).") for img in sorted(emb | {i for j in jobs for i in [BACKENDS[j["backend"]].get("image"), BACKENDS[j["backend"]].get("server_image")] if i and i.startswith("dbbench:")}): - why = engine_pin.check_image_tag(img, _ENGINE_PIN) + why = engine_pin.check_image_tag(img, _ENGINE_PIN, _ENGINE_BACKEND) if why: raise SystemExit(f"REFUSING: {why}.\n Rebuild with ARCADEDB_WHEEL= ./build_images.sh arcadedb.") - readback, problems = engine_pin.verify(_ENGINE_PIN, emb, srv) + readback, problems = engine_pin.verify(_ENGINE_PIN, emb, srv, backend=_ENGINE_BACKEND) if problems: raise SystemExit("REFUSING: an ArcadeDB arm would not run the stage's pinned version (engine_pin.py).\n " + "\n ".join(problems) diff --git a/benchmarks/experiments/stage_drafts_2611.sha256 b/benchmarks/experiments/stage_drafts_2611.sha256 index ceb52ac7b80..d796c7a552c 100644 --- a/benchmarks/experiments/stage_drafts_2611.sha256 +++ b/benchmarks/experiments/stage_drafts_2611.sha256 @@ -1,229 +1,229 @@ 4c69fbd9e12a9112e73a13340e2683711a7fd8db927a53261275f7f5d117ca31 arcadedb-release/qV00.sh -a168325cab96fe46d9b5a8ef6c4bad92acdd107e00fc8335ad15874f785fe548 arcadedb-release/qV01.sh -683d137c511786c505da1752d291814decdc5f892b4de58424a8e57c3b54f9bc arcadedb-release/qV02.sh -b6406f96385b5c9171db7aab27f6b512af96cdc150dc029ee0d19cd55800c4b9 arcadedb-release/qV03.sh -3a1af0073d71b348a28d660ab5e17ba70125d539e5d3457dbc30c8199c002059 arcadedb-release/qV04.sh -826096159e118964c7000a2ba8485d51a78066abd4c4651e5a949b5d1178546f arcadedb-release/qV05.sh -9a244a35c85f1d001482af845f074f0887ec04e8d51c5620d326dbbeef98b0b0 arcadedb-release/qV06.sh -3a3e421d65b60a4371d91a973a7d27002fa799a867e7b54a55d21a8b30662745 arcadedb-release/qV07.sh -4fade8d29ba24c4cdc7c56f7718ad9929bc108b7335c9466e8b97c30de3c54d3 arcadedb-release/qV08.sh -799cdfd95037882cb25b25d11b97adf898b284e48ede3bcb493afcb762849c7f arcadedb-release/qV09.sh -9755f0c9af6673778025e5ad7384ffb994d75e1287bf4bf1c9c7570e10ec9564 arcadedb-release/qV10.sh -d4e94034e9aa1dfe388113ad786a5bcaecba216e6306384901a28e883343e414 arcadedb-release/qV11.sh -b2053a1b2c15f40959b943f1adbe2db1833e353465589f2809657e0722b19a7f arcadedb-release/qV12.sh -676ea6f7183d518cda1fc9048d5b6888497590ff03d1825d60fa9b6c90820092 arcadedb-release/qV13.sh -54cefaa2248430143aafea606b211abf82cef7230e6b0becee6a7b34a9f67cf9 arcadedb-release/qV14.sh -7ef382f657a9ce20966dbc9781163873c5577aaed60c7c300e3e7e2ab5702ec7 arcadedb-release/qV15.sh -ba32dd19e84a04c71fe94720884ce7e43eae9505a2d3fe56f2ad6544e3f577d6 arcadedb-release/qV16.sh -a71c482b5bffe1d0faf15eb14c99b10442b1a02e1723605bfa9860910c90926d arcadedb-release/qV17.sh -eca51330cb01aa21498547c00cf31c8350bb35716098360b5f36ca3ee8b314ca arcadedb-release/qV18.sh -07c7f32a72ae392c9ec8e60ec8ee3580f899fbde17b33f271db786ee7419ea1c arcadedb-release/qV19.sh -dadc9d0677a8d0ab7ae4a85240356b645f44700b072f4fd749a3e8cdbd242965 arcadedb-release/qV20.sh -9e7909ae10422bbb94d8f1abeada8ee5af4d15e308e35a61140f500b7efa8034 arcadedb-release/qV21.sh -90deeb9696e59b52d238c1c2e64c6a301494c23b2574e7d6d623885629d35ade arcadedb-release/qV22.sh -066f37aa8aaa85ca7371369abf875e95b58cb6921f05c62576dd66bc5ff71f22 arcadedb-release/qV23.sh -2bfa57078d37f5cb3cbd5cb5690ed949be0f0d26ad066a9e701292437dfc520c arcadedb-release/qV24.sh -2124facae9ca44e6ef708215f1cd038915e743e0ba1cd532f3fea92d0ad5fa1e arcadedb-release/qV25.sh -bd0af3ff8ede9a92231d4e4211540bc857697870c4a980b0e85f9eb1b13ec425 arcadedb-release/qV26.sh -07424273f1d3ce02065ba1467923e1a2f93ccc71a6a4cf6641f97f9f17313ca3 arcadedb-release/qV27.sh -3aadf894847187bd20b730835bd4907bd875d62f882d83fae41b50e52e43d944 arcadedb-release/qV28.sh -216d90dc14e021b7e63ab8dc19f4dffb7b77bb0c02a1ffa8cbcfa3424cbefc0d arcadedb-release/qV29.sh -b208a5630458f98ccbea29a50560eb303db55a1426f6115cb6236bffae135611 arcadedb-release/qV30.sh -dd645fc4fb6f371a35fe64c1e3ac5ef964c1360e2fb10dee398c4a79f9ee2154 arcadedb-release/qV31.sh -c9c13c5ef9a0fc5b99831fce0461643943a9526419db8981eaf58af8f02c0aa6 arcadedb-release/qV32.sh -8c0e1b53173800097b2311eb3fcd4499bda5a96de2965910249aef3cce2b3a1c arcadedb-release/qV33.sh -498ad98759ddbcff598f6d9461b60c9a06223bca2445d8eddb54fce982867eb2 arcadedb-release/qV34.sh -9cf8a83c1fd3694a590342f2ba439518c004e9182f52bf579bbece02039ca9a6 arcadedb-release/qV35.sh -090aeec74126feaa72e55a3d9644ba784116d36e0e824a2fc0868ffba9007d0d arcadedb-release/qV36.sh -f1c22c68cf374ee38175f79707168d676b966f35e597b4505d0acdced98a9be9 arcadedb-release/qV37.sh -53ff2148d50bfee0e80f675aa04ddf51f00d57c7f131b0b3aa3a0530fda4ed14 arcadedb-release/qV38.sh -e275d1d43aa0aa37f3706a4c76c6f48f4332823b9b6a58caf0f14f585d9fd871 comparators-no-tiers/qS01.sh -644cdc3ede592f651eed80c87fd067388464a1a0e9c5ddc748e25a65123a3944 comparators-no-tiers/qS02.sh -09e8c2015781560aecaac76aed575f128999629f420ed979347b00e5e260e057 comparators-no-tiers/qS03.sh -193cb638bb160da56635acbc61485b145b24fbb2cd09f8a1e8cc02d21ccdfe65 comparators-no-tiers/qS04.sh -3809ce2d9becdb1f75776db10625f05b8db31836895379d8ec871a1056fd861f comparators-no-tiers/qS05.sh -a200f61f72e803c0b812ac16ecca3d6d03e2db3840cdfea9caeffb480f6f0475 comparators-no-tiers/qS06.sh -12c7f88160552f611f6d7e003bd3b0c14c8d86d4b312d6121c7bff086ecdfd48 comparators-no-tiers/qS07.sh -0dce2d82f1668c565f275fa4d08fe1cf5d0009e07eb41bbccfe79ca78b14217c comparators-no-tiers/qS08.sh -fedc53cbd3dca18ad31365ea26ea5aa62441033ccfeeb69e6e7a6d2c123a7b7a comparators-no-tiers/qS09.sh -94efddfa5ac17996b33b7cad0105ab577477a9a5700ebc4f9883db3829de5eea comparators-no-tiers/qS10.sh -edb36a8b8945a31a0be9caa2a9f7bede9a119be01498eef1fd9db698b8599e37 comparators-no-tiers/qS11.sh -d692ebbb764f30c0147b98d90667a2865eb6d0f066bf2767c95ed52e0a7066e3 comparators-no-tiers/qS12.sh -f3ab73bee8dfb035ad654c3aa203722b73a69868889f4af4cc82f09b210b8fbe comparators-no-tiers/qS13.sh -d6d24340938903f525e645600f7f173631458b0a566bcde0bec9d2359726b838 comparators-no-tiers/qS14.sh -40d81816068ee7d47c08489ba01c0b4d5732b44ed29ca004def7bb3cc67a0a0b comparators-no-tiers/qS15.sh -3f1dba8c349f2dc334c7860c3f681ca636b3e02490b203cf84e1f71510f25dac comparators-no-tiers/qS16.sh -d1cb7d096734a5810e895a370e29be82ab15bfe94a14dce9101c640054b0891e comparators-no-tiers/qS17.sh -45f890c92e9849fab5d06377619b7d8066cf929f0b9b6724a713ace1af826b2b comparators-no-tiers/qS18.sh -c751241642026fefa5410d634b54cec75b39709cdc34d19d10173f8b07763b55 comparators-no-tiers/qS19.sh -240500ab89f9aa99b9429f2fa9b0444cf24cf86351ed0297a54ea2f809ede63c comparators-no-tiers/qS20.sh -27100f9db5017857f40bcd5fb9ecda64dc310baa1f3693b430e8cd9254b2ad0a comparators-no-tiers/qS21.sh -da1cafd1882da88bea6a7d3485a7e4c3063a21f0cfc46b0d6e85c39bf295e762 comparators-no-tiers/qS22.sh -1ea841443e37ddc304f8d4aad28a0da2384fbd9ed53c1740bb5f9bd1f1765304 comparators-no-tiers/qS23.sh -cfa8d9f37186bbf48d3a2ffdc6376473fe51a61440ae396f2ecdc9948628205e comparators-no-tiers/qS24.sh -22c32061ff4855e8755c3a7b4bbd75303cf362c5b9d9cdb7f4fe4dc9988354ea comparators-no-tiers/qS25.sh -0d13375e58588a3c80917d6119e83e9dcaab0601781c988c2f4a5385b7431d2b comparators-no-tiers/qS26.sh -21f628f325d1efd575c879fce4cf9e597dbcc54eaad8b851fa8c2948ee6e96f1 comparators-no-tiers/qS27.sh -167438345366e54520592aa43cfdefa18615e6787bd613ef6e628c2fd0298619 comparators-no-tiers/qS28.sh -baee8b1fc33d5b9d8b58a3ad9c3bfcd444ba1d9fa3462fffb9079413425c866b comparators-no-tiers/qS29.sh -4b2da337fbd268e40d3511edb276af58d63509536c5f721c5671b095af3d8277 comparators-no-tiers/qS30.sh -0accbc3a12f5a206a3be69b923ac05debf893417231c462e542875261950c676 comparators-no-tiers/qS31.sh -ca8af9dd3f09d79960d37cc26c17a67a12435f069b96d9348fefb2538109d495 comparators-no-tiers/qS32.sh -0732fd592e2e9b029692ef4a209f4c5b6286f66a740ea972f00b570010b98994 comparators-no-tiers/qS33.sh -601f2eeea0d8bc6b26761ac6762eb237928290ccfff83efa4f45d4259cf9131a comparators-no-tiers/qS34.sh -98169449f300e9aaffa2bc82adaf5876ddc83293d6bf4e5575a94cfd205bd4ab comparators-no-tiers/qS35.sh -20d6d3194f82d43fb81399350fb509aeba0d2ce1d28d72653b79050b99f6e605 comparators-no-tiers/qS36.sh -815ff5474e565f79fbaef02e7bacc8076b71b56b137eed586fedfcbbe7e25541 comparators-no-tiers/qS37.sh -febdd1b9b0213504d67c0552096e6b481a14f73c710be38922f0fabc46040695 comparators-no-tiers/qS38.sh -6db7fc2f1a7929d066f9b69a62d8aae093b3635b4eaa36ca83df016d33c1f083 comparators-no-tiers/qS39.sh -1373c6975d3c6928b3e6b6be2cd450e3fb15901ff5223e332cd86949843234f6 comparators-no-tiers/qS40.sh -c9893269ae0bf27f666db42b16321c969ac07a1fe487bfd85c7069d430fe410c comparators-no-tiers/qS41.sh -953e93cad942aa5adbb008c7458166cb987f9420242d0594afd4a2f9960c47cd comparators-no-tiers/qS42.sh -98c5c470914875a3a3af76d38309ad0e49298308fd5a415197edd71dc548b71d comparators-no-tiers/qS43.sh -ea93b78425fb5668f5e440074a833045ca4fa374ba945b6051d16f668192bfcf comparators-no-tiers/qS44.sh -5e2ea46a2cc5dc44ebc9f1e3fe1c476c6dc0176388ffaf9fa4da2d9eea9f99c6 comparators-no-tiers/qS45.sh -51400b95cba0f4cdcc83f1b72e94fd0a1a8c6633e14e1026faabe1efd94a2f62 comparators-no-tiers/qS46.sh -b88d279ddb103cc118e3a1f65baa2205cafb55f96a5af2959f69d93851b2f225 comparators-no-tiers/qS47.sh -713b2c65b9d6deda03dfa5ba8f8527ae968b3c4cf63afc3807693ae0ada92fa7 comparators-no-tiers/qS48.sh -882e8c69c169fb1f8594bdbc3daf48185e6f82ecdec39e5d1034352ccd480360 comparators-no-tiers/qS49.sh -bfe036d49280c09c0aa03c428ac503cad6d461ca08ed51c8beefbc856a6da824 comparators-no-tiers/qS50.sh -0cefe4bdab8c2cc20487af7401e4926d78b77a2a7a0a4feb294d4b020150da46 comparators-no-tiers/qS51.sh -da480b98b2a0305194f6262ef84ead1aebce481b3bb855d13633b374ffb9d901 comparators-no-tiers/qS52.sh -c054de1073401904d12d310bb3df31066280daa1a2aad3e5134970debf586a25 comparators-no-tiers/qS53.sh -5632f1ef2b7524d37bf8a2323ea47bd66ec763e9d015f89aaa8be1b0b23b112c comparators-no-tiers/qS54.sh -56fc57bd6440c87b84f9e7f0cb745c5aa9b9582129ee2ffce47a135e3d783268 comparators-no-tiers/qS55.sh -bc714f588e84af32514671c82a9daf5f151649fb190708ec7c55ebdae6597d0a comparators-paper-5reps/qU01.sh -5b25affb7576e1973a4e9b811fbc0ca65bb96f79fe71140ff08625cadfa6823d comparators-paper-5reps/qU02.sh -66071833587422ece4c6a9c8faf9474f1a609d1ec1ccef9171e1f6e805a424fd comparators-paper-5reps/qU03.sh -5d4374e58f5d3ee5d911708df6c9d24ac790567416df4cc88690b87fad642548 comparators-paper-5reps/qU04.sh -11c7b8664276029802d319f3d828548adb76793068b432975d2cef47d6a7a97b comparators-paper-5reps/qU05.sh -75ec3ea941cc7f2f7dd0683bb03b75047c933c0cd3cc9f79d533e4d637c1bfd4 comparators-paper-5reps/qU06.sh -1f2d6ef2e6ceeca9dedebc63f882a36056a584dc9a2e18238a7834bf110de413 comparators-paper-5reps/qU07.sh -e38feac6af12e3f29da235fa3c75e7aa5db9622ae735603cdfc6f9771073f1df comparators-paper-5reps/qU08.sh -eb0ed05a0314771742f64e56b95fd6367bdb378293fcfccbc872a85ab78328e6 comparators-paper-5reps/qU09.sh -69ef31106abf0a7dde8e37694e3b01715a5e155d0248b5e4bce18c52674b7de5 comparators-paper-5reps/qU10.sh -2f4a50e6155f80e78d7a304c0a26697997f3c6588138caff47d1568302f2d5b2 comparators-paper-5reps/qU11.sh -a84c96ea6f720491a78f8a9b9e557c34690e66ab93fbac0bd60620ad73552488 comparators-paper-5reps/qU12.sh -602acf5e4f4ee1a2e5fe60de1405dba0d576b247e869f92c731b2c30526ef4ac comparators-paper-5reps/qU13.sh -3dc94540f76462e941a1cd0fe765cec26aa437de306221fc6d3389e4ace343f7 comparators-paper-5reps/qU14.sh -ae8cd43fc3a9f457a523a8573920834c3fb6ab29fa340a74493c8abc919a6b21 comparators-paper-5reps/qU15.sh -7968afecb28f584e3d1073f1dd2680ef32e1b0e357115360427076eb09112b55 comparators-paper-5reps/qU16.sh -fa8efd7a3c1edbc6eda77293f8337f9bdad6b6f8dcec0f5c9a07281a836c6bcd comparators-paper-5reps/qU17.sh -131bb12615e7f789d508d84b828028b02bae0ed75fc0f7146757ed3a926cf56b comparators-paper-5reps/qU18.sh -b809f8b97eafebba8755fe7f438aee5494c095fd45401e01207e2eff236f7556 comparators-paper-5reps/qU19.sh -77c1f92dbbd8bae99891b8e785f659597bbd2018f1ee6354c86f2628e520a425 comparators-paper-5reps/qU20.sh -71b6ce127e491781768699fb161d94f978026c50e68a6e637c1ed1fb0625738b comparators-paper-5reps/qU21.sh -f443bb982320b831e524458fc0b62a3c2fa0c668061b811cbeed4c64834b77cb comparators-paper-5reps/qU22.sh -a653998171ed68f7cc0ad807e24e5d7414559f0aed55611c2550b434e59d626d comparators-paper-5reps/qU23.sh -0c53ddfbf235eb99687e2934f7ef4e98c60f77f868c1d080d97296ab68fb3980 comparators-paper-5reps/qU24.sh -ef514e1c0a53444a2fae3d0c22aaf684995e6adaadd084017fd121c76049ea11 comparators-paper-5reps/qU25.sh -1a2a204df3cfe6171ce68b3e5552c7ab436f006bd027aca07cd4c499ed379eb8 comparators-paper-5reps/qU26.sh -af9397f40150b1395a42776764f67c5da154e94a3698a2c2e0e24618a113ce13 comparators-paper-5reps/qU27.sh -3269c76a5608cc6d188f2db0a6540a5b9c5783ed29d66b4b73f0d324f3cd01be comparators-paper-5reps/qU28.sh -1c44502fe571f477c30b2da48fe206413f839c69a161405b7faae4264a444e94 comparators-paper-5reps/qU29.sh -3fd0b804b68964e46534cf9ed416c49efd25ce0531fb5d5ea156fd7a0a21195d comparators-paper-5reps/qU30.sh -0edab01693343564a6a2cf76ce2f934065745557446a6561cf52593a3d706879 comparators-paper-5reps/qU31.sh -3f2828863cec9f8e7c7f3a4c63b762fdb209e88cb82e82523b4c598d59f92cf0 comparators-paper-5reps/qU32.sh -b5fd5475255629408fc45a82494f350f5cec4805771e01d9f3857fb4247620aa comparators-paper-5reps/qU33.sh -f6d9e49494dc7ddb32e1b87f8d51c502f6a5009a03c24c31dd4777a2388a8136 comparators-paper-5reps/qU34.sh -4ac57c61160cb5409b5e527cebf054b00ac8f2cf050fa3a103022acbd41eaca6 comparators-paper-5reps/qU35.sh -a73beb20a745da3f9a4d437c733226443c25ddcb4d7594a1b261380c6609502c comparators-paper-5reps/qU36.sh -80f58573bb9e41885d3ca0cd47fae2f23b311cccbdb2bae7c21741a001b84c4f comparators-paper-5reps/qU37.sh -73f15b796e2db33c480cae1872e3bed65c02f303998c40219e7ca150bdbeeb7c comparators-paper-5reps/qU38.sh -42e17047a033a3558322c41c351d4dd2fd34ede9423c0a11c494beaddd485ccb comparators-paper-5reps/qU39.sh -64860f0e0f7ac0d24792516e570b29252157f5a8560db9e37c8630dcbbd0ddea comparators-paper-5reps/qU40.sh -93da275f94a81744f11927ab731f13730262199471aec1f547d457e284557b21 comparators-paper-5reps/qU41.sh -1609313877fb9b952acfc5414bc22c8afdc944958768a0dbe12bebb8cd1be942 comparators-paper-5reps/qU42.sh -0e8a37f8b0832d8a647aaa746690b84cd1b0ef821bc25c2dcfc8a46574f96954 comparators-paper-5reps/qU43.sh -8fbb6c2a536c99fc7388dfd525a61f5730b8c0ce662a64603b9b71852474a042 comparators-paper-5reps/qU44.sh -d8c1224e72053c4d83adced6768d2fb3a4818a722acd6faca22e78ba3bc539e7 comparators-paper-5reps/qU45.sh -33ca3dcb7fe4e250b71355a33fac33e49a0a95b9e39e6a22acbea6959ae4deba comparators-paper-5reps/qU46.sh -30dca32ec1a482531a64731f008405c13de5be98713eda42aba4436d5d21ad20 comparators-paper-5reps/qU47.sh -d704b2671cc07e7d4d28b36e52b7be906ff445e0d1344fca7b8dfe241575c5f0 comparators-paper-5reps/qU48.sh -a118c02b9fce61f5d91df697f003e97f2a69d61cec66c0f42ee56bfa4f1de6e4 comparators-paper-5reps/qU49.sh -37d8f14885ca9f73289e6ad3e562eb02a887bece3a1769bbfdd68b15856dcaa6 comparators-paper-5reps/qU50.sh -1d59a550af1dade475bc0c5a3b49690a406d6ddfb2d232384caef0267b010f23 comparators-paper-5reps/qU51.sh -f8582a8b629b26ef9df61f2eae52ad66c3b07a93b9043e04b6c62c9ece795981 comparators-paper-5reps/qU52.sh -df8e105f77dbec1e94e16ecc11a21e300ca8c9991f2b84d131abb32edcc6438d comparators-paper-5reps/qU53.sh -5f3e70afd89695276bcfbcbdefaf33e1d278496ab21829fd46b9751fc6d111d7 comparators-paper-5reps/qU54.sh -4ac77e3a2b16ef5b08ca24d915154b890f11be6d4a7a3bee80c43839270c3b98 comparators-paper-5reps/qU55.sh -1143583d6f86e395685ff4d55b88520527d03354e3ecd3a42f6b2c174d2d7447 comparators/qS01.sh -8339ec23b31ea88e2d266623b5a2f107dac50d16457d88ea5ae5a1d8f7e9f9d3 comparators/qS02.sh -9d02abc3ca2569be03c62e52811944b4338fac7f71891276126dbeec42637b59 comparators/qS03.sh -297813aef03b0e599390ba4c02f85b99d865cde9a30a76135f25464c60708956 comparators/qS04.sh -e99bce6355fffd8643973d90113f243dc055dc5b70628c1c7b7437648cf1bd4c comparators/qS05.sh -3ba50ee528de7870b55ea09d64e102965cc953c46705d89d346244ba583de881 comparators/qS06.sh -c3cbe75602ae7a7766a9cc589f67c73ffc0983d71f7de3b128c9dc884d7eac1d comparators/qS07.sh -3a76f95a5e0734c327fc4223b4e1b484b9a6904ed0ee3848e9040ab5bb9b78bf comparators/qS08.sh -c566ed9df4f41993821ea306b35193f172ff4defd7b1d45af399781c7121a8ab comparators/qS09.sh -66d2c75ffb413703775268a01c62740db2f9e43499ead8677847ddcd66ca901e comparators/qS10.sh -e6bc17137f37ab909ccce2d849d6adcc8b2511a071a97caf6a14b96159b3743a comparators/qS11.sh -32a6004613afb1498a8fc3249d216addbde799699337ae202b3d66d1167d7534 comparators/qS12.sh -c566948ec052b447e34a334eee073b5fbc19d821fc2c9dc85fd78ec0381d6f68 comparators/qS13.sh -9c1feb96e42ef6da07f9ad88edfe168cbbbce2fb34b42a53dd0fb22f586ea70b comparators/qS14.sh -58bd10c20c8b39db875ad26149cf8d0907fd605b868a9c74d643e11a258679c8 comparators/qS15.sh -5d0fb39ec8af993f100e3eab797b80f56db88068aad9a5a0f949d940c8396ffa comparators/qS16.sh -30142766689d69fc1ed03cb4153985b1b76f6a1fa8fb0e88f59952694442ebf2 comparators/qS17.sh -1ea2f7889d980ef06a16c256948f534ad0531b89ea98a42fbb94ef5c791dcb49 comparators/qS18.sh -15437f691ced0ed3722c695c3df7f303f4fb3f61fb19de141795c1310e2a509e comparators/qS19.sh -45f0408e094bf98eec20c352dc049cf89315aa2336c1da6cba867feec4d4d033 comparators/qS20.sh -c6d416ab460be5ab481ca17115b9eab098d994a635a437e90e1df68344d97b4c comparators/qS21.sh -b3a95c3a2423cce11f0e72d354443cdbbdc770c0a51dbd20592fb8b7020e2b33 comparators/qS22.sh -67332238383134c95524d61732d25ba47c7a30b776d8d042f1c598498c7b7e32 comparators/qS23.sh -7e512a84fe517874609a0a17003ff82c72a352c4499117dfe2e1295078906b23 comparators/qS24.sh -9805b4ee93924ddcba93622b0b104034e0c7ca7916fbc1c1588c042ddc0822f8 comparators/qS25.sh -49da2aeee122086240ddc65278aa02f97bfc1bd26da0181ef43d9f400822d292 comparators/qS26.sh -74f2341cd5836eb0a611d129bf83b8ba310731cf749379b3ffcaf2865f244205 comparators/qS27.sh -4f4fa6322e08a6fdc0c37db4e2a6de273cd08780e9e8f1882631977676dbd4ba comparators/qS28.sh -0357b315f8d22a263c728834408f827e16c4cc37587d56e1eb450641022a1465 comparators/qS29.sh -1be81bf7abf7d74c5e7e024574c61229304e9dffa897798cb29fab45bbc8868c comparators/qS30.sh -34f0069f713af392df0f9a24ea61657696ba3773bab8027e52d7dc156143704f comparators/qS31.sh -e9a72156eccc1da1bb68596391e79c48e515cf359ee8ab2f068948ca596f804e comparators/qS32.sh -c07b93dff2152c8421aa88f05bfd72ec8e3f159af4084b1fde171804b7112c2c comparators/qS33.sh -2fd0cb9a0115f1993075a7259be917739697f4ded4337bfd17f570832d9230a0 comparators/qS34.sh -a1d431365857c13b2c372229074794e2c2a997b99161947f9234a0d27e63f5f1 comparators/qS35.sh -9da12f901ed0e42a6e8592714f3ba1f6c184d72a19ededbb819735ee13778056 comparators/qS36.sh -9026f807d6d2847ec11ae29c3550c9df6c61cbc47aeb1b5921f6b15bc345b32e comparators/qS37.sh -7c73c1a3cb5e4540ecc5be0686a32b3bc379a2148f70e99c96f8365f09b184f2 comparators/qS38.sh -279930709a085f57ddbaf790b41f2a04716136084ce11542c97ed05045a2d2cb comparators/qS39.sh -6b5e555ec469e548020d5515fa5bca11cdbc2c09c87f55042f0600f743805701 comparators/qS40.sh -cfb0d24c2f8aace3640e3f24072cf1c5f98e7995d606af9eda2ec2652b1506c3 comparators/qS41.sh -f95ce0aea12d43df9570e1745059229f3682aef915cffcce7978f18f7e94639e comparators/qS42.sh -1bf05ac4a167cc4061fba69438637c7fd68b307d731ac21adb64edf621502c91 comparators/qS43.sh -9d48c4e1beeec25c60b351c9a09ccbca24c8568b2a15a962d3e9aec926f7d6d1 comparators/qS44.sh -9ccd801dc3fb7ff0895475c7134fbec3b3bcf839589250f82c22e59c00b2a847 comparators/qS45.sh -dca6771980ed7018a7812707f51b3618d4039531110499219865ab94450d89c8 comparators/qS46.sh -a38f987dd47b7261d2df9710ed144981186aaebff600b3b0aff26351bc0a0871 comparators/qS47.sh -3189346a5724cee8653b169ae2f4a82bc981905e2ab5700773e132236ed56eb1 comparators/qS48.sh -784532d1643e84f4e5fd0f0770646479c6b98cee0833dfbce04185defe4ea6fd comparators/qS49.sh -8c5201d92729b35895c07bcd30e7df853b3fe0ea6e30319972e4aa55546239e7 comparators/qS50.sh -33f1571fe04b2a694abd834f9e8c424482db2e0ecd1688768eacd59d41ed29a1 comparators/qS51.sh -9015d4d452725100183f79260832c9f6af1eef2655808978d355b81e437d6819 comparators/qS52.sh -bdfafd00d30d558699b88837cc4c458af1491de76a78d7e45ad89ebbb47a64cc comparators/qS53.sh -81e4cce813e358e3f4f34a6698e234a808f9f0e7a8daf6251b6a3d4a700aeba0 comparators/qS54.sh -48a8eb5e5c6ea125dc226a45db36a6c27011d06face30aa794e71c102b041c5a comparators/qS55.sh -9e3ed3c30b4426302080df960f8e65f44a052f02aa9b1720cf87733e52cb3157 comparators/qS56.sh -a8384cfb6ba5e1100eb758bbbaa71f447ee7b3cc3899c75e166216f3cc7d51ab comparators/qS57.sh -09e1008a3671b92662b4913427efe70d39ffd2350d8bea494ce135a813d6a3f2 comparators/qS58.sh -bbfac2dcc7c4c08455c5d803489815333d3560a1a603a8761d09ab020e6620ca comparators/qS59.sh -ca3f380baeb9b86764c65d6742bc6fcdfdc4a0bbbeb1a45840733171ec3d3903 comparators/qS60.sh -94ed76d597622145698c95e2b865869db677d98851f5136269ea82bbab6ec378 comparators/qS61.sh -a7c57ee08782bf33c81098ea0624a218ef52fcae28212576b7b2302f8b5264d7 comparators/qS62.sh -f7a129fd55211f86b9ab086caa085db6631488fe26157967b55ff2c282614014 comparators/qS63.sh -dc51155e8b833cc2c2c7fd8cf4125f681c7da4604359afe10a99b2057c15dcf5 comparators/qS64.sh -c3669f487f503d48f94bcc9b19fe434b47b76a90ae3a3302969046764c0269be comparators/qS65.sh -864b2d74fd698a0d0ecf2da3e2790dad3f04d7ef31207b113ba7e5cce26daeea comparators/qS66.sh -273ed986ce9ece2678df72ccf22b1238ff04db1e425c432fe5847727dbcbd431 comparators/qS67.sh -c7930602a4a57437a504f778104482cd0ae105c0d6ff768bf5d7a0f36290da5c comparators/qS68.sh -36219312cc447f371b03d0d796a15cdd28ea172fefac1d6e5e34c7d46ec17991 comparators/qS69.sh -ae17839539b788ce03b4ebd209f49c41611402556eb367b8e6d963fdc17cc992 comparators/qS70.sh -57a2abfc85967609ddc4ef6ecdecd3e90e4f9004add8d21d6978e1ef7488c1f8 comparators/qS71.sh -7d62debbf09f20f55ff486c0925988684a8904d56763099054b43652ed70e32f comparators/qS72.sh -ddf971a1c28d76e9a54b655318211239510ac786c8edc7c55eec0bc1e396135e comparators/qS73.sh -eda93dc12c75af0cbc80f569e8d693112b02219c9b5aa8d255769379cf25b81e comparators/qS74.sh -8fc96c99784103c25b096f9d34238bf82feda17f50f7ea069ec1b621abaf7f43 comparators/qS75.sh -b346328ca1c9f9aad0bfa5321326375ab180b721257ccf6fb88d545722ab6691 comparators/qS76.sh -e9d839af9033b83211a2f86dec6dfa8dd2808d445633442660b5f2e762cee63c comparators/qS77.sh -020f630e213525c94bbf9ad2cef12b775db35e5cad9cffc9c80585e2ca00fcc6 comparators/qS78.sh -48fd0ad4f27dbe60669a4d2ba04f796d2527f9a43d6bfd4148cc38cd5f47da6d comparators/qS79.sh -b6b7b906f1dd93483339974dd5467366439f4cf98430a05afa41b0e5633f7b17 comparators/qS80.sh +880248ee07911605710644dc10359b7ac4077bd24a523308df680440505ed75d arcadedb-release/qV01.sh +21d4d129a5ffbc288f74b4f2c79ce8af66b40428ca0b3b4b71a01db887dfe617 arcadedb-release/qV02.sh +25c5063ca6f56d5f9ef3184ba7ed557fb49ba229ed573d0adebe2be3c55c7d04 arcadedb-release/qV03.sh +2a4018d8c41b273549e4307bc0e74c39e2dc503c75d07a79bf54c0190336849d arcadedb-release/qV04.sh +c5e06bdb46da6c74888baa74dc45f445dd16ca43e4a7d9073d940fd7b3229e96 arcadedb-release/qV05.sh +002dd1472ee1c3abfd08912ff94dcb4236709290d2ff093704188bf4debb2f1d arcadedb-release/qV06.sh +dfe781f4874d720158fc47b635011594f5e35f8504f7f2b09df00e82b43258e0 arcadedb-release/qV07.sh +870491cc544c2237e8a36effe58738d4acb9e5a079d7a0f5d5696178e12bc050 arcadedb-release/qV08.sh +2d350ccac0991d9675d8a0bfc9bd4110590a4d8f6ce392ba320957406322a6b9 arcadedb-release/qV09.sh +3d831b3c3a679d6d217a18a5ca462214430d2c9f29e2de5a0ee3533ca2c31265 arcadedb-release/qV10.sh +7861362045fc18a69c4bae4c033578d06ba0549dacd5837919dc07ea356c538d arcadedb-release/qV11.sh +cb03d4d22be931f37947d7d0095c150e2371a8c02e0e8ad0225283c7ea4199ae arcadedb-release/qV12.sh +eb17073d66cf712f6b987951c17daa9755a25d7b2405db4015e1213bc0eb953c arcadedb-release/qV13.sh +43b77f81b55888478fa3ef0c563152eda2f09e0a3671abbd50490b8bffc236af arcadedb-release/qV14.sh +f9ec5c46e26f5b8f27a2a0c5bc754c28a176bcc236052132b53462d202bf5804 arcadedb-release/qV15.sh +b34bb921f19a6536ad6dadfef72d5ffa52bb4e0f92a52360086c5bdc43607dfb arcadedb-release/qV16.sh +d4516fdb9511ba91a917e44c1d91bab87df1cc5828ffe90880d43a64f313d453 arcadedb-release/qV17.sh +4c8a1cfd0be5947adf7d728e50da228c83fe4fac6b5c5bec38f4b95fadc74a00 arcadedb-release/qV18.sh +23b9c80c70a6ab985ed5854203290917e7bdf4057e87a010948732f3641284c0 arcadedb-release/qV19.sh +309554edea8b488d127681e31514d5adc3c80dfda1335b371c49ba59a66f41d4 arcadedb-release/qV20.sh +eca89faac681d20074f0d3a98476929039a51d9351b5783ccabec633fee0d04f arcadedb-release/qV21.sh +3f88e0c337fbbcf0b0c484a009c6fce6a8ecb1cf0e9e6e30ffb8d679c5b1a51b arcadedb-release/qV22.sh +3a7c19a718744741089ed997d9296a2152002df3bd1d196cbba7349cda889688 arcadedb-release/qV23.sh +97a76ce182c9a1327d3f8c08873245eaa4a59a85709cd1dca24a7deb884eccea arcadedb-release/qV24.sh +bf9502169ce99a43a03819a0fa60347e4071a704dd5e830717acc6e4253dc69b arcadedb-release/qV25.sh +b9b606fd1ba37d92ed2a87b175391c35837ae51da3bc59920d0c1cbe536ff3df arcadedb-release/qV26.sh +743b6985da5b519d398830e9a123f09d624bdfe5672863b80e95dd8742fe958c arcadedb-release/qV27.sh +dc5f3b21ae4b249a3a95ac1c517e5ebbe47a2cefb41494eac9aa6cc66ac1fc95 arcadedb-release/qV28.sh +c6fdc7d73043f1b7e4dedb0ab27b041bc70e4e257bb8bc5a4db6f55b34330971 arcadedb-release/qV29.sh +b79fcc289e65eb75cad99864ad935f4f346d10ee6761b6751d01c3a5a23fd4fa arcadedb-release/qV30.sh +68d3ed8cdd0975014ada3c6caeb721cf145a4f8b7a8b8a5017a1e0e1341b2363 arcadedb-release/qV31.sh +787c9ba2deb5a73f386f7f2657e7d15d6ba4bd347f553a0c8e54813016f56dc4 arcadedb-release/qV32.sh +7669bb3a3da91289309e89633b96ccad26cd6f71deee9d157b12452abdbdadf8 arcadedb-release/qV33.sh +85b69af5eadb3562b3acddcb75301cec4982c0779cef78cbf8b0b021bce52809 arcadedb-release/qV34.sh +5b09b660d9919f081d5663911f262037de650955740e7ce4d0eb07b6aef724ea arcadedb-release/qV35.sh +3fd4e7e38f20f89c52e5417a4012523eaeae45c19457a874f0e9e0fca12089e3 arcadedb-release/qV36.sh +7ecf5294cb22381e41591ccf524ea0b5bf2bc28b33545dd105b298f210c0c171 arcadedb-release/qV37.sh +b3b33ce4560d394f21e49ab4289ee110574a1c3fc471484b1dcbee4ff8777b61 arcadedb-release/qV38.sh +05e92a1798cee192a308d6a4f04a8484c356904fa508048a33f43c508f7bf26d comparators-no-tiers/qS01.sh +35664ca5813579380f5b7fab1733ea8b139a98a5eb02abe3bb1230af1423b2c8 comparators-no-tiers/qS02.sh +39d2d2135f966ce952b1c4765912257ccf912f3cd55a1e3af3ee3eb49b14b568 comparators-no-tiers/qS03.sh +7bfa62c5fc8ac4cf35f47ff2d76671ff59e2e8ff63ef3ab79057a34b8b602ed1 comparators-no-tiers/qS04.sh +3e5666111d1df24ef22edd407861d92262fa70c545b55ec57783e03d71da9624 comparators-no-tiers/qS05.sh +3b8581e36c7f75edb6af296d72ed8875d1ada665ed63a5ae5e1d7cb539a6e5de comparators-no-tiers/qS06.sh +d3362906ccad89c31f8e3c0fc31031c39f971f08d68a971f754dc59a4c50ada2 comparators-no-tiers/qS07.sh +7f8d43e613c27c020589d0ddada5bab185451407bbe1d5eb9ad8310bb2ee329b comparators-no-tiers/qS08.sh +5405e051bda18e17a75eb4983c4644ffc8fbad4e10c2bed5479a4edecc1aad14 comparators-no-tiers/qS09.sh +da895dc77b0d1ccdfccff70b68849f74009416d2ac1e61ddf4ac7fbc1a59bf6c comparators-no-tiers/qS10.sh +08a69a0bf1a608b20ad4fd42e4abf4558c3fbcb68ef3701b959037061c36888c comparators-no-tiers/qS11.sh +9e4094e6d4c06749a106d63d0ca7caf209b5d98f1b752ddb4b1940fdb9ef983e comparators-no-tiers/qS12.sh +edbb97509a5a22bf3d5b2a1c18a839ff8123e2b7cfa2ee7958ed9ae1720ae39f comparators-no-tiers/qS13.sh +ceeedcc0f9e0cc48ac054389586c71107be0b919cc0f04a3e7866bf472bef2b1 comparators-no-tiers/qS14.sh +9a6557563800105dadd11018d37f4208f3e1e56b5c9c2a4200e8654c078387eb comparators-no-tiers/qS15.sh +e2c4ccabbb8469d214718385bb8f94593b4a223484a4db9e9515854657f506d7 comparators-no-tiers/qS16.sh +a5b912affe563bfa7dd4957bc5c5bcf49177f7c6e90a28c68e27852732bb715a comparators-no-tiers/qS17.sh +d694fe08b239a6effaddba046ed97202bdf2dbbe531b13d4cd90c577f1d6c849 comparators-no-tiers/qS18.sh +3b3d99a36b182ce0333c21f66db3432eebafd2fd73ee867971d961099d13b714 comparators-no-tiers/qS19.sh +81b46a1cea3b7a587cf7dc445a05fa97b542b4591486d99293a85a860cefb549 comparators-no-tiers/qS20.sh +b492613948c311bea80ddf4c741131877b975d4174efb8b7207a975ab4e361c7 comparators-no-tiers/qS21.sh +65d0087317f02e73d127c0299d99248912c4f393e18090bcd8877dcc6db0adf8 comparators-no-tiers/qS22.sh +0413d6592454c417491e22d11eb4422becd8252b0422e75e498ab21d56d33e98 comparators-no-tiers/qS23.sh +87ba215bac324f9eb5583b5ecbceb893f0ae3d1a64238bbec472b6784c671f17 comparators-no-tiers/qS24.sh +6bfc1ea4ea72f94e2db516d99500075c52ac3b300d61b850c8ab82350ce7b59b comparators-no-tiers/qS25.sh +cd31e0faa645e114cb01bedeedb73aa4fdcd7ca4e3208203f1302d69764ff171 comparators-no-tiers/qS26.sh +c7fc222203c1de8423b9fab7d05345e6263e40a0f13521d89b94b6e5cd89c03b comparators-no-tiers/qS27.sh +f8cfe4d2bc52a851ca35632881a45a2d6a9aa579939776db30d85044d236e90c comparators-no-tiers/qS28.sh +9a51f207ec605ea1eadc906fd09067af1e95adb85e20155fb775dd412b3a7cb0 comparators-no-tiers/qS29.sh +59908f6617b07b88153da415ba62b45aa467f0ec51de1f156997d368f2bf939c comparators-no-tiers/qS30.sh +26e3d92e24f526bedb9118725d11d2094e313b1aacb9f7e201de970111075a19 comparators-no-tiers/qS31.sh +4e4bac92f8c3490242cebd53acede6fd3d0d794817ce67f1a2df1d3c0a9a6f85 comparators-no-tiers/qS32.sh +871f512bdb3a7e231d4629f7c20a4ec0b8276bc091879993dd065040787e2281 comparators-no-tiers/qS33.sh +3a3b033eca3c3a808246bb4e0e16f039dca6a4302f09ef3a60f56ddf2611ff95 comparators-no-tiers/qS34.sh +9ef3a6d8cd9baa2ea7e3693c58a30dede0986a33196b4756dd3a418e329aeb83 comparators-no-tiers/qS35.sh +46c1272238fffcbfe428ea475fb81b1978f6b2105b5d226f6e5de5e1815c3083 comparators-no-tiers/qS36.sh +fa878751da894da8a6836c41382ecd477dd8d8e298d4dda175d24e0cb238b2a4 comparators-no-tiers/qS37.sh +0b61a3deaf0fe662df0b6878cd04cb29fc4818451e6492e67540a545469ec913 comparators-no-tiers/qS38.sh +d28a3e3851a819c21340d1f51e8fe007c6559932568aabc0f748c2cc9c8e28fc comparators-no-tiers/qS39.sh +08398058104b0020e7a9fab97501463d74e05ced98122b0caa0a11db85eab5b2 comparators-no-tiers/qS40.sh +25299f90cc78ce175c824a74c797fa27b1cfb1511157affdca4db988485e0911 comparators-no-tiers/qS41.sh +7cf0c1a9bbf06164584bf9c13069cb1bb0a4a4ee56c1ea41bfe00ec11095f5f5 comparators-no-tiers/qS42.sh +655be78808c020c0accd2b700afc82a30b42ac221a6b31e5037394c6fd2fea0c comparators-no-tiers/qS43.sh +72b7db64099d72ece8a612f9ff18953e4781674ed147b6a1752a08b79ada9fd4 comparators-no-tiers/qS44.sh +1a8125125f129af8202b91764b8599e4ff3df1dbce853247540fcd6a69211741 comparators-no-tiers/qS45.sh +786a4a7c795bb2bb6d0e36708cf0a9a9a8a2abdbac8b4b52912ab84c83293377 comparators-no-tiers/qS46.sh +66f134f12a02e79771c83ef571e972bb72d07ad245c9263825eeda212d3a24e3 comparators-no-tiers/qS47.sh +a46532d857cbc03435cf4eb47444da224315ff88f62ef0460d892c837003df62 comparators-no-tiers/qS48.sh +a38afd5209b7cefaa49a6d146123d5516ce00ba9e0bad785bbd8eb866c038f92 comparators-no-tiers/qS49.sh +f8d59bad9fdb1ba26fe49542b577473f7bff7e053b25c53e739a6d7c9c425403 comparators-no-tiers/qS50.sh +9d82cae07da7bf8b427f230c0f9773844695c18988fc785db76895d98279f9be comparators-no-tiers/qS51.sh +d7a97f6cd817c9127a1e86cd23b6e88a46205bfdd8dbe192cec98eff180b56b5 comparators-no-tiers/qS52.sh +f621e3bef174795c66fbea4ac65e0e02e0c0784e9d4f6114874b5d771d5d40fb comparators-no-tiers/qS53.sh +85960f6e614dc4ce24157940d8e2f8a18aacab422054de819cdf97e09a2652d0 comparators-no-tiers/qS54.sh +a33faea331d8ab7b6487d1bd597783667ac27a0d9e0b0d72c8ede389a026a1af comparators-no-tiers/qS55.sh +e3f630ef19df80338b535c68aeb6319a11f508736cc8439faa08e96d32185a25 comparators-paper-5reps/qU01.sh +ff043af353a9e200fdae3e70ef7c4f8dfc1f4b8d35561799c3317a1f31ae2fd9 comparators-paper-5reps/qU02.sh +221eaa0020680705a5ecd0235b6458035c99ecad8060e77d32c1f8c51c5a4604 comparators-paper-5reps/qU03.sh +afdd694681bee7cb1d3c964d258520741df46ed8d08d336d7cf59c138af15c7e comparators-paper-5reps/qU04.sh +7f977afdc49a4e8804b875c511d940c5bb8001932231a60ea2327aa2851c0262 comparators-paper-5reps/qU05.sh +c024bd60c694e665ffa247dbf26acc32aa318c6a5f8c1fd33685bd2f7786b3ac comparators-paper-5reps/qU06.sh +1302a6b124e531b021e680ebea2734a8031538d44ded8a48e0ea584c0d91e25d comparators-paper-5reps/qU07.sh +5c7e5083f589a62e7ad13e572ef6bf018b90cc87638544fc61f6ca8c7db68ae1 comparators-paper-5reps/qU08.sh +04e764ecebe65ce5b610d3ede2927704b8fd02d5a82f4436f88635741bc46153 comparators-paper-5reps/qU09.sh +bb966d4942aceef6566390a1f48b3ab44b6297be1a47875f56801b81c50fd947 comparators-paper-5reps/qU10.sh +5006a6fc396d4145135f4e61512a5cb9758e6a1b825271ca251bf9c302a53245 comparators-paper-5reps/qU11.sh +5da837088e809c27720123b4729cc6340d2786f0673b0c912f45e4c2a1406683 comparators-paper-5reps/qU12.sh +4160981ff62218f88adb20d549d8db358aea3ee3a00c8ea84ab833ff7a9be07a comparators-paper-5reps/qU13.sh +2cd8fc9df4b3bb1edb2c418549b24d31d65f1f3ba3c964ade3929416519d8b6e comparators-paper-5reps/qU14.sh +29277d4cff04883119d41a4aa0b50b112406d5290a3067bf1e2a73a8600230b3 comparators-paper-5reps/qU15.sh +f24470fe5aacc9f15ee6af1abb95398f8ffd2b2609ce6a9a363c01ae5b19db89 comparators-paper-5reps/qU16.sh +daa0b367074201485c4419edbe122edae5218336087e6ef399c6338125f6f9e6 comparators-paper-5reps/qU17.sh +674275fcddfd150769c9c46022d3498a10007b8fbb6b45933ae452f371b4d6e1 comparators-paper-5reps/qU18.sh +bf3e86e153caa92c3e375f6d336cd1cfab7b676a91ba56a874a65985ca6719b0 comparators-paper-5reps/qU19.sh +7e3b7b95bf469417c498811ad6215d522a8855362d244c45de0017e94dbac48b comparators-paper-5reps/qU20.sh +7a9df35ccc6fe0ae364a497717af663e8cd600cdd3cc55638172d2f62b1e10ee comparators-paper-5reps/qU21.sh +b24fafb149f0db0a909c2ac82faf570af1a5b613c3df5614eb29f79841ccb704 comparators-paper-5reps/qU22.sh +71bfeb0e337d53356853b1c5ef9cf719666c67f7720a238c1cdfbd5f56470843 comparators-paper-5reps/qU23.sh +dba3abf155072f16d23323e2fd8f92b1f15daf6a2244f3d3b317c9eb09912333 comparators-paper-5reps/qU24.sh +92390096534f41311abb129513ec16bd7e38ac69380593043ad560486a01d5a6 comparators-paper-5reps/qU25.sh +cb55d72e2e8fac3e18382ddaf28edfb8251268d3799c25a306d9b7094f0a78d6 comparators-paper-5reps/qU26.sh +45303025aa7631912306d175ecb9febf441f38ec92891a3160e1d8bf1ca4befc comparators-paper-5reps/qU27.sh +5a3635fa1b8475a0ccb4f58787f68da50e59085ceabae3f3b7643c8fdad5e945 comparators-paper-5reps/qU28.sh +990ef0a01bc08211715bfe72042bbf54e9d0a6a6d3b714fc0e7515982ff9e55b comparators-paper-5reps/qU29.sh +15de51b4b723289935fa166e425161c84f75a69104b189abd6022fb48f744ea2 comparators-paper-5reps/qU30.sh +e0112432691c164ffc45efbb4c6f7c2568a24fdef55459ed7bb251c8a820d69f comparators-paper-5reps/qU31.sh +0e92522fed9416f2f8fff1e72be287227f576dd5092370781c772cbf66992c2f comparators-paper-5reps/qU32.sh +0af143ac2cf03477c4e56b5c986406140bcebd4e134ecf84aa77543c57850a80 comparators-paper-5reps/qU33.sh +41d0ae690b6d0e2238912ab4ba64e67693660db97cc576fbea8727c1eab9b844 comparators-paper-5reps/qU34.sh +e4e5e2a2ab8ee25594c42921efacd0977855ecb5ece85210ef975d811be89fef comparators-paper-5reps/qU35.sh +8c8f215c01b6f7dba5917cef1311b372852ec03c2eb32d2249433e0abd10f084 comparators-paper-5reps/qU36.sh +b18011acc51bc523acc4ceaa270ea8f9a6f670579897fbeab9c92a4c70f40411 comparators-paper-5reps/qU37.sh +bf58bb1d9dbd4a6bc02548260b5d0a5a150d5660ee73f530221453c2218d40b7 comparators-paper-5reps/qU38.sh +51824b93fdb5983b7bd1c1c6fd73ada340bac85fcc7601e7670e2d7277824f16 comparators-paper-5reps/qU39.sh +2903f437c5e1d2cc4ca997b23d8bd811dc33647f774720d6c2dcfe9bd0248e0b comparators-paper-5reps/qU40.sh +02ab1400ddd4e5af509fab9103acfd2a0e685eba439446d2acf165ea36eeaee9 comparators-paper-5reps/qU41.sh +63380a92bcfd78ba6bb901d8da285cdc7fb0673761bc8afbcb46c52f4c20ff3f comparators-paper-5reps/qU42.sh +5d1977fc28637a67e943679dec6223a9b8769a9d026ed41b17ad6f750a21692e comparators-paper-5reps/qU43.sh +ac891df35d52bcbb5de4d3d6bf055ba4d5b19ffbe59103237876cd1415ed6668 comparators-paper-5reps/qU44.sh +be49884a93c42dce03a7eedeaad8eb0108fa900df1f5a2e9dff2a47e23c22147 comparators-paper-5reps/qU45.sh +bc84b5821b9f6d2603ba7f04c70af9ecbd0d6def95ac8ac64b873d1b645b05f3 comparators-paper-5reps/qU46.sh +db9b62a87c5275abc5bdc070720a6f12563814d07172daa94b49a43f69ec67c3 comparators-paper-5reps/qU47.sh +e48494dcf0ae7b01372f3c9225aa54b63cd7ca2a79e178c36edeb2016c4df153 comparators-paper-5reps/qU48.sh +7f1bf5c0eb46578d49c85eb5bb8fa4ee413ad6cf681bf0a1af34a8fcb3ddec4c comparators-paper-5reps/qU49.sh +61091542b7086cb9016c1f9721469e12a3cf87cb792d7d9a751059a76472b58a comparators-paper-5reps/qU50.sh +f2b24cdf41954d73cd084df3425a78279d2ba2e5018a9f89280e09ee1b187c9c comparators-paper-5reps/qU51.sh +d42189a120c09a091bfe59ff53618de401114b10ecf455b38e96304997554f1c comparators-paper-5reps/qU52.sh +b1da4f4d366a6acb2e2e5a8f975c69f5a66d4787966c3b31d858cf0c694f05a3 comparators-paper-5reps/qU53.sh +03ad2a5d14b41344344ce2b8b8a8d09f47c908c071f74a0f6a6972cc884fd084 comparators-paper-5reps/qU54.sh +7c71642170516a4561af2c776ee82bd1305a9aca89fd975e6614590a2cf9c447 comparators-paper-5reps/qU55.sh +e75ff08f647e5213dbffaa1a688e6c5eae9f49ace556c2f39098401f21274428 comparators/qS01.sh +e75d0f9641cfcdc8842b27af653b4b04dcbc143eaceb9612a2e28f25bba4dda5 comparators/qS02.sh +df357adf599df1d0bc686a0196c008f6d3d45943b1c44955ea6f5ff8af639042 comparators/qS03.sh +8f51429d2fc57d4a54b78059602508476b0333764529c0d39d6c529d6dc19b6c comparators/qS04.sh +19579a09802154f891da67d05847d2e15a85c4789a4427d4478de10c0a7da03d comparators/qS05.sh +4876984bb1807623e5eb85d4ee47432da530b39522ae7c7904fb60c4c8a4441c comparators/qS06.sh +8a5d2a7daed2a4b4170ed4c2eaa4f34cf3018c63e61e1e02ed3f34eb4e239ac7 comparators/qS07.sh +e10bcc5a09fe25581d671daffa628236b35e926c61e54f37cba68dd7d63af42a comparators/qS08.sh +120be2dc61c502dd4f8099ff5671a9cc876bbee5dfb1802eefbda3bca8c42316 comparators/qS09.sh +6650994a6dec2ad1c4beddee6b89fa25e03b698529c3ccb88a84224c699ac0e5 comparators/qS10.sh +96892e784021c235be204b26de8cd51d9066eceef13d950421806bb954d6111d comparators/qS11.sh +8ba158334be5c4c6ece5368645afd519e046ba55bced8c26abd591e1d6d306ce comparators/qS12.sh +0e1d4c4b62528afa4b8cc5da161f87b1bc237e2359b371033bfd6ab732c3691b comparators/qS13.sh +dfb81488622d8beb601f7e29cc95659014fb0b180994c6d7643101362b65cbcb comparators/qS14.sh +12243664b979fedfd9cdf4ac8e42908f1c4f90841a29deb5617f24be103f0003 comparators/qS15.sh +f79e036f839df972c99e2c30619fdf56e31cbaddea573577c9a898051e4a65c8 comparators/qS16.sh +1731e2d19960cd2a0800808b216196e9b7f1ade3ca03c13ef26f484437d6fe6c comparators/qS17.sh +a5893d2478b157a54dc44b8a894a884ff04c79a40c2bf98405ae2a239980e231 comparators/qS18.sh +d1c0bda875d9f1b306e6432be595aab8b07dab93e759153ccdca118781aa4ad3 comparators/qS19.sh +5c653781598fe5d66be23fdc0cf0226191c28ba8c2dfacacdf08df239cdeb9c0 comparators/qS20.sh +2992d25250487f127b22937217d13ee8a55c31579ac630d3b80d1118863d8509 comparators/qS21.sh +3a19f92f4bffe74851450c33dc565773f81b993aabeaf0c7e36162da0adcc021 comparators/qS22.sh +7a22ce51b4c3af839ab2293cfff2989b683dd3c0c4e3a9339412a9e7f2d55e98 comparators/qS23.sh +44f170993b659042be1d5bae9e713141519aefacaab09c09c98a0ba94787230d comparators/qS24.sh +ee5c5994173ebe3a840bebc2c86a747b54704aa6754fd713c0d2822a95beb586 comparators/qS25.sh +60ad2fa854302623b5097ce3e9d4f884f4fd767d5c2c6964f9ab61f437be0fc4 comparators/qS26.sh +474ed30a0c7cef26e8c82a0785797988b6697ef27ca2d401d0cb6af819bd21e3 comparators/qS27.sh +e8a5e3b27d5ec52f6af09899fa8d4dbd50568115baada390f52a94fd1783cf63 comparators/qS28.sh +654c81ab4db74ea3dcdba70127c8f372d402a5a52ede5df0067c98b4f2cd6f98 comparators/qS29.sh +8b67b37dbc2ad64301f4bbe3f68535427bb98fd1440df4ec37492e4677ef8bdb comparators/qS30.sh +1e63d3c0e8664f77aef8e5d266bbb1a6d5da90fe0785ea602b2e3e01f40c10ba comparators/qS31.sh +c633fbe1901dfad7741f79cae8ac10a34fe29a17f4d2e3d1255c67d76477ff67 comparators/qS32.sh +dec0cd70778cb9bbe1b35467a12832199b056f937c28006fa42c3905d8d058a9 comparators/qS33.sh +21f65778b6c86e4ad6a74c50e6cc8dc8dd3aca426f921e07f801eb3ede34ca19 comparators/qS34.sh +f674448d208a5c4be942ad8c8571f2ddf3ebf169dfd56fc5661ba09479eb8399 comparators/qS35.sh +eafc282e6d847614b1afc23d775d397f1352b7c94a3ce466be72b09838c65a1c comparators/qS36.sh +0bf13f446dad06bbb1728973aa62451d5d93008deb233a06a28105280a4a7f02 comparators/qS37.sh +ad7fe8174fb4bc53fe2dfc26089674f1f4c55b64afce794049d62d0474224453 comparators/qS38.sh +a5d74c361988ba31cc21e2c0e9b5ca5a87b28991d3ed6b84412ac0f9db4872df comparators/qS39.sh +18e034ac094f816c9260929233ce482b4d853c84258d57477d3712e2a95a35b1 comparators/qS40.sh +044c227b93d5dcdb29d43f35195db663ca59d3045056219b15d5684ca24a10b2 comparators/qS41.sh +86e4231659e61677786dca59c36bec9bb970c11c2f05163f46012f5a66a47b07 comparators/qS42.sh +6ca2b04f9a585990402311bd67683deb5057b606a6de0c89b01577e9deeff97f comparators/qS43.sh +353383751d46cf739f4a4b7022bd49530d1682efbdba08eef6c8e53285210a9c comparators/qS44.sh +6b6f65746696ba82a74c4e45df0edcd9fbda2e1a178e5109328fadb1bf6525d1 comparators/qS45.sh +9e5f2266a4bbcd676b0c600cf45778a2f59be583a58d3a9c439417f925801b36 comparators/qS46.sh +4ccff1b45f678a506a780f1144163ffd874793acfe9140cddc208bc8f2cfea95 comparators/qS47.sh +749256cb54bb12c3c747072e44ec93cbc2062b58a752575eb090d8cb07d05f05 comparators/qS48.sh +084d17b2ce4eba2d011e4105c8f718597b8bc1df83473099c42531a00f97c265 comparators/qS49.sh +ca18d85d1d642e48fb297baad741e0c9e771de8db2f4a88d4f45eb8060a7a45c comparators/qS50.sh +45a9c9ee819c19443f3361a758ca424a535c7c00357af0ca9c36e673cec802fd comparators/qS51.sh +a4d17f1ffa2e75e9bf9cec376f0cb7f20ce5762bcfda0de516cd28cac16e32d5 comparators/qS52.sh +b7b8f3d1c4db924f3cf7344766177f3998f86893ac3d4e87f187047aa3f782d0 comparators/qS53.sh +14f61d236c2aec1c54853289eec640935ee805b6ead36b86a9e7986d9c994f98 comparators/qS54.sh +0ec894a77848e2012dfa18d468fca8e04dab833118ab9872b4029317736bf4cb comparators/qS55.sh +d3bb6a8a06c1440c8c723e095623953ae0bc038321b210f7540c74036c5f1dff comparators/qS56.sh +5c54fb12962d7aa4967e1e3782b6494e68e167d234f843986f209cfb3ded15f9 comparators/qS57.sh +c1ca92894f14f4daf131f75448eb6da06e091b910a992f0ccf6a5e7d9efdd34a comparators/qS58.sh +0396b07deb34d6c1fdedf49b15f663f5e4b4854fe15b8b8e8d1ef54ed025b34f comparators/qS59.sh +8ac06c5dcf6acda59125f37a4b6b4394f403c6e33960ca18db839cc272a4e3dc comparators/qS60.sh +60c4fd7e14c854a0ed5c6ecc73d017a6e19bb1ba5f7cc4a0191abf4c4d26f7a9 comparators/qS61.sh +d700d242ad5d1584552641b200af30f4684cbb74ab6f0d3d64723946bf34ffc8 comparators/qS62.sh +75fa90d4ddd0ce01096637b3cfd302c95f20d32d5e60064766b51f1ed51e0e83 comparators/qS63.sh +78c5cebd2939e71142971c7095b69306a353c245213c203e0f0d7fb18267c4a2 comparators/qS64.sh +b2047e92a10b769ed424d5f81de37ff8889bbd9db1055c87a836618057aadf79 comparators/qS65.sh +88223026bec0a1db63470a7521b52f33094b2a762b500766c012b6d87a949304 comparators/qS66.sh +eb3384545e70bd4003090252d55fd0bf439287b95dcfa56cc28835201d128e3d comparators/qS67.sh +4440dcce39193142e89aa6f9b0b5abc32c70dcffd4c9131448dc0c1a4f533060 comparators/qS68.sh +b42c8b2af13cfbf993f2481805b15c245f63c6cbb79b8fa41d35de9b2b9377f5 comparators/qS69.sh +5687822adc8d9141978d482ad916836e26050edc5588500eb2a8bf8ef3340a03 comparators/qS70.sh +79c6bdcc81a7125981206ef24b0714cb8c97d00f77950aa6cb38e8093ac88d2b comparators/qS71.sh +d3cf1ac2529ff62b23b1e8fb1bff8f682e2db90db5ad4ee7adde4a61670a319e comparators/qS72.sh +bcdc7a6962901a3423f3e91c1f801a0d028b2ca22d37906271d7e77bd18733ed comparators/qS73.sh +43646d87943bb1053307ea1908c907f1fc224cc4dbd1d1e8fd6bd1e35c6ea8f9 comparators/qS74.sh +bb9ded5f1b0f7a99af1abcd155323b0230cbd32454d05a599844c41def9c7154 comparators/qS75.sh +7399cc0d017f79e21fcf37f2a55d40420b9ecf9bbdb0ffe0abf719a1ec80f37e comparators/qS76.sh +3c0642acd4d61f481c942828663e49a8ed17388fb28f9651392b0fc7b31a5a2a comparators/qS77.sh +23a7338aaba164b668043b02a8ff9ba3d054808641b615a20aa69d3618f45fac comparators/qS78.sh +cc0d47d2f83ac1612adf68d5696cd97d2ab98d7a2e7429d041fed3e42ac2effd comparators/qS79.sh +ad1c0af629570ced16e59b1d2c878fe273866cdf8406b0bd6ea6079ff52584a2 comparators/qS80.sh diff --git a/benchmarks/experiments/test_engine_pin.py b/benchmarks/experiments/test_engine_pin.py index 8fb1135b66a..2e87026c545 100644 --- a/benchmarks/experiments/test_engine_pin.py +++ b/benchmarks/experiments/test_engine_pin.py @@ -322,5 +322,5 @@ def test_the_stage_generator_builds_the_arcadedb_target_whatever_the_images_name def test_the_stage_generators_checks_read_the_versioned_image(): text = (HERE / "make_2611_stages.py").read_text() assert "--entrypoint python3 dbbench:arcadedb " not in text - assert text.count('"dbbench:arcadedb-$WV"') == 2 + assert text.count('"dbbench:arcadedb-$WV$WB"') == 2 # the wheel's build is the tag's suffix (26.11.1) assert 'dbbench:arcadedb-$(basename "$ARCADEDB_WHEEL"' in text diff --git a/benchmarks/experiments/test_jpype_pin.py b/benchmarks/experiments/test_jpype_pin.py index c8f748afbf5..b753922d7b5 100644 --- a/benchmarks/experiments/test_jpype_pin.py +++ b/benchmarks/experiments/test_jpype_pin.py @@ -315,8 +315,8 @@ def _run_snippet(tmp_path, preamble, snippet): @pytest.mark.parametrize("image_says,aborts", [(PIN, False), ("1.7.2", True), ("1.8.0.dev0", True), ("", True)]) def test_a_runner_stage_aborts_when_the_image_runs_another_jpype(stages, tmp_path, image_says, aborts): text = (stages / f"{_stage_ids(stages)[0]}.sh").read_text() - snippet = _snippet(text, "IJ=$(docker run", "ABORT: dbbench:arcadedb runs JPype") - r, status = _run_snippet(tmp_path, f"WV=26.10.2; docker() {{ printf '%s\\n' '{image_says}'; }}", snippet) + snippet = _snippet(text, " IJ=$(docker run", "ABORT: dbbench:arcadedb runs JPype") + r, status = _run_snippet(tmp_path, f"WV=26.10.2; WB=-jvm; docker() {{ printf '%s\\n' '{image_says}'; }}", snippet) if not aborts: assert r.returncode == 0 and "PASSED" in r.stdout and status == "" return @@ -326,6 +326,15 @@ def test_a_runner_stage_aborts_when_the_image_runs_another_jpype(stages, tmp_pat assert "Dockerfile.bench" in status and "rebuild dbbench:arcadedb" in status +def test_a_native_stage_has_no_jpype_to_pin(stages, tmp_path): + """The native image has no JVM and no JPype: its stage says so and does not abort on the empty answer a native image gives.""" + text = (stages / f"{_stage_ids(stages)[0]}.sh").read_text() + a = text.index('if [ "$WB" = "-native" ]; then') + snippet = text[a:text.index("\nfi\n", a) + 4] + r, status = _run_snippet(tmp_path, "WV=26.11.1; WB=-native; docker() { printf '%s\\n' ''; }", snippet) + assert r.returncode == 0 and "PASSED" in r.stdout and "native build: no JVM, no JPype to pin" in status + + def test_the_stages_carry_the_pin_the_generator_holds_not_a_literal_of_their_own(stages, monkeypatch, tmp_path): """A pin moved in bench_common (with the Dockerfile) reaches the generated scripts.""" dockerfile = tmp_path / "Dockerfile.bench" diff --git a/benchmarks/experiments/test_native_jvm_arms.py b/benchmarks/experiments/test_native_jvm_arms.py new file mode 100644 index 00000000000..f3454155436 --- /dev/null +++ b/benchmarks/experiments/test_native_jvm_arms.py @@ -0,0 +1,251 @@ +"""One release measured on both ArcadeDB builds: the identity of an arm is (engine_version, build), not engine_version alone. + +Run with `python -m pytest test_native_jvm_arms.py -q` from this directory. Nothing here times a cell, starts an engine or builds an image +(docker is not run; the engine is a fake). What each test proves is named in its docstring. +""" +import subprocess +import sys +import types +from pathlib import Path + +import pytest + +HERE = Path(__file__).resolve().parent +sys.path.insert(0, str(HERE)) + +import arm_identity as AI # noqa: E402 +import engine_pin as E # noqa: E402 +import _arcadedb_import as AIMP # noqa: E402 + +V = "26.11.1.dev0" + + +def row(build="__absent__", backend="arcadedb_sparse_embedded", rep=1, ts="2026-11-01T00:00:00+00:00", **kw): + r = {"lane": "l3s", "scale": "tiny", "n_docs": 100000, "workload": "search", "backend": backend, "rep": rep, "ts_utc": ts, + "engine_version": V, "gav": None} + if build != "__absent__": + r["engine_backend"] = build + r.update(kw) + return r + + +# ------------------------------------------------------------------ arm_identity +def test_build_of_old_rows_are_the_jvm_build_and_served_arms_and_comparators_have_none(): + assert AI.build_of(row()) == "jvm" # before the field existed + assert AI.build_of(row("native")) == "native" and AI.build_of(row("jvm")) == "jvm" + assert AI.build_of(row(None, backend="arcadedb_sparse_server")) is None # served: null on purpose + assert AI.build_of(row(backend="arcadedb_sparse_server")) is None # an older served row is not "jvm" either + assert AI.build_of(row(None, backend="qdrant_sparse")) is None + assert AI.build_of(row(backend="arcadedb_http_client")) is None + + +def test_the_same_release_on_two_builds_is_two_identities(): + a, b = row("native"), row("jvm") + assert AI.arm_identity(a) != AI.arm_identity(b) and a["engine_version"] == b["engine_version"] + assert AI.arm_identity(row()) == AI.arm_identity(row("jvm")) # an old row and a new JVM row are one arm + + +def test_a_cell_with_both_builds_is_refused_unless_one_is_chosen(): + rows = [row("native"), row("jvm"), row("native", backend="qdrant_sparse"), row(None, backend="arcadedb_sparse_server")] + with pytest.raises(SystemExit) as e: + AI.select_build(rows, None) + assert "BENCH_ENGINE_BACKEND" in str(e.value) + nat = AI.select_build(rows, "native") + assert [AI.build_of(r) for r in nat if AI.in_process(r)] == ["native"] + assert len(nat) == 3 # comparators and served arms stay + assert [r["backend"] for r in AI.select_build(rows, "jvm")].count("arcadedb_sparse_embedded") == 1 + + +def test_one_build_needs_no_choice_and_a_bad_choice_is_an_error(monkeypatch): + rows = [row("native", rep=1), row("native", rep=2)] + assert AI.select_build(rows, None) == rows + monkeypatch.setenv(AI.ENV, "native") + assert AI.wanted_build() == "native" + monkeypatch.setenv(AI.ENV, "graal") + with pytest.raises(SystemExit): + AI.wanted_build() + + +def test_pooling_two_builds_in_one_entry_is_refused(): + with pytest.raises(SystemExit): + AI.refuse_pooled_builds([row("native"), row("jvm")], "sparse/tiny") + assert AI.refuse_pooled_builds([row("jvm"), row()], "x") + + +# ------------------------------------------------------------------ the canonical key +def test_the_freeze_key_and_the_merge_key_keep_both_builds_of_a_cell(): + """Under the old key the later build replaced the other on ts_utc; both keys now carry the build.""" + import merge_campaign as MC + rows = [row("native", ts="2026-11-01T00:00:00+00:00"), row("jvm", ts="2026-11-02T00:00:00+00:00")] + assert MC.key(rows[0]) != MC.key(rows[1]) + assert len(MC.canonicalise(rows)) == 2 + assert MC.key(row()) == MC.key(row("jvm")) # old rows still collapse with new JVM rows + import make_paper_tables as M + import inspect + assert "_AI.build_of(r)" in inspect.getsource(M.load_canonical) + assert MC.version_of(row("native")).endswith("native") and MC.version_of(row("jvm")) == V + + +# ------------------------------------------------------------------ tags and wheels +def test_the_build_comes_from_the_wheel_file_name(): + assert E.backend_from_wheel(f"/d/arcadedb-{V}-py3-none-manylinux_2_28_x86_64.whl") == "native" + assert E.backend_from_wheel(f"/d/arcadedb_jvm-{V}-py3-none-manylinux_2_28_x86_64.whl") == "jvm" + assert E.backend_from_wheel("/d/arcadedb_embedded-26.10.2-cp312-x-y.whl") == "" + assert E.pin_from_wheel(f"/d/arcadedb_jvm-{V}-py3-none-any.whl") == V + + +def test_native_and_jvm_images_of_one_release_have_two_tags(): + n, j = E.versioned_image(V, "native"), E.versioned_image(V, "jvm") + assert (n, j) == (f"dbbench:arcadedb-{V}-native", f"dbbench:arcadedb-{V}-jvm") + assert E.versioned_image("26.10.2") == "dbbench:arcadedb-26.10.2" # the old form, for the old wheel + for t in (n, j): + assert E.build_target(t) == "arcadedb" + assert E.build_target("dbbench:arcadedb-26.10.2") == "arcadedb" # old tag still resolves + + +def test_a_tag_must_be_the_pinned_build(): + n, j, old = (E.versioned_image(V, "native"), E.versioned_image(V, "jvm"), f"dbbench:arcadedb-{V}") + assert E.check_image_tag(n, V, "native") is None and E.check_image_tag(j, V, "jvm") is None + assert "jvm" in E.check_image_tag(j, V, "native") + assert "unsuffixed" in E.check_image_tag(old, V, "native") # an old image cannot stand in for a native wheel + assert E.check_image_tag(old, V) is None and E.check_image_tag(n, V) is None # no build named: old and new tags both read + assert E.check_image_tag("dbbench:arcadedb-26.10.2-native", V, "native") # wrong version still refused + + +def test_verify_reads_the_build_out_of_the_image(): + img = E.versioned_image(V, "native") + + def run(cmd, **kw): + # python probe output: version, distribution, jar names (the native package names its engine as one) + return types.SimpleNamespace(returncode=0, stdout=f"{V}\narcadedb-jvm\narcadedb-engine-{V.replace('.dev0', '-SNAPSHOT')}.jar\n", stderr="") + + _rb, problems = E.verify(V, [img], [], run=run, backend="native") + assert any("jvm build" in p and "native" in p for p in problems) # a JVM package inside a -native tag is caught + _rb, problems = E.verify(V, [E.versioned_image(V, "jvm")], [], run=run, backend="jvm") + assert problems == [] + + +def test_build_images_and_the_stage_template_tag_with_the_build(): + sh = (HERE / "build_images.sh").read_text() + assert '${_bk:+-$_bk}' in sh and "arcadedb_jvm) _bk=jvm" in sh + import make_2611_stages as G + assert 'WB=""' in G.ARCADE_PRE_TAIL and "arcadedb_jvm) WB=-jvm" in G.ARCADE_PRE_TAIL + assert "dbbench:arcadedb-$WV$WB" in G.ARCADE_POST_CHECKS and 'if [ "$WB" = "-native" ]' in G.ARCADE_POST_CHECKS + v = (HERE / "verify_pair_c25.sh").read_text() + assert "${WB:+-$WB}" in v and "NOT CHECKED (native build)" in v + for f in ("build_images.sh", "verify_pair_c25.sh"): + assert subprocess.run(["bash", "-n", str(HERE / f)]).returncode == 0 + + +# ------------------------------------------------------------------ native isolate arguments +def test_jvm_kwargs_become_isolate_arguments(): + kept, dropped = AIMP.native_isolate_args("4g", "-Darcadedb.txWalFlush=0 -Xms4g -XX:+UseG1GC -Darcadedb.queryMaxHeapRAM=2048") + assert kept == ["-Xmx4g", "-Darcadedb.txWalFlush=0", "-Xms4g", "-Darcadedb.queryMaxHeapRAM=2048"] and dropped == ["-XX:+UseG1GC"] + assert AIMP.native_isolate_args(None, None) == ([], []) + assert AIMP.native_isolate_args("1g", ["-Dx=1"]) == (["-Xmx1g", "-Dx=1"], []) + + +def test_apply_native_merges_and_lets_an_existing_setting_win(): + env = {} + assert AIMP.apply_native({"heap_size": "4g", "jvm_args": "-Darcadedb.txWalFlush=1"}, env) == "-Xmx4g -Darcadedb.txWalFlush=1" + env = {AIMP.NATIVE_ENV: "-Xmx2g"} + assert AIMP.apply_native({"heap_size": "4g", "jvm_args": "-Darcadedb.txWalFlush=0"}, env) == "-Xmx2g -Darcadedb.txWalFlush=0" + assert AIMP.apply_native(None, {}) == "" + + +def test_the_native_wrapper_sets_the_arguments_once_before_the_first_call(monkeypatch): + seen = [] + mod = types.SimpleNamespace(__backend__="native", create_database=lambda p, **kw: seen.append(("c", p)), + open_database=lambda p, **kw: seen.append(("o", p)), + DatabaseFactory=object) + monkeypatch.setattr(AIMP, "_loaded", mod) + monkeypatch.setattr(AIMP, "_isolate_created", False) + monkeypatch.delenv(AIMP.NATIVE_ENV, raising=False) + AIMP._wrap_native(mod) + mod.create_database("/a", jvm_kwargs={"heap_size": "3g", "jvm_args": None}) + mod.open_database("/a", jvm_kwargs={"heap_size": "9g", "jvm_args": None}) # a later call cannot change the isolate + import os + assert os.environ[AIMP.NATIVE_ENV] == "-Xmx3g" and seen == [("c", "/a"), ("o", "/a")] + assert AIMP.effective_native_args() == "-Xmx3g" + monkeypatch.delenv(AIMP.NATIVE_ENV, raising=False) + + +def test_the_jvm_build_is_not_wrapped(): + f = lambda *a, **k: None # noqa: E731 + mod = types.SimpleNamespace(__backend__="jvm", create_database=f, open_database=f) + AIMP._wrap_native(mod) + assert mod.create_database is f + + +# ------------------------------------------------------------------ durability asked in SQL +class FakeDb: + def __init__(self, value): + self.value = value + self.sql = [] + + def query(self, lang, q): + self.sql.append((lang, q)) + s = [{"key": "arcadedb.txWalFlush", "value": self.value, "overridden": False, "default": 0}, + {"key": "arcadedb.txWalGroupCommit", "value": False}] + return types.SimpleNamespace(first=lambda: types.SimpleNamespace(get=lambda k: s if k == "settings" else None)) + + +@pytest.mark.parametrize("value,want", [(0, "DURABILITY_ARCADEDB"), (1, "DURABILITY_ARCADEDB_STRICT"), (2, "DURABILITY_ARCADEDB_STRICT_FULL_SYNC")]) +def test_durability_is_read_from_the_engine_in_sql_without_jpype(monkeypatch, value, want): + """No jpype in sys.modules at all (the native build has none): the answer still comes from the engine, through the database.""" + import bench_common as B + monkeypatch.setitem(sys.modules, "jpype", None) # an import of jpype now raises ImportError + db = FakeDb(value) + assert B.arcade_durability_readback(db=db) == getattr(B, want) + assert db.sql == [("sql", "SELECT settings FROM schema:database")] + + +def test_a_snapshot_taken_while_the_database_was_open_answers_after_it_closed(monkeypatch): + import bench_common as B + monkeypatch.setitem(sys.modules, "jpype", None) + B.remember_engine_settings(FakeDb(1)) + try: + assert B.arcade_durability_readback() == B.DURABILITY_ARCADEDB_STRICT + finally: + B._ENGINE_SETTINGS.clear() + + +def test_with_no_database_and_no_jpype_the_string_says_it_is_asserted(monkeypatch): + import bench_common as B + monkeypatch.setitem(sys.modules, "jpype", None) + B._ENGINE_SETTINGS.clear() + assert "asserted" in B.arcade_durability_readback() + + +# ------------------------------------------------------------------ the consumers +def _vc_failures(tmp_path, names): + import json + import version_consistency_check as VC + payload = {"tables": [{"id": f"t{i}", "entries": [{"backend": "arcadedb_x", "version_name": n}]} for i, n in enumerate(names)]} + f = tmp_path / "p.json" + f.write_text(json.dumps(payload)) + return VC.check(str(f)) + + +def test_version_consistency_treats_native_and_jvm_spellings_of_one_release_as_one_version(tmp_path): + ok = _vc_failures(tmp_path, ["ArcadeDB 26.11.1 (native)", "ArcadeDB 26.11.1 (jvm)"]) + assert not [f for f in ok if "different version strings" in f or "spells one version" in f], ok + bad = _vc_failures(tmp_path, ["ArcadeDB 26.11.1 (native)", "ArcadeDB 26.11.2 (jvm)"]) + assert bad, "two releases must still fail" + + +def test_page_check_names_the_new_row_fields_as_conditions_not_columns(): + import re + import page_check as PC + for field in ("engine_backend", "native_isolate_args", "engine_load_ms"): + assert any(re.match(p, field) for p, _why in PC.NOT_PRINTED), field + + +def test_the_exporter_refuses_an_entry_that_pools_two_builds(): + pytest.importorskip("numpy") + pytest.importorskip("pandas") + import export_web as X + with pytest.raises(SystemExit) as e: + X._one_version([row("native"), row("jvm")], "sparse/tiny") + assert "different builds" in str(e.value) + assert X._one_version([row("jvm"), row("jvm", rep=2)], "x")[0]["engine_version"] == V if isinstance(X._one_version([row("jvm")], "x"), list) else True diff --git a/benchmarks/experiments/verify_pair_c25.sh b/benchmarks/experiments/verify_pair_c25.sh index 5d12639a9e6..0e379b61aff 100755 --- a/benchmarks/experiments/verify_pair_c25.sh +++ b/benchmarks/experiments/verify_pair_c25.sh @@ -32,7 +32,11 @@ docker image inspect "$IMG" >/dev/null 2>&1 || fail "no image $IMG" W=$(ls -t "$REPO"/bindings/python/dist/*.whl 2>/dev/null | head -1) [ -n "$W" ] || fail "no wheel in bindings/python/dist" -[ -n "$EMBEDDED_IMAGE" ] || EMBEDDED_IMAGE="dbbench:arcadedb-$(basename "$W" | cut -d- -f2)" +# The build is the tag's suffix: arcadedb--*.whl (native) -> dbbench:arcadedb--native, arcadedb_jvm--*.whl -> -jvm, the older arcadedb_embedded wheel -> none. +WV=$(basename "$W" | cut -d- -f2) +WB="" +case "$(basename "$W" | cut -d- -f1)" in arcadedb) WB=native ;; arcadedb_jvm) WB=jvm ;; esac +[ -n "$EMBEDDED_IMAGE" ] || EMBEDDED_IMAGE="dbbench:arcadedb-$(basename "$W" | cut -d- -f2)${WB:+-$WB}" # 1. the image's jars ISHA=$(docker run --rm --entrypoint sh "$IMG" -c 'cat /home/arcadedb/lib/arcadedb-engine-*.jar' \ @@ -43,7 +47,16 @@ p=z.read('com/arcadedb/arcadedb.properties').decode() print(next(l.split('=',1)[1].strip() for l in p.splitlines() if l.strip().startswith('buildNumber')))" 2>/dev/null) [ "$ISHA" = "$SHA" ] || fail "image jars are ${ISHA:0:9}, expected $SHORT" +# THE NATIVE BUILD HAS NO JARS, NO JVM AND NO JPYPE: checks 2, 3 and 5 are about those and are skipped for it, said out loud (NOT CHECKED), and check 4 reads +# the engine the image's native package compiled in (`_build_info.ENGINE_VERSION`) instead of a jar's buildNumber. The engine COMMIT of a native wheel is not +# readable from the package, so for it the pair rests on the image's jars (check 1) and the version read here and by engine_pin.verify at the first cell. +IJ=n/a; HAVE_J=n/a +if [ "$WB" = native ]; then + echo "NOT CHECKED (native build): wheel jars (2), JVM major (3), JPype pin (5); the wheel carries no jars, no JVM and no JPype" +fi + # 2. the wheel's jars +[ "$WB" = native ] || { WSHA=$(python3 -c " import sys, zipfile, io z=zipfile.ZipFile('$W') @@ -52,8 +65,10 @@ e=zipfile.ZipFile(io.BytesIO(z.read(n))) p=e.read('com/arcadedb/arcadedb.properties').decode() print(next(l.split('=',1)[1].strip() for l in p.splitlines() if l.strip().startswith('buildNumber')))" 2>/dev/null) [ "$WSHA" = "$SHA" ] || fail "wheel jars are ${WSHA:0:9}, expected $SHORT" +} # 3. THE JVMs. Same major on both sides or the deployment axis is not transport. +[ "$WB" = native ] || { IJ=$(docker run --rm --entrypoint sh "$IMG" -c 'java -version 2>&1' | head -1 | grep -oE '"[0-9]+' | tr -d '"') # The venv's JRE when this checkout has one (the bench host does); otherwise the JRE inside the # embedded image, which is the JVM the embedded arm actually runs (a worktree has no .venv). @@ -72,6 +87,7 @@ print(subprocess.run([str(d / 'jre' / 'bin' / 'java'), '-version'], capture_outp fi [ -n "$IJ" ] && [ -n "$EJ" ] || fail "could not read a JVM version (image='$IJ' embedded='$EJ')" [ "$IJ" = "$EJ" ] || fail "JVM major differs: image=$IJ embedded=$EJ" +} # 4. THE WHEEL THE EMBEDDED ARM ACTUALLY RUNS, which is baked into # dbbench:arcadedb at build time and is NOT the wheel file checked above. @@ -81,7 +97,10 @@ fi # served arm ran 26.9.1-SNAPSHOT, and the whole l4 table compared two # ArcadeDB versions with this script reporting PAIR VERIFIED. A wheel FILE on # disk is not the engine a cell ran; the image is. -if docker image inspect "$EMBEDDED_IMAGE" >/dev/null 2>&1; then +if [ "$WB" = native ] && docker image inspect "$EMBEDDED_IMAGE" >/dev/null 2>&1; then + NV=$(docker run --rm --entrypoint python3 "$EMBEDDED_IMAGE" -c "from arcadedb._build_info import ENGINE_VERSION as v; print(v)" 2>/dev/null | tr -dc '0-9a-zA-Z.-') + [ "${NV%.dev*}" = "${WV%.dev*}" ] || fail "$EMBEDDED_IMAGE's native engine is '$NV', the wheel is $WV (rebuild it with ARCADEDB_WHEEL set)" +elif docker image inspect "$EMBEDDED_IMAGE" >/dev/null 2>&1; then BSHA=$(docker run --rm --entrypoint python3 "$EMBEDDED_IMAGE" -c " import importlib.metadata as m, pathlib, zipfile, io, sys d = next(pathlib.Path(m.distribution(n).locate_file(p)) for n, p in (('arcadedb-jvm', 'arcadedb'), ('arcadedb', 'arcadedb'), ('arcadedb-embedded', 'arcadedb_embedded')) if any(m.distributions(name=n))) @@ -100,9 +119,11 @@ fi # is Dockerfile.bench's ARG default, read from this script's own directory like REPO above, never from # the environment. HERE="$(cd "$(dirname "$(readlink -f "${BASH_SOURCE[0]}")")" && pwd)" +[ "$WB" = native ] || { WANT_J=$(sed -n 's/^ARG JPYPE_VERSION=//p' "$HERE/Dockerfile.bench") [ -n "$WANT_J" ] || fail "no 'ARG JPYPE_VERSION=' default in $HERE/Dockerfile.bench" HAVE_J=$(docker run --rm --entrypoint python3 "$EMBEDDED_IMAGE" -c 'import jpype; print(jpype.__version__)' 2>/dev/null | tr -dc '0-9a-zA-Z.-') [ "$HAVE_J" = "$WANT_J" ] || fail "$EMBEDDED_IMAGE runs JPype '$HAVE_J', the pin is $WANT_J (rebuild it: ./build_images.sh arcadedb)" +} echo "PAIR VERIFIED $SHORT jars=image+wheel+embedded-image jvm=$IJ jpype=$HAVE_J image=$IMG embedded=$EMBEDDED_IMAGE repo=$REPO wheel=$W" diff --git a/benchmarks/experiments/version_consistency_check.py b/benchmarks/experiments/version_consistency_check.py index 65d1feb7faf..3ea9494b655 100644 --- a/benchmarks/experiments/version_consistency_check.py +++ b/benchmarks/experiments/version_consistency_check.py @@ -231,6 +231,8 @@ def check(path: str) -> list[str]: # this check exists to catch is two DIFFERENT builds of one engine # compared against each other, so it compares the part before the # first "+" and lets the tail differ. + # NATIVE AND JVM ARE ONE VERSION (26.11.1): a trailing "(native)" / "(jvm)" names the build of one release, not a second release. + values = {re.sub(r"\s*\((?:native|jvm)\)\s*$", "", v) for v in values} bases = {v.split("+")[0].strip() for v in values} if len(bases) == 1 and len(values) > 1: continue From 760931972ecb8588c77593c682fdb4214c29ecb8 Mon Sep 17 00:00:00 2001 From: Taewoon Kim Date: Fri, 30 Oct 2026 02:53:13 +0900 Subject: [PATCH 04/23] benchmarks: e4 durability through SQL, and the native engine commit verified from SELECT version() e4_decomp: the decomposition probe takes the in-process server database's settings in SQL while it is open, so the row's durability is asked of the engine on native too; the probe applies its heap to the native isolate instead of start_jvm() there. engine_pin.verify(commit=...) runs SELECT version() inside each embedded image and compares the "(build /...)" with the stage's engine commit (runner passes ARCADEDB_ENGINE_COMMIT); verify_pair_c25.sh check 4b does the same for a native wheel, so its commit is verified, not inferred. Co-Authored-By: Claude Sonnet 5.5 --- .../experiments/deployment_decomp_probe.py | 11 ++++- benchmarks/experiments/e4_decomp.py | 1 + benchmarks/experiments/engine_pin.py | 43 ++++++++++++++++- benchmarks/experiments/runner.py | 2 +- .../experiments/test_native_jvm_arms.py | 48 +++++++++++++++++++ benchmarks/experiments/verify_pair_c25.sh | 14 ++++-- 6 files changed, 112 insertions(+), 7 deletions(-) diff --git a/benchmarks/experiments/deployment_decomp_probe.py b/benchmarks/experiments/deployment_decomp_probe.py index 5f5de46185f..5c1a4e62040 100644 --- a/benchmarks/experiments/deployment_decomp_probe.py +++ b/benchmarks/experiments/deployment_decomp_probe.py @@ -355,8 +355,12 @@ def main() -> int: # has to be pinned on the JVM before the server exists. Same -Xms=-Xmx # policy as every other lane: fixed heap for latency parity, with the # working set reported separately from cgroup anon. - start_jvm = _arcadedb_import.jvm_module().start_jvm - start_jvm(heap_size=args.heap, jvm_args=[f"-Xms{args.heap}"]) + if _arcadedb_import.backend() == "native": + # no JVM to start: the same heap policy goes to the isolate before the first engine call (_arcadedb_import.apply_native) + _arcadedb_import.apply_native({"heap_size": args.heap, "jvm_args": [f"-Xms{args.heap}"]}) + else: + start_jvm = _arcadedb_import.jvm_module().start_jvm + start_jvm(heap_size=args.heap, jvm_args=[f"-Xms{args.heap}"]) # Arms 1 and 2 share this process, so they share the JVM, heap, GC, engine # instance and page cache. That sharing IS the control. @@ -366,6 +370,9 @@ def main() -> int: config={"host": "127.0.0.1", "http_port": 2489, "mode": "development"}, ) as server: db = server.create_database(DB_NAME) + # the engine's own durability setting, asked in SQL while the database is open (e4_decomp stamps it on the row) + import bench_common + bench_common.remember_engine_settings(db) load_embedded(db, ROWS) print(f"loaded {ROWS:,} rows into the server-managed database", flush=True) diff --git a/benchmarks/experiments/e4_decomp.py b/benchmarks/experiments/e4_decomp.py index 132a4eb2da3..ef4c02da86c 100644 --- a/benchmarks/experiments/e4_decomp.py +++ b/benchmarks/experiments/e4_decomp.py @@ -117,6 +117,7 @@ def main(): # of every gate, so the gate that would have caught this never saw them. # The lane runs one engine at its default class; the readback says which. import bench_common + # Asked in SQL (arcade_durability_readback, one path for both builds): the probe took the in-process server's database settings while it was open. payload["durability"] = bench_common.arcade_durability_readback() payload["durability_class"] = bench_common.DURABILITY_CLASS stamp_clients(payload) diff --git a/benchmarks/experiments/engine_pin.py b/benchmarks/experiments/engine_pin.py index 973c95d4b18..4562d0f9459 100644 --- a/benchmarks/experiments/engine_pin.py +++ b/benchmarks/experiments/engine_pin.py @@ -54,6 +54,18 @@ from importlib import import_module print('arcadedb-engine-' + import_module(os.path.basename(pkg) + '._build_info').ENGINE_VERSION + '.jar') """ +# THE ENGINE'S OWN BUILD STAMP, asked in SQL so it is one path for the native and the JVM package: `SELECT version() AS v` returns +# " (build //)" (checked on the 26.10.2 native wheel: full 40-character sha). The commit of a native wheel is +# not readable from its files (no jars), so this is how it is verified rather than inferred. +_BUILD_PROBE = """ +import importlib, importlib.util, os, tempfile +mod = importlib.import_module(next(n for n in ('arcadedb', 'arcadedb_embedded') if importlib.util.find_spec(n))) +d = tempfile.mkdtemp() +db = mod.create_database(os.path.join(d, 'p')) +print(db.query('sql', 'SELECT version() AS v').first().get('v')) +db.close() +""" +BUILD_RE = re.compile(r"\(build ([0-9a-f]{7,40})/") # The native distribution is `arcadedb`, the JVM one `arcadedb-jvm`; the old `arcadedb-embedded` is a JVM build. BACKEND_OF = {"arcadedb": "native", "arcadedb-jvm": "jvm", "arcadedb-embedded": "jvm"} @@ -149,6 +161,27 @@ def read_embedded(image, run=None): "engine_jar": _engine_jar_versions(lines[2].split())}, None +def build_sha(version_string): + """The commit in a `SELECT version()` answer ("26.10.2 (build //)"), or "" when it names none.""" + m = BUILD_RE.search(str(version_string or "")) + return m.group(1) if m else "" + + +def commit_matches(reported, pinned): + """Prefix comparison at the shorter length (the engine prints 40 characters, a stage may pin 9), never under 7.""" + n = min(len(reported or ""), len(pinned or ""), 40) + return n >= 7 and reported[:n].lower() == pinned[:n].lower() + + +def read_embedded_build(image, run=None): + """(sha, None) the engine inside a bench image reports for itself (a short-lived database, `SELECT version()`), or ("", why).""" + out, err = _docker(["--entrypoint", "python3", image, "-c", _BUILD_PROBE], run) + if err: + return "", err + sha = build_sha(out.strip().splitlines()[-1] if out.strip() else "") + return (sha, None) if sha else ("", f"the engine in {image} answered version() with {out.strip()[-120:]!r}, which names no build") + + def read_served(image, run=None): """({"engine_jar": [...]}, None): the engine jar(s) a server image loads, or (None, why).""" out, err = _docker(["--entrypoint", "sh", image, "-c", f"ls {SERVER_LIB}"], run) @@ -157,10 +190,11 @@ def read_served(image, run=None): return {"engine_jar": _engine_jar_versions(out.split())}, None -def verify(pin, embedded_images=(), served_images=(), run=None, backend=""): +def verify(pin, embedded_images=(), served_images=(), run=None, backend="", commit=""): """(readback, problems): what each image reports, and one sentence per way it is not the pin. Empty `problems` means every image runs `pin`. With `backend` ("native"/"jvm") an embedded image must also carry that build's distribution (arcadedb / arcadedb-jvm), read out of the image. + With `commit` (the stage's ARCADEDB_ENGINE_COMMIT) the engine inside each embedded image must also report that build commit, read in SQL. """ problems, readback = [], {} if not pin: @@ -174,6 +208,13 @@ def verify(pin, embedded_images=(), served_images=(), run=None, backend=""): continue if got["wheel"] != pin: problems.append(f"{image} has {got['dist']} {got['wheel']}, the stage pins {pin}") + if commit: + sha, berr = read_embedded_build(image, run) + if berr: + problems.append(f"{image}: {berr}") + elif not commit_matches(sha, commit): + problems.append(f"{image} runs engine commit {sha[:9]}, the stage pins {commit[:9]}") + readback[image]["engine_commit"] = sha if backend and got["backend"] != backend: problems.append(f"{image} carries {got['dist']} ({got['backend']} build), the stage pins the {backend} build") if got["engine_jar"] != [pin]: diff --git a/benchmarks/experiments/runner.py b/benchmarks/experiments/runner.py index fe55593e3a6..741da96e5d4 100644 --- a/benchmarks/experiments/runner.py +++ b/benchmarks/experiments/runner.py @@ -2274,7 +2274,7 @@ def _require_engine_pin(jobs): why = engine_pin.check_image_tag(img, _ENGINE_PIN, _ENGINE_BACKEND) if why: raise SystemExit(f"REFUSING: {why}.\n Rebuild with ARCADEDB_WHEEL= ./build_images.sh arcadedb.") - readback, problems = engine_pin.verify(_ENGINE_PIN, emb, srv, backend=_ENGINE_BACKEND) + readback, problems = engine_pin.verify(_ENGINE_PIN, emb, srv, backend=_ENGINE_BACKEND, commit=_LOCAL_ENGINE_COMMIT or "") if problems: raise SystemExit("REFUSING: an ArcadeDB arm would not run the stage's pinned version (engine_pin.py).\n " + "\n ".join(problems) diff --git a/benchmarks/experiments/test_native_jvm_arms.py b/benchmarks/experiments/test_native_jvm_arms.py index f3454155436..a565aaa29d6 100644 --- a/benchmarks/experiments/test_native_jvm_arms.py +++ b/benchmarks/experiments/test_native_jvm_arms.py @@ -249,3 +249,51 @@ def test_the_exporter_refuses_an_entry_that_pools_two_builds(): X._one_version([row("native"), row("jvm")], "sparse/tiny") assert "different builds" in str(e.value) assert X._one_version([row("jvm"), row("jvm", rep=2)], "x")[0]["engine_version"] == V if isinstance(X._one_version([row("jvm")], "x"), list) else True + + +# ------------------------------------------------------------------ the engine commit, asked in SQL +SHA = "e854692736b48971f92dff78a1e727bc570bb6a2" +ANSWER = f"26.10.2 (build {SHA}/1792085686249/main)" + + +def test_the_commit_is_read_out_of_select_version(): + assert E.build_sha(ANSWER) == SHA and E.build_sha("26.10.2") == "" and E.build_sha(None) == "" + assert E.commit_matches(SHA, SHA[:9]) and E.commit_matches(SHA[:9], SHA) and not E.commit_matches(SHA, "0" * 9) + assert not E.commit_matches(SHA[:4], SHA[:4]) # four characters name nothing + + +def _fake(answer, dist="arcadedb"): + def run(cmd, **kw): + if "SELECT version()" in " ".join(cmd): + return types.SimpleNamespace(returncode=0, stdout=answer + "\n", stderr="") + return types.SimpleNamespace(returncode=0, stdout=f"{V}\n{dist}\narcadedb-engine-{V.replace('.dev0', '-SNAPSHOT')}.jar\n", stderr="") + return run + + +def test_verify_checks_the_engine_commit_of_a_native_image(): + img = E.versioned_image(V, "native") + rb, problems = E.verify(V, [img], [], run=_fake(ANSWER.replace("26.10.2", V)), backend="native", commit=SHA) + assert problems == [] and rb[img]["engine_commit"] == SHA + _rb, problems = E.verify(V, [img], [], run=_fake(ANSWER.replace(SHA, "1" * 40)), backend="native", commit=SHA) + assert any("engine commit 111111111" in p for p in problems) + _rb, problems = E.verify(V, [img], [], run=_fake("26.11.1.dev0"), backend="native", commit=SHA) + assert any("names no build" in p for p in problems) + _rb, problems = E.verify(V, [img], [], run=_fake(ANSWER), backend="native") # no commit pinned: not asked + assert problems == [] + + +def test_the_pair_script_extracts_the_native_commit_with_the_same_answer_shape(): + """The sed in verify_pair_c25.sh run on a real answer.""" + text = (HERE / "verify_pair_c25.sh").read_text() + line = next(l for l in text.splitlines() if "sed -n 's/.*(build" in l) + sed_expr = line[line.index("sed -n"):].rstrip(")") + out = subprocess.run(["bash", "-c", f"printf '%s\\n' '{ANSWER}' | {sed_expr}"], capture_output=True, text=True) + assert out.stdout.strip() == SHA, (sed_expr, out) + assert "4b. THE COMMIT THE NATIVE ENGINE REPORTS" in text and "NOT CHECKED (native build)" in text + + +def test_e4_reads_durability_through_the_sql_snapshot_and_the_probe_starts_native_without_a_jvm(): + probe = (HERE / "deployment_decomp_probe.py").read_text() + assert "bench_common.remember_engine_settings(db)" in probe and 'backend() == "native"' in probe + e4 = (HERE / "e4_decomp.py").read_text() + assert "arcade_durability_readback()" in e4 and "jpype" not in e4.lower() diff --git a/benchmarks/experiments/verify_pair_c25.sh b/benchmarks/experiments/verify_pair_c25.sh index 0e379b61aff..78beaec7724 100755 --- a/benchmarks/experiments/verify_pair_c25.sh +++ b/benchmarks/experiments/verify_pair_c25.sh @@ -48,11 +48,11 @@ print(next(l.split('=',1)[1].strip() for l in p.splitlines() if l.strip().starts [ "$ISHA" = "$SHA" ] || fail "image jars are ${ISHA:0:9}, expected $SHORT" # THE NATIVE BUILD HAS NO JARS, NO JVM AND NO JPYPE: checks 2, 3 and 5 are about those and are skipped for it, said out loud (NOT CHECKED), and check 4 reads -# the engine the image's native package compiled in (`_build_info.ENGINE_VERSION`) instead of a jar's buildNumber. The engine COMMIT of a native wheel is not -# readable from the package, so for it the pair rests on the image's jars (check 1) and the version read here and by engine_pin.verify at the first cell. +# the engine the image's native package compiled in (`_build_info.ENGINE_VERSION`) instead of a jar's buildNumber. The engine COMMIT of a native wheel is +# not in a file, but the engine reports it: check 4b runs `SELECT version()` inside the embedded image (" (build /...)") and compares the sha with $SHA. IJ=n/a; HAVE_J=n/a if [ "$WB" = native ]; then - echo "NOT CHECKED (native build): wheel jars (2), JVM major (3), JPype pin (5); the wheel carries no jars, no JVM and no JPype" + echo "NOT CHECKED (native build): wheel jars (2), JVM major (3), JPype pin (5); the wheel carries no jars, no JVM and no JPype. Its engine commit IS checked (4b)" fi # 2. the wheel's jars @@ -100,6 +100,14 @@ fi if [ "$WB" = native ] && docker image inspect "$EMBEDDED_IMAGE" >/dev/null 2>&1; then NV=$(docker run --rm --entrypoint python3 "$EMBEDDED_IMAGE" -c "from arcadedb._build_info import ENGINE_VERSION as v; print(v)" 2>/dev/null | tr -dc '0-9a-zA-Z.-') [ "${NV%.dev*}" = "${WV%.dev*}" ] || fail "$EMBEDDED_IMAGE's native engine is '$NV', the wheel is $WV (rebuild it with ARCADEDB_WHEEL set)" + # 4b. THE COMMIT THE NATIVE ENGINE REPORTS, asked in SQL inside the image that will run the cells. + NSHA=$(docker run --rm --entrypoint python3 "$EMBEDDED_IMAGE" -c " +import os, tempfile, arcadedb +db = arcadedb.create_database(os.path.join(tempfile.mkdtemp(), 'p')) +print(db.query('sql', 'SELECT version() AS v').first().get('v')) +db.close()" 2>/dev/null | tail -1 | sed -n 's/.*(build \([0-9a-f]\{7,40\}\)\/.*/\1/p') + [ -n "$NSHA" ] || fail "$EMBEDDED_IMAGE's native engine did not answer SELECT version() with a build commit" + [ "${NSHA:0:9}" = "$SHORT" ] || fail "$EMBEDDED_IMAGE's native engine reports commit ${NSHA:0:9}, expected $SHORT" elif docker image inspect "$EMBEDDED_IMAGE" >/dev/null 2>&1; then BSHA=$(docker run --rm --entrypoint python3 "$EMBEDDED_IMAGE" -c " import importlib.metadata as m, pathlib, zipfile, io, sys From 29e09ced1a3dd2e847cd7c82456a942099cc1038 Mon Sep 17 00:00:00 2001 From: Taewoon Kim Date: Fri, 30 Oct 2026 10:21:12 +0900 Subject: [PATCH 05/23] benchmarks: the page shows ArcadeDB's native and JVM builds side by side as two arms The user decided (2026-10-30) that the 26.11.1 page shows both builds, where it printed one build per freeze. Rows already carry engine_backend; the page now keeps both and labels the build last: "ArcadeDB (embedded, native build)" next to "ArcadeDB (embedded, JVM build)". arm_identity: select_build no longer refuses a cell that holds both builds (BENCH_ENGINE_BACKEND=native|jvm still freezes one on purpose); splits_page (some row ran native) decides when labels name the build, so a freeze of old rows renders as before; build_of reads an empty engine_backend (an old row read back from the CSV) as the JVM build; build_label/strip_build; pair_problems. export_web: the generic table path, l4, lifecycle and durability group on the build (the entry keeps the runner's backend_key and gains `build`), _one_version still refuses an entry that pools two; the served arm stays one arm (the server is the same Java server either way, its rows carry no build); a build with no row where the other has one is declared "not measured" (one generated sentence per table and build); lifecycle names the start column "engine start ms" when it holds both builds and says what each is; native sorts before JVM; the exporter refuses a freeze whose builds name different engine_version or engine_commit. page_check: _check_build_pair (one commit, equal version_name, a row per build or a declared absence, no in-process arm without a build on a split page) and a per-build roster. PAGE-SPEC, READING-RESULTS and the generated-block preamble say so. Not yet build-aware: the dense and sparse multipass overlays (files, not frozen rows) carry no build; the gate refuses the dense embedded arms on a split page until the overlay says its build, and make_paper_tables' pinned-sparse path still stops on a two-build overlay. Tests: test_build_pair_page.py (26; every one fails on the pre-change code), test_native_jvm_arms.py updated for the no-refusal rule. Co-Authored-By: Claude Opus 5.5 --- benchmarks/experiments/PAGE-SPEC.md | 6 +- benchmarks/experiments/READING-RESULTS.md | 2 +- benchmarks/experiments/arm_identity.py | 92 ++++-- benchmarks/experiments/export_web.py | 132 +++++++-- benchmarks/experiments/make_paper_tables.py | 7 +- benchmarks/experiments/page_check.py | 89 +++++- benchmarks/experiments/refresh_web_page.py | 2 +- .../experiments/test_build_pair_page.py | 263 ++++++++++++++++++ .../experiments/test_native_jvm_arms.py | 7 +- 9 files changed, 543 insertions(+), 57 deletions(-) create mode 100644 benchmarks/experiments/test_build_pair_page.py diff --git a/benchmarks/experiments/PAGE-SPEC.md b/benchmarks/experiments/PAGE-SPEC.md index 1a6b3b19808..0c8a3b86768 100644 --- a/benchmarks/experiments/PAGE-SPEC.md +++ b/benchmarks/experiments/PAGE-SPEC.md @@ -10,7 +10,7 @@ The DBBench page at `humem.ai/research/dbbench` is the PRIMARY artifact: its lan 1. **Serial, full cpuset.** Every published latency, throughput, percentile, and memory cell runs one at a time on mini's `cpuset 0-11` (the 12 P-core THREADS on 6 physical P-cores; see 0a), `workers=1`, `tier=paper`. Parallel shards exist for sweeps and exploration only and may never reach the page. Enforced: `runner.py` refuses `workers != 1` at paper tier; `load_canonical` drops partial cpusets. 2. **N=5, median [min-max].** Five repetitions per cell, reached in two passes (PROTOCOL.md section 1). Any table with a cell below n=5 states its n in a condition, and no page sentence may assert an n that a cell it covers does not meet. Per-table operation and repetition counts are generated conditions (`export_web._counts_note`), never typed. -3. **One engine commit per table.** Every ArcadeDB row in one table comes from the same upstream commit, stamped as `engine_commit`. Comparators are digest-pinned images. See section 1. Enforced at production time: `runner.py` refuses a paper-tier ArcadeDB run when `ARCADEDB_ENGINE_COMMIT` is unset, before any cell starts, and `bench_common.run_conditions()` reads it so bespoke probes carry it too. A rule the page depends on may not be a printed reminder to a human. +3. **One engine commit per table.** Every ArcadeDB row in one table comes from the same upstream commit, stamped as `engine_commit`; the native and JVM builds of one release are two arms built from that one commit (section 1). Comparators are digest-pinned images. See section 1. Enforced at production time: `runner.py` refuses a paper-tier ArcadeDB run when `ARCADEDB_ENGINE_COMMIT` is unset, before any cell starts, and `bench_common.run_conditions()` reads it so bespoke probes carry it too. A rule the page depends on may not be a printed reminder to a human. 4. **One corpus per table.** Every row in a table describes the same dataset at the same size. A table whose rows disagree on `n_docs` (or the lane's size field) is a defect, not a comparison. The fingerprint must come from the corpus the lane loaded, not from a module constant (`l2_graph.py` and `l3_sparse.py` point it at the loaded data; BUGS.md F6 is the sparse lane running the synthetic corpus under a paper label). 5. **A knob the campaign sets must reach the cell, be honoured by the lane, and resolve inside the container.** `runner.py`'s env passthrough is a CLOSED tuple: a knob absent from it silently runs the in-script default. Three checks, each of which has failed once: DELIVERED (diff every lane's `environ.get("UPPER_CASE")` against what the runner passes), HONOURED (an unknown mode is a hard error, not a silent fallback to a literal default), and RESOLVED where the process runs (the mount is printed in the preamble and a missing corpus is refused on the host). Run the delivery diff whenever a lane gains a knob. 6. **The config PROFILE is recorded on every row.** ArcadeDB ships profiles that rewrite defaults wholesale: one sets `VECTOR_INDEX_GRAPH_BUILD_CACHE_SIZE=-1` with both cache heap percentages at 50, another pins both caches to a flat 10,000, against a default of `graphBuildCacheSize=0` (auto) and `graphBuildCacheMaxHeapPercent=25`. The profile can move a build several-fold on its own, so two rows under different profiles are not comparable. The percentage applies only on the AUTO path; quoting it for a run that pinned an absolute cache size is wrong. @@ -57,6 +57,8 @@ ArcadeDB is pinned by **upstream commit SHA**, not by a release number. The buil - Every row carries `engine_commit` (short SHA of the upstream commit the wheel and server image were both built from). - The page prints it as the engine identity, linked to the commit on GitHub. Every ArcadeDB row is named from its own engine string and commit, never from an image tag; served rows carry the row's `server_image_ref` (BUGS.md F16, F32). The page header lists the identities actually present. - Embedded and server arms in one table are built from the SAME commit, paired, and run the same JVM (DECISIONS #54). +- **Two builds, two arms (26.11.1).** ArcadeDB has two builds of one release: `arcadedb` (the native image, the default install) and `arcadedb-jvm` (JPype and a bundled JDK), both built from one engine commit. A page whose frozen rows hold the native build shows every in-process ArcadeDB arm twice, side by side, with the build last in the label: `ArcadeDB (embedded, native build)` and `ArcadeDB (embedded, JVM build)`, `ArcadeDB (embedded, fp32, native build)` and so on, in every table that has the arm (the lifecycle table puts the build after the situation: `Graph (embedded, native build)`). The native build comes first. The build is `engine_backend` on the row; a row without it, or with it empty, is the JVM build (`arm_identity.build_of`). The served ArcadeDB arm stays one arm: the server is the same Java server whichever package drives it, and its rows carry no build. A freeze of rows from before 26.11.1 has no native row, so it prints no build and renders as it always did. `BENCH_ENGINE_BACKEND=native|jvm` still freezes one build on purpose; the labels then still name it. +- **Both builds, one engine commit, or no page.** `export_web` refuses a freeze whose native and JVM rows name different `engine_version` or `engine_commit` values, and `page_check` fails the same page again (`_check_build_pair`). The gate also fails an in-process ArcadeDB arm that names no build on a page that names builds, two builds of one arm and size whose `version_name` differ, and a build that has a row at a size where the other has none. That last case is not left out: the exporter prints the missing build as a declared absence of kind `not measured` (a whole-row declaration with one generated sentence per table and build, naming the arms and sizes), which the roster and the pair gate read as accounted for. A table never takes a median over both builds: the exporter groups on the build and `_one_version` refuses an entry that spans two. Where the lifecycle table holds both builds its start column is `engine start ms`, the `start_jvm()` call on the JVM build and the library load with the isolate creation on the native build, and a sentence under the table says so. Not yet build-aware: the dense and sparse multipass overlays (files, not frozen rows) carry no build, so on a page that names builds the gate refuses their ArcadeDB embedded arms until the overlay says which build it measured. - A campaign FREEZES one commit and holds it start to finish. Upstream landing a fix mid-run does not restart the campaign; it becomes a dated changelog entry and a candidate for the next re-pin. - Re-pinning is a deliberate, dated act recorded in the changelog with what moved. No table falls back to an older pin, anywhere (DECISIONS #62). - **One ArcadeDB version until the final DBBench release, then history (DECISIONS #189).** Until the final DBBench page and repo are released (after the late-November 2026 paper), a new ArcadeDB release's pass REPLACES the old one: after one scripted old-against-new comparison (a regression is filed before the next cut), the old version's ArcadeDB rows, page data, pin-specific code, pinned test fixtures and wording are removed. Comparator rows are kept across it and carry their own engine's version, never ArcadeDB's. From the final release on, a new engine version ADDS rows and the page keeps the history: @@ -296,7 +298,7 @@ And a closing list of what the page does NOT measure: concurrency and load (it n ## Published tables (generated by refresh_web_page.py from results/web_benchmarks.json on every publish; do not edit by hand) -Every table carries a Size column and direction arrows; the best value per column within a size is bold on the page; ingest is a pair of columns where the lane records it. Rows: ArcadeDB first, comparators alphabetical, embedded before server, int8 before fp32. +Every table carries a Size column and direction arrows; the best value per column within a size is bold on the page; ingest is a pair of columns where the lane records it. Rows: ArcadeDB first (the native build, then its JVM build), comparators alphabetical, embedded before server, int8 before fp32. | id | title | rows | sizes | columns | |---|---|---|---|---| diff --git a/benchmarks/experiments/READING-RESULTS.md b/benchmarks/experiments/READING-RESULTS.md index 3c9ba14bc69..d2ddc3af37d 100644 --- a/benchmarks/experiments/READING-RESULTS.md +++ b/benchmarks/experiments/READING-RESULTS.md @@ -26,7 +26,7 @@ The single rule, from which most of the rest follows: **`engine_version` is deliberately null for non-ArcadeDB rows in L4.** Use `backend_version`. `run_conditions()` stamps `engine_version` from the installed wheel, which is right for the ArcadeDB row and wrong for DuckDB and QuestDB, so `l4_tsbs.py` asks each backend for its own version instead. A wrong version that passes a provenance audit is worse than none. -**`engine_backend` says which ArcadeDB build produced a row: `native` (the `arcadedb` wheel) or `jvm` (`arcadedb-jvm`, the older `arcadedb_embedded`).** `engine_version` is the same string for both wheels of one release, and `backend` is the arm name, so neither can tell them apart. It is read from `arcadedb.__backend__` by `bench_common.engine_backend()` and is null where the process never loaded ArcadeDB (every comparator, and a served ArcadeDB arm, whose engine runs in a container). +**`engine_backend` says which ArcadeDB build produced a row: `native` (the `arcadedb` wheel) or `jvm` (`arcadedb-jvm`, the older `arcadedb_embedded`).** `engine_version` is the same string for both wheels of one release, and `backend` is the arm name, so neither can tell them apart. It is read from `arcadedb.__backend__` by `bench_common.engine_backend()` and is null where the process never loaded ArcadeDB (every comparator, and a served ArcadeDB arm, whose engine runs in a container). The page shows both builds as two arms, `native build` and `JVM build`, one next to the other (PAGE-SPEC section 1); a row without the field is the JVM build. **Arms are selected by flag, not by filename.** `claims_check.ts_arm()` takes `primitive=` and `numpy_cols=` because the published row is one specific arm. Assert the flags rather than globbing the directory. diff --git a/benchmarks/experiments/arm_identity.py b/benchmarks/experiments/arm_identity.py index b63b13d2f31..df301099bfa 100644 --- a/benchmarks/experiments/arm_identity.py +++ b/benchmarks/experiments/arm_identity.py @@ -7,12 +7,15 @@ definition of `build`: * `engine_backend` is "native" or "jvm": that value. - * a row of an ArcadeDB in-process arm WITHOUT the field (every row before 26.11.1): "jvm", because that is what it ran. + * a row of an ArcadeDB in-process arm WITHOUT the field, or with it empty (every row before 26.11.1, and those rows read back from the + frozen CSV): "jvm", because that is what it ran. * a served arm, the HTTP client container and every comparator: None. The served engine is a JVM server whichever Python package drives it, and the field is null there on purpose, so it never selects or splits them. `select_build(rows, want)` is the one place a freeze picks a build: with BENCH_ENGINE_BACKEND=native|jvm the other build's in-process -ArcadeDB rows leave the freeze (they stay in runs.jsonl); without it, a cell that holds BOTH builds is refused by name, not pooled. +ArcadeDB rows leave the freeze (they stay in runs.jsonl); without it BOTH builds stay, and the page shows them side by side as two arms +(`splits_page`, `build_label`: "ArcadeDB (embedded, native build)" next to "ArcadeDB (embedded, JVM build)"). A table entry still never +pools the two (`refuse_pooled_builds`). `pair_problems(rows)` is the gate over a two-build freeze: one engine commit, one engine_version. No import of ArcadeDB, no I/O: safe in every consumer, including the ones that read old rows. """ import os @@ -38,7 +41,8 @@ def build_of(r): eb = r.get("engine_backend") if eb in BUILDS: return eb - if in_process(r) and "engine_backend" not in r: + # No usable field: an old row, or a row read back from the frozen CSV, where a row from before 26.11.1 has the column empty. + if in_process(r) and not eb: return "jvm" return None @@ -63,33 +67,81 @@ def wanted_build(env=None): return v -def mixed_cells(rows): - """{(lane, scale, n_docs, backend): builds} for the cells whose in-process ArcadeDB rows come from more than one build.""" - seen = {} - for r in rows: - if in_process(r) and build_of(r): - seen.setdefault((r.get("lane"), r.get("scale"), r.get("n_docs"), r.get("backend")), set()).add(build_of(r)) - return {k: sorted(v) for k, v in seen.items() if len(v) > 1} - - def select_build(rows, want=None): - """The rows a freeze may publish: one build of the in-process ArcadeDB arms, or a refusal that says why not. + """The rows a freeze may publish. `want` None reads BENCH_ENGINE_BACKEND. With a build, rows of the other build are dropped (served arms and comparators carry no - build and stay). Without one, any cell holding both builds stops the freeze: a median over native and JVM rows names neither. + build and stay): a deliberate one-build freeze. Without one every row stays, and the page shows the two builds as two arms + (`splits_page`); the exporter groups on the build, so no median is taken over native and JVM rows together. """ want = wanted_build() if want is None else want if want: return [r for r in rows if not (in_process(r) and build_of(r) not in (None, want))] - mixed = mixed_cells(rows) - if mixed: - sample = ", ".join("/".join(str(x) for x in k) for k in sorted(mixed, key=str)[:3]) - raise SystemExit( - f"REFUSING: {len(mixed)} cell(s) hold rows of both ArcadeDB builds (native and jvm), e.g. {sample}. A table cell is one " - f"build; set {ENV}=native or {ENV}=jvm to freeze that build (the other build's rows stay in the log).") return rows +def builds_in(rows): + """The builds the in-process ArcadeDB rows of `rows` ran on, as a set of "native" / "jvm".""" + return {build_of(r) for r in rows if in_process(r)} - {None} + + +def splits_page(rows): + """True when the page must name the build on every in-process ArcadeDB arm: some row ran on the native build. + + Rows from before the field are all "jvm", so a freeze of old rows is never split and renders exactly as it did. A freeze that holds + only native rows is split too: an unlabelled arm would read as the JVM build. + """ + return "native" in builds_in(rows) + + +BUILD_NAMES = {"native": "native build", "jvm": "JVM build"} + + +def build_label(label, build): + """"ArcadeDB (embedded)" -> "ArcadeDB (embedded, native build)"; a label with no parenthesis gets one.""" + name = BUILD_NAMES[build] + return f"{label[:-1]}, {name})" if label.endswith(")") else f"{label} ({name})" + + +def build_in_label(label): + """The build a label names ("native" / "jvm"), or None.""" + for b, name in BUILD_NAMES.items(): + if str(label).endswith(f", {name})") or str(label).endswith(f"({name})"): + return b + return None + + +def strip_build(label): + """The label without its build: "ArcadeDB (embedded, native build)" -> "ArcadeDB (embedded)"; a label with no build is returned as is.""" + b = build_in_label(label) + if not b: + return label + name = BUILD_NAMES[b] + label = str(label) + return label[:-len(f", {name})")] + ")" if label.endswith(f", {name})") else label[:-len(f" ({name})")] + + +def pair_problems(rows): + """What stops a two-build freeze from being published, as a list of sentences (empty: fine). + + The two builds of one release are built from ONE engine commit. A freeze whose native and JVM rows name different engine_versions + or different engine commits compares two engines, not two builds of one, so the page must not print them side by side. + """ + out = [] + by = {} + for r in rows: + if in_process(r) and build_of(r): + by.setdefault(build_of(r), []).append(r) + if len(by) < 2: + return out + for field in ("engine_version", "engine_commit"): + seen = {b: sorted({str(r.get(field) or "") for r in rs} - {""}) for b, rs in by.items()} + flat = {v for vs in seen.values() for v in vs} + if len(flat) > 1: + out.append(f"the two ArcadeDB builds name different {field} values {seen}; one release is built from one engine commit") + return out + + def refuse_pooled_builds(rs, where): """SystemExit when one entry's rows span two builds of ArcadeDB (the exporter's check, beside _one_version's).""" ids = identities(rs) diff --git a/benchmarks/experiments/export_web.py b/benchmarks/experiments/export_web.py index 465f4dac8d6..31591a9b6c6 100644 --- a/benchmarks/experiments/export_web.py +++ b/benchmarks/experiments/export_web.py @@ -1412,6 +1412,24 @@ def _agg(rows, field, across_scales=False): # # The headline query is the FIRST one in each list, so the p99 belongs to the query the section leads with. _FROZEN_ROWS = [] # set by main() once the CSV is read; the column switches read it +# TWO BUILDS, TWO ARMS (26.11.1). True when some in-process ArcadeDB row ran on the native build: every such arm then names its build and +# the two builds sit side by side (arm_identity.splits_page). False for a freeze of old rows, which render as they always did. +_SPLIT = False + + +def _arm_build(r): + """"native" or "jvm" for an in-process ArcadeDB row of a split page, else "" (served arms, comparators, an unsplit page).""" + return (_AI.build_of(r) or "") if (_SPLIT and _AI.in_process(r)) else "" + + +def _with_build(label, build): + """The arm label with its build ("ArcadeDB (embedded, native build)"), or the label as it is when the arm has no build.""" + return _AI.build_label(label, build) if build else label + + +def _strip_build(label): + """The label without its build suffix: the key the lane's own tables (order lists, deployments) are written in.""" + return _AI.strip_build(label) OCT_TABLE_METRICS = { # Both document tables come from this one lane spec and are split by # _restructure_tables; the union of their columns lives here. @@ -1795,6 +1813,43 @@ def _declare_absence(table_id, backend, column, kind, why): _DECLARED_ABSENCES[table_id].append(rec) +def _declare_missing_builds(tables): + """A build with no row where the other build has one is printed as not measured, never left out. + + On a split page (arm_identity.splits_page) every in-process ArcadeDB arm should stand twice, once per build. Where one build has + entries at a size and the other has none, the missing arm is declared (kind "not measured", whole row) and one sentence per table + and build names the arms and sizes. A deliberate one-build freeze (BENCH_ENGINE_BACKEND) declares nothing: the other build was + left out on purpose, and the arm labels still say which build the page is. + """ + if not _SPLIT or _AI.wanted_build(): + return + for t in tables: + have = collections.defaultdict(set) # (label without build, size) -> builds present + for e in t.get("entries", []): + if e.get("build") in _AI.BUILDS: + have[(_strip_build(str(e["backend"])), str(e.get("scale_label") or e.get("scale")))].add(e["build"]) + missing = collections.defaultdict(lambda: collections.defaultdict(list)) # build -> arm -> sizes + for (label, size), builds in sorted(have.items()): + for b in _AI.BUILDS: + if b not in builds: + missing[b][label].append(size) + for b in _AI.BUILDS: + arms = missing.get(b) + if not arms: + continue + parts, vals = [], [] + for label, sizes in sorted(arms.items()): + arm = _with_build(label, b) + _declare_absence(t["id"], arm, None, "not measured", + f"{arm} was not measured at {_join_and(sizes)}.") + parts.append(f"{arm} at {_join_and(sizes)}") + vals += [arm, *sizes] + t.setdefault("conditions", []) + note = _gen(f"The {_AI.BUILD_NAMES[b]} was not measured for {_join_and(parts)}, so those rows are absent.", *vals) + if note not in t["conditions"]: + t["conditions"].append(note) + + def _withhold_cells(tables): """Take out the cells whose answer was wrong, and say so on the table.""" notes = collections.defaultdict(list) @@ -2689,7 +2744,8 @@ def _l4_canonical(all_rows): # KEYED ON THE TIER TOO: grouped by display label alone, a table with # two tiers (ts100, ts1000) would publish every cell as a median across # BOTH corpora, a number belonging to neither (BUGS F110). - grouped[(label, str(r.get("scale")))].append(r) + # A split page keys the two builds apart (the label carries the build), so no l4 median pools them. + grouped[(_with_build(label, _arm_build(r)), str(r.get("scale")))].append(r) return grouped or None @@ -2933,22 +2989,29 @@ def _engine_of(r): # registered as an absence the coverage gate reads. declared.append(r) continue - by.setdefault((r.get("workload"), r.get("scale"), _srv, _engine_of(r)), []).append(r) + by.setdefault((r.get("workload"), r.get("scale"), _srv, _engine_of(r), _arm_build(r)), []).append(r) entries = [] # The engine's own start step: start_jvm() on the JVM build (`jvm_start_ms`), which the native build has no counterpart of (null); - # there the same step is `engine_load_ms` (library load and isolate creation). A page is one build (select_build), so one column. + # there the same step is `engine_load_ms` (library load and isolate creation). A page holding both builds names the column for both (below). _start_field, _start_label = (("engine_load_ms", "engine load ms") if any(_AI.build_of(r) == "native" for r in rows) else ("jvm_start_ms", "JVM start ms")) + # BOTH BUILDS ON ONE TABLE: the step is start_jvm() on the JVM build and library load plus isolate creation on the native one, so + # the column is named for what both are, and each arm reads its own field (the sentence under the table says which is which). + _both_builds = {"native", "jvm"} <= _AI.builds_in(rows) + if _both_builds: + _start_label = "engine start ms" _lc_columns = ([_start_label, "first open ms", "cold process ms"] + [LIFECYCLE_SCENARIO_LABELS[k] for k in LIFECYCLE_PAGE_SCENARIOS]) - for (situation, scale, _srv, engine), rs in sorted(by.items()): + for (situation, scale, _srv, engine, build), rs in sorted(by.items()): if situation in LIFECYCLE_WITHHELD: continue + _start_field = ("engine_load_ms" if build == "native" else "jvm_start_ms") if _both_builds else _start_field _ours = engine == "ArcadeDB" _mode = "server" if _srv else "embedded" entry = { - "backend": _lifecycle_row_label(rs[0].get("backend"), situation), + "backend": _with_build(_lifecycle_row_label(rs[0].get("backend"), situation), build), + "build": build or None, "backend_key": str(rs[0].get("backend")), "is_arcadedb": _ours, "engine": engine, @@ -2993,7 +3056,7 @@ def _engine_of(r): got = _agg(rs, field) if got is not None: entry["metrics"][label] = got - _trs = twin_by.get((situation, scale, _srv, engine)) + _trs = twin_by.get((situation, scale, _srv, engine)) if not build else None if _trs and entry["metrics"]: twin_metrics = {} for field, label in ((_start_field, _start_label), ("first_open_ms", "first open ms"), @@ -3047,6 +3110,7 @@ def _engine_of(r): "title": "Session cost, open to close", "dataset": "synthetic, one structure per row", "conditions": [_R("lifecycle", k) for k in ("session", "cold_start", "clean_close", "server_rows")] + + ([_R("lifecycle", "engine_start")] if _both_builds else []) + [_gen(f"{LIFECYCLE_SITUATION_LABELS.get(k, k)} is withheld: {v}", v) for k, v in sorted(LIFECYCLE_WITHHELD.items())] + declared_notes, "columns": _lc_columns, @@ -3240,10 +3304,10 @@ def _durability_table(all_rows): # out of this one for the same reason. if str(r.get("backend")) in OFF_PAGE_ARMS: continue - by.setdefault((str(r.get("backend")), str(r.get("scale")) if per_scale else None), + by.setdefault((str(r.get("backend")), str(r.get("scale")) if per_scale else None, _arm_build(r)), []).append(r) - for (backend, sc), rs in sorted(by.items(), key=lambda kv: (_durability_scale_rank(lane, kv[0][1]), - kv[0][0])): + for (backend, sc, build), rs in sorted(by.items(), key=lambda kv: (_durability_scale_rank(lane, kv[0][1]), + kv[0][0], kv[0][2])): if sc is not None: # ONE ROW PER CORPUS SIZE for an operation whose cost grows with # the corpus, named in the Size column beside the operation. @@ -3261,7 +3325,8 @@ def _durability_table(all_rows): no_setting = any(str(r.get("durability_no_setting")).lower() in ("true", "1") for r in rs) entry = { - "backend": display_name(backend), + "backend": _with_build(display_name(backend), build), + "build": build or None, "is_arcadedb": "arcadedb" in backend, "scale": op_key_s, "scale_label": op_label_s, @@ -3431,7 +3496,7 @@ def multimodel_engine(name): """The capability table's row for an engine name (MULTIMODEL_ALIASES).""" return MULTIMODEL_ALIASES.get(name, name) MULTIMODEL_CELLS = {"measured": "measured", "declared": "declared", "none": "no arm"} -MULTIMODEL_KINDS = {"censored": "censored", "withheld": "withheld", +MULTIMODEL_KINDS = {"censored": "censored", "withheld": "withheld", "not measured": "not measured", "unexpressible": "cannot express", "envelope": "out of memory", "failed": "failed", # A row a defect in the engine's own release decided @@ -3644,8 +3709,8 @@ def _l4_table(all_rows, twin_rows=()): # does rather than interleaving two corpora under one heading. _tier_rank = {"ts100": 0, "ts1000": 1} for key in sorted(grouped, key=lambda k: (_tier_rank.get(k[1], 99), - order.index(k[0]) if k[0] in order else 99, - k[0])): + order.index(_strip_build(k[0])) if _strip_build(k[0]) in order else 99, + _strip_build(k[0]), {"native": 0, "jvm": 1}.get(_AI.build_in_label(k[0]), 2))): label, _scale = key rs = grouped[key] entry = { @@ -3660,7 +3725,8 @@ def _l4_table(all_rows, twin_rows=()): "scale_label": f"{_l4_points(_scale)} points", "workload": L4_SHAPE["workload"], "n_docs": str(rs[0].get("n_points")), - "deployment": L4_DEPLOYMENT[label], + "deployment": L4_DEPLOYMENT[_strip_build(label)], + "build": _AI.build_in_label(label), "image": None, # One engine name plus one version. The two ArcadeDB arms recorded # this differently ("26.8.1" from one adapter, "arcadedb 26.8.1" @@ -4197,6 +4263,7 @@ def _milvus_seal_proportion(): }, "lifecycle": { "session": ("The SESSION is open + action + close. Reporting open and close alone hides work triggered between them.", []), + "engine_start": ("Engine start ms is the start_jvm() call on the JVM build, and the library load with the creation of the isolate on the native build; the two steps are the engine's own start, which is why they share a column.", []), "cold_start": ("Cold start is measured in a fresh subprocess and reported beside every session number, because a millisecond open inside a process that takes half a second to reach its first database call is not a millisecond to whoever launched it.", []), "clean_close": ("A clean close should be O(what was written), not O(what is stored): write nothing and closing should cost the same at 10k documents and 10M.", []), "server_rows": ("Server rows have no JVM start, first open, or cold process: the server is already running when the probe connects, so those three columns describe the embedded process only. The session columns are measured for both.", []), @@ -4594,8 +4661,14 @@ def _pg_memory_note(table): def _gav_pair_note(table): """Only when the table carries both arms of the ablation: the sentence must not describe a pair that is not there.""" names = {str(e["backend"]) for e in table.get("entries", [])} - pairs = [(a, a[:-1] + ", GAV)") for a in sorted(names) - if a.startswith("ArcadeDB (") and "GAV" not in a and (a[:-1] + ", GAV)") in names] + def _gav_twin(a): + # the view goes before the build: "ArcadeDB (embedded, native build)" -> "ArcadeDB (embedded, GAV, native build)" + b = _AI.build_in_label(a) + if not b: + return a[:-1] + ", GAV)" + return _strip_build(a)[:-1] + f", GAV, {_AI.BUILD_NAMES[b]})" + pairs = [(a, _gav_twin(a)) for a in sorted(names) + if a.startswith("ArcadeDB (") and "GAV" not in a and _gav_twin(a) in names] if not pairs: return None listed = "; ".join(f"{a} and {b}" for a, b in pairs) @@ -6835,7 +6908,9 @@ def key(e): base = re.sub(r"\s*\(.*$", "", str(e["backend"])).lower() return (scale_rank(e), 0 if e.get("is_arcadedb") else 1, base, DEPLOYMENT_ORDER.get(e.get("deployment"), 2), - PRECISION_ORDER.get(e.get("precision"), 2)) + PRECISION_ORDER.get(e.get("precision"), 2), + # the native build (the default install) first, its JVM twin right after it + {"native": 0, "jvm": 1}.get(e.get("build"), 2)) table["entries"] = sorted(entries, key=key) # THE HEAP BESIDE THE PEAK (DECISIONS #116 item 7, decided: the # memory column stays the peak an operator has to provision, with the heap @@ -7124,11 +7199,16 @@ def main() -> int: print(f" cap rule: {len(_cut)} rows of {len(_over)} over-the-cap arm-cells withheld, shown censored instead") _gone = {id(r) for r in _cut} rows = [r for r in rows if id(r) not in _gone] - # ONE ARCADEDB BUILD PER PAGE (26.11.1): native and jvm rows of a release share engine_version and arm names, so a table cell - # holding both would print one median over two engines. BENCH_ENGINE_BACKEND picks the build; both with no pick is refused. + # TWO ARCADEDB BUILDS, TWO ARMS (26.11.1): native and jvm rows of a release share engine_version and arm names, so the tables + # group on the build (_arm_build) and label it; no cell is a median over both. BENCH_ENGINE_BACKEND freezes one build on purpose. rows = _AI.select_build(rows) - global _FROZEN_ROWS + # ONE ENGINE COMMIT FOR BOTH BUILDS: a freeze whose native and JVM rows name two engines is refused, not printed side by side. + _bad = _AI.pair_problems(rows) + if _bad: + raise SystemExit("REFUSING: " + "; ".join(_bad)) + global _FROZEN_ROWS, _SPLIT _FROZEN_ROWS = rows + _SPLIT = _AI.splits_page(rows) # WITHHELD: ArcadeDB rows whose engine_version identifies no build. # # Not rendered, and said out loud rather than dropped quietly. A number we @@ -7200,11 +7280,11 @@ def main() -> int: # rather than a wrong number, and the l2olap table below is what would # have sprung it. grouped[(r["lane"], r["scale"], r["backend"], wl, - r.get("gav") or "")].append(r) + r.get("gav") or "", _arm_build(r))].append(r) twin_grouped = defaultdict(list) for r in _twin_rows: wl = "" if r["lane"] in MERGED_WORKLOAD_LANES else r.get("workload", "") - twin_grouped[(r["lane"], r["scale"], r["backend"], wl, r.get("gav") or "")].append(r) + twin_grouped[(r["lane"], r["scale"], r["backend"], wl, r.get("gav") or "", "")].append(r) tables = [] for lane, spec in LANES.items(): @@ -7213,7 +7293,7 @@ def main() -> int: # graph lane's rows, so the spec may name the lane it reads. src_lane = spec.get("lane_source", lane) only_scales = spec.get("only_scales") - for (ln, scale, backend, workload, gav), rs in sorted(grouped.items()): + for (ln, scale, backend, workload, gav, build), rs in sorted(grouped.items()): if ln != src_lane: continue if only_scales and scale not in only_scales: @@ -7221,7 +7301,7 @@ def main() -> int: # Row-level, not group-level: merged lanes (l1, l1tpc) group both # workloads under one key with a blank label, so a group-level # test never matched and the per-query OLAP table was empty (BUGS F30). - _trs = twin_grouped.get((ln, scale, backend, workload, gav), []) + _trs = twin_grouped.get((ln, scale, backend, workload, gav, ""), []) if spec.get("only_workload"): rs = [r for r in rs if r.get("workload") == spec["only_workload"]] if not rs: @@ -7289,8 +7369,11 @@ def main() -> int: if lane == "l2olap" and gav != "False" and "arcade" in backend: label = (f"{label[:-1]}, GAV)" if label.endswith(")") else f"{label} (GAV)") + # The build goes last, after the precision and the view, so the two builds of one arm differ only in that suffix. + label = _with_build(label, build) entry = { "backend": label, + "build": build or None, # The runner's own key for the arm, so a gate can hold the # table against runner.LANES without parsing the label # (page_check._check_lane_roster). @@ -7606,6 +7689,7 @@ def main() -> int: _t.setdefault("conditions", []) if _note not in _t["conditions"]: _t["conditions"].append(_note) + _declare_missing_builds(tables) # THE OVERRIDES A READER MEETS (CAMPAIGN section 7 row 21). PROTOCOL.md # section 7 lists every default this benchmark overrides, and the rows its # last column marked NOWHERE were named in no sentence under any table. One diff --git a/benchmarks/experiments/make_paper_tables.py b/benchmarks/experiments/make_paper_tables.py index 365a0abe9f4..f0c41e8c4d0 100644 --- a/benchmarks/experiments/make_paper_tables.py +++ b/benchmarks/experiments/make_paper_tables.py @@ -554,8 +554,8 @@ def load_canonical(apply_corpus=True, include_twins=False): # The paper tables, the gates and the claims read the headline rows. The twin rows go to the freeze (include_twins) so # the exporter can show them beside the headline, labelled, and are never pooled into a headline median. out = [r for r in best.values() if include_twins or r.get("arcadedb_http_client_role") != "sensitivity"] - # ONE BUILD PER FREEZE: BENCH_ENGINE_BACKEND=native|jvm picks the build whose in-process ArcadeDB rows the tables, the exporter - # and the gates see; a cell that holds both builds with no choice made is refused here, never pooled downstream. + # BOTH BUILDS BY DEFAULT: the freeze keeps the native and the JVM rows (the key above carries the build, so neither replaces the + # other) and the exporter shows them as two arms. BENCH_ENGINE_BACKEND=native|jvm freezes one build on purpose. return _AI.select_build(out) @@ -800,7 +800,8 @@ def _pinned_sparse_rows(arm="arcadedb_sparse_embedded"): # partial or mixed, and still refuses. if not out: return {} - # Both builds of one release at one pin: the freeze names the one it publishes (arm_identity.select_build), the other stays in the log. + # Both builds of one release at one pin: this overlay path holds ONE build (arm_identity.select_build when the freeze names one); + # it is not yet build-aware, so with both and no choice it still stops here. _builds = {_AI.build_of(r) for t in out for r in out[t]} if len(_builds) > 1: _want = _AI.wanted_build() diff --git a/benchmarks/experiments/page_check.py b/benchmarks/experiments/page_check.py index 76f5c5d9d86..5e4845395a9 100644 --- a/benchmarks/experiments/page_check.py +++ b/benchmarks/experiments/page_check.py @@ -35,6 +35,7 @@ sys.path.insert(0, str(HERE)) from publish_instrument import PUBLISH_INSTRUMENT, results_dir +import arm_identity as _AI DEFAULT_JSON = results_dir() / "web_benchmarks.json" @@ -541,6 +542,11 @@ def _check_instrument(payload): return bad +def _rows_for_pairs(args): + rows_path = Path(args.rows) if args.rows else _frozen_path() + return list(csv.DictReader(rows_path.open())) if rows_path.exists() else [] + + def main() -> int: ap = argparse.ArgumentParser() ap.add_argument("--json", default=str(DEFAULT_JSON)) @@ -592,6 +598,9 @@ def main() -> int: r_checked, r_bad = _check_lane_roster(payload) print(f"\n{r_checked} lane registration(s) held against the tables, " f"{r_bad} roster finding(s)") + print("\ntwo ArcadeDB builds: one engine commit, a row per build or a declared absence") + b_bad = _check_build_pair(payload, _rows_for_pairs(args)) + print(f" {b_bad} build-pair finding(s)") print("\nmulti-model coverage table, cell for cell against the tables it is derived from") m_checked, m_bad = _check_multimodel(payload) print(f"\n{m_checked} coverage cell(s) re-derived, {m_bad} disagree") @@ -630,7 +639,7 @@ def main() -> int: j_bad = _check_jvm_defaults_disclosure(payload) print(f" {j_bad} table(s) missing the sentence") return 1 if (i_bad or d_bad or a_bad or l_bad or c_bad - or r_bad or m_bad or not u_ok or k_bad or o_bad or n_bad or iv_bad or ov_bad or j_bad or l_notice_bad) else 0 + or r_bad or b_bad or m_bad or not u_ok or k_bad or o_bad or n_bad or iv_bad or ov_bad or j_bad or l_notice_bad) else 0 # -------------------------------------------------------------------------- @@ -784,7 +793,7 @@ def main() -> int: "condition of the cell read from the process (_arcadedb_import.effective_native_args), null on the JVM build, never a column"), (r"^engine_backend$", "which ArcadeDB build ran the cell, \"native\" or \"jvm\" (arcadedb.__backend__, bench_common.engine_backend): the identity of an arm beside " - "engine_version (arm_identity.py). export_web.select_build keeps one build per page and the cell's conditions name it, never a column"), + "engine_version (arm_identity.py). the page prints the build in the arm's label (export_web groups on it), never as a column"), (r"^jpype_version$", "which JPype the process that ran the cell had loaded (CAMPAIGN 7 row 73, bench_common.jpype_version): a condition of the embedded " "ArcadeDB arms, read from the process (`jpype.__version__`), empty for every arm that never imports it, never a column"), @@ -1320,6 +1329,67 @@ def _roster_arms(lane, RN, table): return registered, owed +def _page_split(payload): + """True when the payload shows the two ArcadeDB builds as two arms (an entry names its build).""" + return any(e.get("build") in _AI.BUILDS for t in payload.get("tables", []) for e in t.get("entries", [])) + + +def _declared_stripped(blabel, absent): + """A declaration names the arm by its label with the build, or by the label alone when it covers both builds.""" + return _AI.build_in_label(blabel) is not None and _AI.strip_build(blabel) in absent + + +def _check_build_pair(payload, rows): + """TWO BUILDS, ONE RELEASE: the native and JVM arms of a split page come from one engine commit, and neither goes missing silently. + + Fails when (1) an in-process ArcadeDB entry of a split page names no build, (2) the two builds of one arm and size carry different + version_name values (one release is built from one engine commit, so the commit-bearing name must be equal), (3) one build has a + row at a size and the other has neither a row nor a declared absence ("not measured") for it, or (4) the frozen rows of the two + builds name different engine_version or engine_commit values. A page of one build only (a deliberate BENCH_ENGINE_BACKEND freeze, + or rows from before 26.11.1) passes with nothing to compare. Returns the number of findings. + """ + bad = 0 + if not _page_split(payload): + print(" (no table names a build: nothing to pair)") + return 0 + for t in payload.get("tables", []): + arms = collections.defaultdict(dict) + absent = {str(a.get("backend")) for a in (t.get("declared_absences") or []) if not a.get("column")} + for e in t.get("entries", []): + if not (e.get("is_arcadedb") and _AI.in_process({"backend": e.get("backend_key") or ""})): + continue + b = e.get("build") + if b not in _AI.BUILDS: + print(f" NOBUILD {t['id']}: {e.get('backend')!r} is an in-process ArcadeDB arm on a page that names builds, and names none") + bad += 1 + continue + bare = _AI.strip_build(str(e["backend"])) + arms[(bare, str(e.get("scale")))][b] = e + for (bare, size), got in sorted(arms.items()): + if len(got) == 2: + names = {got[b].get("version_name") for b in _AI.BUILDS} + if len(names) > 1: + print(f" VERSION {t['id']}: {bare} at {size}: the builds name different engines {sorted(map(str, names))}") + bad += 1 + continue + (have,) = got + miss = [b for b in _AI.BUILDS if b != have][0] + mlabel = _AI.build_label(bare, miss) + if mlabel in absent or bare in absent or EW_family(bare) in absent: + print(f" declared {mlabel} on {t['id']} at {size}: not measured") + continue + print(f" ONEBUILD {t['id']}: {bare} at {size} has a {_AI.BUILD_NAMES[have]} row and no {_AI.BUILD_NAMES[miss]} row or declaration") + bad += 1 + for problem in _AI.pair_problems(rows): + print(f" COMMIT {problem}") + bad += 1 + return bad + + +def EW_family(label): + return re.sub(r"\s*\(.*$", "", str(label)).strip() + + def _check_lane_roster(payload): """A3, the roster: every backend runner.LANES registers for a lane is on each table the lane feeds, as a row or a declared absence. @@ -1374,6 +1444,21 @@ def _check_lane_roster(payload): # on the other workload's table: not a row, not a declared absence. if not RN.arm_runs(lane, _wl, key): continue + # TWO BUILDS, TWO ARMS: on a split page an in-process ArcadeDB arm is owed one row per build, and a build with none is a + # declared absence like any other ("not measured"), under the arm's label with the build. + if _page_split(payload) and _AI.in_process({"backend": key}) and key in keyed: + have = {e.get("build") for e in keyed[key]} + for b in _AI.BUILDS: + checked += 1 + if b in have: + continue + blabel = _AI.build_label(EW.display_name(key), b) + if blabel in absent or EW.engine_family(blabel) in absent or _declared_stripped(blabel, absent): + print(f" declared {blabel} [{key}] on {tid}: no row, absence declared") + continue + print(f" MISSING {blabel} [{key}] on {tid}: the other build has a row, this one has none and no declaration") + bad += 1 + continue checked += 1 label = EW.display_name(key) if key in keyed: diff --git a/benchmarks/experiments/refresh_web_page.py b/benchmarks/experiments/refresh_web_page.py index 0493786d3e7..7cb0be043e6 100644 --- a/benchmarks/experiments/refresh_web_page.py +++ b/benchmarks/experiments/refresh_web_page.py @@ -115,7 +115,7 @@ def size_key(label: str): lines = ["## Published tables (generated by refresh_web_page.py from results/web_benchmarks.json on every publish; do not edit by hand)", "", - "Every table carries a Size column and direction arrows; the best value per column within a size is bold on the page; ingest is a pair of columns where the lane records it. Rows: ArcadeDB first, comparators alphabetical, embedded before server, int8 before fp32.", + "Every table carries a Size column and direction arrows; the best value per column within a size is bold on the page; ingest is a pair of columns where the lane records it. Rows: ArcadeDB first (the native build, then its JVM build), comparators alphabetical, embedded before server, int8 before fp32.", "", "| id | title | rows | sizes | columns |", "|---|---|---|---|---|"] diff --git a/benchmarks/experiments/test_build_pair_page.py b/benchmarks/experiments/test_build_pair_page.py new file mode 100644 index 00000000000..f5afa9dde69 --- /dev/null +++ b/benchmarks/experiments/test_build_pair_page.py @@ -0,0 +1,263 @@ +"""The page shows ArcadeDB's native build and JVM build side by side, as two arms of one release. + +Run with `python -m pytest test_build_pair_page.py -q` from this directory (the exporter tests need numpy and pandas and stage the +publish_2611 fixture, like test_export_client_twin.py). The unit tests need neither. What each test proves is in its docstring. +""" +import csv +import gzip +import json +import os +import shutil +import subprocess +import sys +from pathlib import Path + +import pytest + +HERE = Path(__file__).resolve().parent +sys.path.insert(0, str(HERE)) + +import arm_identity as AI # noqa: E402 +import page_check as PC # noqa: E402 + +FIXTURE = HERE / "fixtures" / "publish_2611" / "results" +PIN = "261100fix" +V = "26.11.1.dev0" + + +def row(build="__absent__", backend="arcadedb_graph_embedded", **kw): + r = {"lane": "l2", "scale": "sf1", "workload": "oltp", "backend": backend, "engine_version": V, "engine_commit": "abc123def"} + if build != "__absent__": + r["engine_backend"] = build + r.update(kw) + return r + + +# ------------------------------------------------------------------ arm_identity: what splits a page, what labels it +def test_a_freeze_of_old_rows_is_not_split_and_a_native_row_splits_it(): + old = [row(), row(backend="arcadedb_graph_server"), row("", backend="arcadedb_graph_embedded"), row("jvm")] + assert not AI.splits_page(old) # empty field read back from the CSV is the JVM build + assert AI.builds_in(old) == {"jvm"} + assert AI.splits_page(old + [row("native")]) and AI.builds_in(old + [row("native")]) == {"native", "jvm"} + assert AI.splits_page([row("native")]) # native only: an unlabelled arm would read as the JVM build + assert not AI.splits_page([row("native", backend="arcadedb_graph_server"), row(backend="qdrant_dense")]) + + +def test_an_empty_engine_backend_from_the_csv_is_the_jvm_build_for_in_process_arms_only(): + assert AI.build_of(row("")) == "jvm" and AI.build_of(row(None)) == "jvm" + assert AI.build_of(row("", backend="arcadedb_graph_server")) is None + assert AI.build_of(row("", backend="qdrant_dense")) is None + + +def test_both_builds_stay_in_a_freeze_unless_one_is_chosen(): + rows = [row("native"), row("jvm"), row(None, backend="arcadedb_graph_server")] + assert AI.select_build(rows, None) == rows # the page shows both: nothing is refused or dropped + assert [AI.build_of(r) for r in AI.select_build(rows, "native") if AI.in_process(r)] == ["native"] + assert len(AI.select_build(rows, "jvm")) == 2 + + +@pytest.mark.parametrize("label", ["ArcadeDB (embedded)", "ArcadeDB (embedded, GAV)", "ArcadeDB (one transaction)", "ArcadeDB"]) +@pytest.mark.parametrize("build", ["native", "jvm"]) +def test_the_build_goes_last_in_the_label_and_comes_off_again(label, build): + got = AI.build_label(label, build) + assert got.endswith(f"{AI.BUILD_NAMES[build]})") and AI.build_in_label(got) == build and AI.strip_build(got) == label + assert AI.build_in_label(label) is None and AI.strip_build(label) == label + + +def test_the_two_labels_of_one_arm_differ_only_in_the_build(): + assert AI.build_label("ArcadeDB (embedded)", "native") == "ArcadeDB (embedded, native build)" + assert AI.build_label("ArcadeDB (embedded)", "jvm") == "ArcadeDB (embedded, JVM build)" + # "native" is also a storage path in the time-series lane; the build says "build" so the two never read alike + assert AI.build_label("ArcadeDB (embedded, native time series)", "native") == "ArcadeDB (embedded, native time series, native build)" + + +# ------------------------------------------------------------------ one engine commit +def test_two_builds_of_one_commit_pass_and_a_single_build_has_nothing_to_compare(): + assert AI.pair_problems([row("native"), row("jvm"), row(None, backend="arcadedb_graph_server")]) == [] + assert AI.pair_problems([row("native"), row("native", engine_commit="other")]) == [] + + +def test_two_builds_naming_different_engines_are_refused(): + bad_v = AI.pair_problems([row("native"), row("jvm", engine_version="26.11.2.dev0")]) + assert len(bad_v) == 1 and "engine_version" in bad_v[0] + bad_c = AI.pair_problems([row("native"), row("jvm", engine_commit="fff000999")]) + assert len(bad_c) == 1 and "engine_commit" in bad_c[0] + assert len(AI.pair_problems([row("native"), row("jvm", engine_version="x", engine_commit="y")])) == 2 + + +# ------------------------------------------------------------------ the gate over a payload +def entry(label, build, version="arcadedb 26.11.1 (abc123def)", key="arcadedb_graph_embedded", scale="sf1"): + return {"backend": label, "build": build, "backend_key": key, "is_arcadedb": True, "scale": scale, "version_name": version} + + +def payload(entries, absences=()): + return {"tables": [{"id": "l2", "entries": list(entries), "declared_absences": list(absences)}]} + + +NATIVE = entry("ArcadeDB (embedded, native build)", "native") +JVM = entry("ArcadeDB (embedded, JVM build)", "jvm") + + +def test_the_gate_passes_a_pair_of_one_release_and_a_page_with_no_builds(capsys): + assert PC._check_build_pair(payload([NATIVE, JVM]), [row("native"), row("jvm")]) == 0 + old = payload([{"backend": "ArcadeDB (embedded)", "build": None, "backend_key": "arcadedb_graph_embedded", "is_arcadedb": True}]) + assert PC._check_build_pair(old, [row()]) == 0 # an old freeze: nothing to pair + + +def test_the_gate_fails_when_the_builds_name_different_engines(capsys): + assert PC._check_build_pair(payload([NATIVE, dict(JVM, version_name="arcadedb 26.11.1 (fff000999)")]), []) == 1 + assert "VERSION" in capsys.readouterr().out + assert PC._check_build_pair(payload([NATIVE, JVM]), [row("native"), row("jvm", engine_commit="fff000999")]) == 1 + assert "COMMIT" in capsys.readouterr().out + + +def test_the_gate_fails_when_one_build_is_missing_without_a_declaration(capsys): + assert PC._check_build_pair(payload([NATIVE]), []) == 1 + assert "ONEBUILD" in capsys.readouterr().out + assert PC._check_build_pair(payload([JVM]), []) == 1 + + +def test_a_missing_build_that_the_page_declares_not_measured_passes(capsys): + absence = {"backend": "ArcadeDB (embedded, JVM build)", "column": None, "kind": "not measured", "why": "x"} + assert PC._check_build_pair(payload([NATIVE], [absence]), []) == 0 + assert PC._check_build_pair(payload([NATIVE], [dict(absence, backend="ArcadeDB (embedded)")]), []) == 0 # the bare arm covers both + other = dict(absence, backend="ArcadeDB (server)") + assert PC._check_build_pair(payload([NATIVE], [other]), []) == 1 # another arm's declaration does not + + +def test_the_gate_fails_an_in_process_arm_that_names_no_build_on_a_page_that_does(capsys): + assert PC._check_build_pair(payload([NATIVE, JVM, entry("ArcadeDB (embedded, fp32)", None)]), []) == 1 + assert "NOBUILD" in capsys.readouterr().out + served = entry("ArcadeDB (server)", None, key="arcadedb_graph_server") + assert PC._check_build_pair(payload([NATIVE, JVM, served]), []) == 0 # one served arm, no build + + +# ------------------------------------------------------------------ the exporter, on the publish fixture +def _stage(dest): + shutil.copytree(FIXTURE, dest) + with gzip.open(dest / "runs.jsonl.gz", "rt") as src, open(dest / "runs.jsonl", "w") as out: + shutil.copyfileobj(src, out) + (dest / "runs.jsonl.gz").unlink() + return dest + + +def _run(script, results): + env = {k: v for k, v in os.environ.items() if not k.startswith("BENCH_")} + env.update({"BENCH_RESULTS_DIR": str(results), "BENCH_ENGINE_COMMIT": PIN, "PYTHONHASHSEED": "0"}) + return subprocess.run([sys.executable, str(HERE / script)], cwd=HERE, env=env, capture_output=True, text=True, timeout=900) + + +def _with_native(results, keep=lambda r: True, commit=None): + """Rewrite the freeze: every in-process ArcadeDB row gains a native twin (for the rows `keep` accepts).""" + rows = list(csv.DictReader((results / "runs_paper.csv").open())) + fields = list(rows[0]) + (["engine_backend"] if "engine_backend" not in rows[0] else []) + twins = [] + for r in rows: + if AI.in_process(r) and keep(r): + t = dict(r, engine_backend="native") + if commit: + t["engine_commit"] = commit + twins.append(t) + assert twins + with (results / "runs_paper.csv").open("w", newline="") as fh: + w = csv.DictWriter(fh, fieldnames=fields, restval="") + w.writeheader() + w.writerows(rows + twins) + + +@pytest.fixture(scope="module") +def pages(tmp_path_factory): + """(payload of the old freeze, payload with native twins everywhere, payload with native twins in every lane but l2, the dir).""" + pytest.importorskip("numpy") + pytest.importorskip("pandas") + root = tmp_path_factory.mktemp("pair") + base = _stage(root / "base") + assert _run("make_paper_tables.py", base).returncode == 0 + out = [] + for name, keep in (("old", None), ("both", lambda r: True), ("partial", lambda r: r.get("lane") != "l2")): + d = root / name + shutil.copytree(base, d) + if keep: + _with_native(d, keep) + done = _run("export_web.py", d) + assert done.returncode == 0, done.stderr[-2000:] + out.append(json.loads((d / "web_benchmarks.json").read_text())) + return (*out, root) + + +def _t(p, tid): + return next(t for t in p["tables"] if t["id"] == tid) + + +def _inproc(t): + return [e for e in t["entries"] if e.get("is_arcadedb") and "server" not in e["backend"]] + + +def test_an_old_freeze_renders_exactly_as_before(pages): + old = pages[0] + for t in old["tables"]: + assert all(e.get("build") is None for e in t["entries"]), t["id"] + assert not any("build" in e["backend"] for e in t["entries"] if e.get("is_arcadedb")), t["id"] + assert not [a for a in t.get("declared_absences", []) if a["kind"] == "not measured"], t["id"] + + +def test_every_in_process_arm_of_a_lane_table_stands_twice_native_first(pages): + both = pages[1] + for tid in ("docs_oltp", "docs_olap", "l2", "e2", "l4"): + arms = [e["backend"] for e in _inproc(_t(both, tid))] + assert arms and all(AI.build_in_label(a) for a in arms), (tid, arms) + bare = [AI.strip_build(a) for a in arms] + for b in set(bare): + i = [k for k, x in enumerate(bare) if x == b] + labels = [arms[k] for k in i] + assert any(AI.build_in_label(x) == "native" for x in labels) and any(AI.build_in_label(x) == "jvm" for x in labels) + assert AI.build_in_label(arms[i[0]]) == "native" # the default install first + assert [e["backend"] for e in _t(both, "l2")["entries"] if e.get("is_arcadedb")][:2] == [ + "ArcadeDB (embedded, native build)", "ArcadeDB (embedded, JVM build)"] + + +def test_the_served_arm_stays_one_row_and_both_builds_name_one_engine(pages): + both = pages[1] + for tid in ("docs_oltp", "l2"): + t = _t(both, tid) + served = [e for e in t["entries"] if e.get("is_arcadedb") and e.get("deployment") == "server"] + assert served and all(e["build"] is None and "build" not in e["backend"] for e in served) + names = {e["version_name"] for e in _inproc(t)} + assert len(names) == 1, names + rows = list(csv.DictReader((pages[3] / "both" / "runs_paper.csv").open())) + assert PC._check_build_pair({"tables": [t for t in both["tables"] if t["id"] != "l3d"]}, rows) == 0 + + +def test_the_dense_overlay_arms_name_no_build_and_the_gate_says_so(pages): + """OPEN: the dense multipass overlay (files, not frozen rows) carries no engine_backend, so its fp32 and int8 arms print without a + build. On a page that names builds the gate refuses that, which is the loud behaviour wanted until the overlay says its build.""" + both = pages[1] + assert PC._check_build_pair({"tables": [_t(both, "l3d")]}, []) > 0 + + +def test_neither_build_is_a_median_over_both(pages): + old, both = pages[0], pages[1] + one = {(e["scale"], e["metrics"]["new-order p50 ms"]["n"]) for e in _inproc(_t(old, "docs_oltp")) if "new-order p50 ms" in e["metrics"]} + two = {(e["scale"], e["metrics"]["new-order p50 ms"]["n"]) for e in _inproc(_t(both, "docs_oltp")) if "new-order p50 ms" in e["metrics"]} + assert one and two == one # twins of the same rows: each build keeps the old n, none doubles it + + +def test_a_lane_with_one_build_prints_the_other_as_not_measured_and_the_gate_reads_it(pages): + part = pages[2] + t = _t(part, "l2") + assert [AI.build_in_label(e["backend"]) for e in _inproc(t)] == ["jvm"] * len(_inproc(t)) + absent = [a for a in t["declared_absences"] if a["kind"] == "not measured"] + assert absent and all("native build" in a["backend"] for a in absent) + assert any("native build was not measured for" in c for c in t["conditions"]) + assert not [a for a in _t(part, "docs_oltp").get("declared_absences", []) if a["kind"] == "not measured"] + assert PC._check_build_pair({"tables": [t]}, []) == 0 + stripped = dict(t, declared_absences=[a for a in t["declared_absences"] if a["kind"] != "not measured"]) + assert PC._check_build_pair({"tables": [_t(part, "docs_oltp"), stripped]}, []) > 0 # undeclared, the page is refused + + +def test_a_freeze_whose_builds_name_two_commits_is_not_exported(pages): + d = pages[3] / "mixed" + shutil.copytree(pages[3] / "base", d) + _with_native(d, commit="fffffffff") + done = _run("export_web.py", d) + assert done.returncode != 0 and "engine_commit" in (done.stderr + done.stdout) diff --git a/benchmarks/experiments/test_native_jvm_arms.py b/benchmarks/experiments/test_native_jvm_arms.py index a565aaa29d6..f827572541c 100644 --- a/benchmarks/experiments/test_native_jvm_arms.py +++ b/benchmarks/experiments/test_native_jvm_arms.py @@ -45,11 +45,10 @@ def test_the_same_release_on_two_builds_is_two_identities(): assert AI.arm_identity(row()) == AI.arm_identity(row("jvm")) # an old row and a new JVM row are one arm -def test_a_cell_with_both_builds_is_refused_unless_one_is_chosen(): +def test_both_builds_stay_unless_one_is_chosen(): + """Both builds are two arms of one page (test_build_pair_page.py), so a freeze no longer refuses a cell that holds both.""" rows = [row("native"), row("jvm"), row("native", backend="qdrant_sparse"), row(None, backend="arcadedb_sparse_server")] - with pytest.raises(SystemExit) as e: - AI.select_build(rows, None) - assert "BENCH_ENGINE_BACKEND" in str(e.value) + assert AI.select_build(rows, None) == rows nat = AI.select_build(rows, "native") assert [AI.build_of(r) for r in nat if AI.in_process(r)] == ["native"] assert len(nat) == 3 # comparators and served arms stay From 4ec4f929a48066f05484b20ef25bfe47ca1c2aa8 Mon Sep 17 00:00:00 2001 From: Taewoon Kim Date: Fri, 30 Oct 2026 10:31:01 +0900 Subject: [PATCH 06/23] benchmarks: overlays, censored cells and notes per build; served ArcadeDB cells in the JVM chain only Follow-up to the two-builds page (coordinator decisions, 2026-10-30). 1. Overlays belong to the build that produced them. The multipass drivers already stamp engine_backend on every pass (run_conditions); the native chain now names its files mp__native_b.json / sp__native_.json beside the JVM ones (make_2611_stages overlay_env), and export_web reads the build from the file content (an unstamped file is the JVM build, like an old row), so the dense and sparse second-pass arms stand once per build. Chosen: an unstamped overlay is resolved by that default, so the l3d NOBUILD failure is gone for stamped and unstamped overlays alike; a native build with no overlay is a declared "not measured". make_paper_tables refuses a partial native overlay, labels both builds apart in its stats, and validates each build in the pinned-sparse path (the paper table is one build: the JVM one unless BENCH_ENGINE_BACKEND says otherwise). 2. Censored cells are keyed arm + build + size ("backend@build" on the backend element, split again by the two consumers and the one-row-per-cell rule), so a native OOM or cap never removes the JVM entry. A killed cell's row comes from the runner, not the engine process, so runner.py stamps engine_backend from the pinned wheel (arm_identity.stamp_build); otherwise it would read as the JVM build. 3. unchanged: a declared "not measured" absence satisfies the gate. 4. The server is the ARCADEDB_SERVER_IMAGE container, the same whichever wheel is installed: make_2611_stages skips the served ArcadeDB arms when the wheel is the native build (served_arcadedb_arms, added to the existing skip set, so the exactly-once proof still holds; the dense profile check expects the embedded arm only), lint fails a native draft with a served arm, and --also-drafts DIR makes --check fail when a served arm is in both chains. stage_drafts_2611.sha256 is unchanged (its chains use the arcadedb_embedded wheel). 5. The heap and memory notes read the frozen rows behind each build's entry (_entry_rows, _entry_heaps, _campaign_stat take the build). Tests: test_build_pair_page.py (new asserts for 1, 2, 5), test_chain_served_arms.py (4); 15 of them fail on the previous code. Co-Authored-By: Claude Opus 5.5 --- benchmarks/experiments/PAGE-SPEC.md | 2 +- benchmarks/experiments/arm_identity.py | 12 ++ benchmarks/experiments/export_web.py | 68 +++++-- benchmarks/experiments/make_2611_stages.py | 62 ++++++- benchmarks/experiments/make_paper_tables.py | 39 +++- benchmarks/experiments/runner.py | 2 + .../experiments/test_build_pair_page.py | 173 +++++++++++++++++- .../experiments/test_chain_served_arms.py | 90 +++++++++ 8 files changed, 415 insertions(+), 33 deletions(-) create mode 100644 benchmarks/experiments/test_chain_served_arms.py diff --git a/benchmarks/experiments/PAGE-SPEC.md b/benchmarks/experiments/PAGE-SPEC.md index 0c8a3b86768..0a23a041957 100644 --- a/benchmarks/experiments/PAGE-SPEC.md +++ b/benchmarks/experiments/PAGE-SPEC.md @@ -58,7 +58,7 @@ ArcadeDB is pinned by **upstream commit SHA**, not by a release number. The buil - The page prints it as the engine identity, linked to the commit on GitHub. Every ArcadeDB row is named from its own engine string and commit, never from an image tag; served rows carry the row's `server_image_ref` (BUGS.md F16, F32). The page header lists the identities actually present. - Embedded and server arms in one table are built from the SAME commit, paired, and run the same JVM (DECISIONS #54). - **Two builds, two arms (26.11.1).** ArcadeDB has two builds of one release: `arcadedb` (the native image, the default install) and `arcadedb-jvm` (JPype and a bundled JDK), both built from one engine commit. A page whose frozen rows hold the native build shows every in-process ArcadeDB arm twice, side by side, with the build last in the label: `ArcadeDB (embedded, native build)` and `ArcadeDB (embedded, JVM build)`, `ArcadeDB (embedded, fp32, native build)` and so on, in every table that has the arm (the lifecycle table puts the build after the situation: `Graph (embedded, native build)`). The native build comes first. The build is `engine_backend` on the row; a row without it, or with it empty, is the JVM build (`arm_identity.build_of`). The served ArcadeDB arm stays one arm: the server is the same Java server whichever package drives it, and its rows carry no build. A freeze of rows from before 26.11.1 has no native row, so it prints no build and renders as it always did. `BENCH_ENGINE_BACKEND=native|jvm` still freezes one build on purpose; the labels then still name it. -- **Both builds, one engine commit, or no page.** `export_web` refuses a freeze whose native and JVM rows name different `engine_version` or `engine_commit` values, and `page_check` fails the same page again (`_check_build_pair`). The gate also fails an in-process ArcadeDB arm that names no build on a page that names builds, two builds of one arm and size whose `version_name` differ, and a build that has a row at a size where the other has none. That last case is not left out: the exporter prints the missing build as a declared absence of kind `not measured` (a whole-row declaration with one generated sentence per table and build, naming the arms and sizes), which the roster and the pair gate read as accounted for. A table never takes a median over both builds: the exporter groups on the build and `_one_version` refuses an entry that spans two. Where the lifecycle table holds both builds its start column is `engine start ms`, the `start_jvm()` call on the JVM build and the library load with the isolate creation on the native build, and a sentence under the table says so. Not yet build-aware: the dense and sparse multipass overlays (files, not frozen rows) carry no build, so on a page that names builds the gate refuses their ArcadeDB embedded arms until the overlay says which build it measured. +- **Both builds, one engine commit, or no page.** `export_web` refuses a freeze whose native and JVM rows name different `engine_version` or `engine_commit` values, and `page_check` fails the same page again (`_check_build_pair`). The gate also fails an in-process ArcadeDB arm that names no build on a page that names builds, two builds of one arm and size whose `version_name` differ, and a build that has a row at a size where the other has none. That last case is not left out: the exporter prints the missing build as a declared absence of kind `not measured` (a whole-row declaration with one generated sentence per table and build, naming the arms and sizes), which the roster and the pair gate read as accounted for. A table never takes a median over both builds: the exporter groups on the build and `_one_version` refuses an entry that spans two. Where the lifecycle table holds both builds its start column is `engine start ms`, the `start_jvm()` call on the JVM build and the library load with the isolate creation on the native build, and a sentence under the table says so. **Overlays belong to the build that produced them.** The dense and sparse multipass drivers stamp `engine_backend` on every pass they write (`run_conditions`), the native chain names its files `mp__native_b.json` and `sp__native_.json` beside the JVM build's, and the exporter reads the build from the file's own content, so the in-process dense and sparse arms stand once per build like any other arm. An overlay file without the field is the JVM build, as an old row is; a page with native rows and no native overlay therefore prints the JVM arm and declares the native one `not measured`. A partial native overlay (not every build of every in-process arm) refuses the freeze (`make_paper_tables._refuse_partial_native`). The paper's pinned-sparse path validates each build on its own and, with no `BENCH_ENGINE_BACKEND`, takes the JVM build. **Censored cells and the notes read per build.** A cap, timeout or out-of-memory outcome is keyed arm, build and size, so a native outcome never removes the JVM entry; the runner stamps `engine_backend` on the row of a cell it killed, because that row comes from the runner and not from the engine process, and an old outcome without the field is the JVM build. The heap and memory notes read the frozen rows behind each build's entry. **The server runs in one chain.** The served ArcadeDB arm is the `ARCADEDB_SERVER_IMAGE` container, the same Java server whichever wheel is installed, so the JVM chain runs the served cells and the native chain skips them (`make_2611_stages.served_arcadedb_arms`); `--check --drafts DIR --also-drafts DIR2` fails when a served ArcadeDB arm is in both chains, and the lint fails a native draft that runs one. - A campaign FREEZES one commit and holds it start to finish. Upstream landing a fix mid-run does not restart the campaign; it becomes a dated changelog entry and a candidate for the next re-pin. - Re-pinning is a deliberate, dated act recorded in the changelog with what moved. No table falls back to an older pin, anywhere (DECISIONS #62). - **One ArcadeDB version until the final DBBench release, then history (DECISIONS #189).** Until the final DBBench page and repo are released (after the late-November 2026 paper), a new ArcadeDB release's pass REPLACES the old one: after one scripted old-against-new comparison (a regression is filed before the next cut), the old version's ArcadeDB rows, page data, pin-specific code, pinned test fixtures and wording are removed. Comparator rows are kept across it and carry their own engine's version, never ArcadeDB's. From the final release on, a new engine version ADDS rows and the page keeps the history: diff --git a/benchmarks/experiments/arm_identity.py b/benchmarks/experiments/arm_identity.py index df301099bfa..db59a90d081 100644 --- a/benchmarks/experiments/arm_identity.py +++ b/benchmarks/experiments/arm_identity.py @@ -80,6 +80,18 @@ def select_build(rows, want=None): return rows +def stamp_build(row, wheel_build): + """Give an in-process ArcadeDB row the build of the wheel its runner was pinned to, when the row names none. + + A cell killed by its cap or by the kernel writes a row from the runner, not from the engine process, so the row has no + engine_backend of its own; without this a native timeout would read as the JVM build (the default for an unstamped row). + Returns the row. + """ + if wheel_build in BUILDS and in_process(row) and not row.get("engine_backend"): + row["engine_backend"] = wheel_build + return row + + def builds_in(rows): """The builds the in-process ArcadeDB rows of `rows` ran on, as a set of "native" / "jvm".""" return {build_of(r) for r in rows if in_process(r)} - {None} diff --git a/benchmarks/experiments/export_web.py b/benchmarks/experiments/export_web.py index 31591a9b6c6..4f9bd7932a4 100644 --- a/benchmarks/experiments/export_web.py +++ b/benchmarks/experiments/export_web.py @@ -1090,6 +1090,24 @@ def _num(value): ] +def _overlay_files_by_build(root, stem, tail, backend_key): + """[(build, files)] for one overlay arm: "" for a comparator or a served arm and for a page that names no build, else "native" and + "jvm" read from the first pass of each file (arm_identity.build_of; an unstamped ArcadeDB file is the JVM build). The native + files carry `_native` after the stem; a glob on the plain stem never matches them.""" + names = sorted(root.glob(f"{stem}{tail}")) + sorted(root.glob(f"{stem}_native{tail}")) + if not (_SPLIT and _AI.in_process({"backend": backend_key})): + return [("", sorted(root.glob(f"{stem}{tail}")))] if names else [] + got = {} + for fp in names: + try: + passes = json.loads(fp.read_text(encoding="utf-8")) + except ValueError: + continue + if passes: + got.setdefault(_AI.build_of(dict(passes[0], backend=backend_key)), []).append(fp) + return [(b, got[b]) for b in ("native", "jvm") if b in got] + + def _dense_overlay_entries(scale="deep10m"): """The DEEP-10M tier, every engine, both passes. @@ -1127,10 +1145,14 @@ def _dense_overlay_entries(scale="deep10m"): if not root.is_dir(): return [] out = [] + # THE OVERLAY BELONGS TO THE BUILD THAT PRODUCED IT. The driver stamps engine_backend on every pass it writes (run_conditions); a + # native run's files are mp__native_b.json beside the JVM build's mp__b.json, and the build is read from the + # file's own content. A file without the field is the JVM build, like an old row (arm_identity.build_of). + _units = [] for arm, backend_key, label, ours in arms: - hits = sorted(root.glob(f"mp_{arm}_b*.json")) - if not hits: - continue + for _ob, hits in _overlay_files_by_build(root, f"mp_{arm}", "_b*.json", backend_key): + _units.append((arm, backend_key, _with_build(label, _ob), ours, hits, _ob)) + for arm, backend_key, label, ours, hits, _ob in _units: cold, warm, recall, build, peak, ver = [], [], [], [], [], set() for h in hits: passes = json.loads(h.read_text(encoding="utf-8")) @@ -1203,10 +1225,10 @@ def _dense_overlay_entries(scale="deep10m"): # served arm's multipass file sees only the client container (3.6 GiB # against the pair's 28), and no file carries disk. Same build, same # envelope; the overlay's own peak is the fallback. - _pk = _campaign_stat(_cb, scale, MEM_PEAK_FIELD) or _agg(peak, MEM_PEAK_FIELD) + _pk = _campaign_stat(_cb, scale, MEM_PEAK_FIELD, build=_ob) or _agg(peak, MEM_PEAK_FIELD) if _pk is not None: metrics["peak memory GiB"] = _pk - _dk = _campaign_stat(_cb, scale, "disk_data_mb") + _dk = _campaign_stat(_cb, scale, "disk_data_mb", build=_ob) if _dk is not None: metrics["disk GiB"] = _dk # Only OUR arms carry a version string; the comparator containers do @@ -1220,6 +1242,7 @@ def _dense_overlay_entries(scale="deep10m"): v = _campaign_engine_string(_cb) out.append({ "backend": label, + "build": _ob or None, "backend_key": _cb, "is_arcadedb": ours, # NOT `arm == "int8"`. arm is a FILENAME token, and only ArcadeDB's @@ -1272,7 +1295,7 @@ def _peak_mem(r): return v if v is not None else _num(r.get(MEM_PEAK_FALLBACK)) -def _campaign_stat(backend, scale, field, lanes=("l3d", "l3s")): +def _campaign_stat(backend, scale, field, lanes=("l3d", "l3s"), build=""): """_agg of one field over the newest campaign row per rep for (backend, scale): the overlay files (multipass) carry no disk reading, but the campaign cell of the same arm in the same envelope does, and the on-disk @@ -1287,6 +1310,8 @@ def _campaign_stat(backend, scale, field, lanes=("l3d", "l3s")): continue if r.get("backend") != backend or r.get("scale") != scale or r.get("error"): continue + if build and _arm_build(r) != build: # one build's cell, never the other's + continue # PAGE-SPEC 4a rule 3: settled=False blocks publication of # the DISK reading. This path feeds the dense and sparse disk # columns straight from the campaign log, bypassing @@ -2790,10 +2815,11 @@ def _sparse_multipass_table(): # optional so the table renders before they land, and the sparse # condition says whether 100k has a second pass by looking, not by text. for tier in ("medium", "small", "tiny"): + _sp_units = [] for arm, backend, label in SPARSE_MP_ARMS + SPARSE_MP_OPTIONAL_ARMS: - fp = root / f"sp_{arm}_{tier}.json" - if not fp.is_file(): - continue + for _ob, _fps in _overlay_files_by_build(root, f"sp_{arm}", f"_{tier}.json", backend): + _sp_units += [(arm, backend, _with_build(label, _ob), _ob, _fps[0])] + for arm, backend, label, _ob, fp in _sp_units: passes = json.loads(fp.read_text(encoding="utf-8")) cold = [r for r in passes if r.get("rep") == 0] warm = [r for r in passes if r.get("rep", 0) >= 1] @@ -2806,6 +2832,7 @@ def _sparse_multipass_table(): w99 = statistics.median(r["query_p99_ms"] for r in warm if r.get("query_p99_ms") is not None) if any(r.get("query_p99_ms") is not None for r in warm) else None entries.append({ "backend": label, + "build": _ob or None, "backend_key": backend, "is_arcadedb": "arcade" in backend, "precision": SPARSE_PRECISION.get(backend), @@ -4464,8 +4491,9 @@ def _same_arm(r, _eb=e.get("backend")): # ArcadeDB ones that were. return "(" not in dn and str(_eb).startswith(dn + " (") + # one build's rows behind one build's entry (an entry that names no build reads every row of its arm) _match = lambda r: (r.get("lane") == lane and (not wl or r.get("workload") == wl) - and _same_arm(r)) + and _same_arm(r) and (not e.get("build") or _arm_build(r) == e["build"])) rs = [r for r in _FROZEN_ROWS if _match(r) and str(r.get("scale")) == str(e.get("scale"))] if not rs and not exact_only: rs = [r for r in _FROZEN_ROWS if _match(r)] @@ -4498,6 +4526,7 @@ def _heap_not_pinned(e, lane, wl): rs = [r for r in _FROZEN_ROWS if r.get("lane") == lane and (not wl or r.get("workload") == wl) and display_name(str(r.get("backend"))) == e.get("backend") + and (not e.get("build") or _arm_build(r) == e["build"]) and str(r.get("scale")) == str(e.get("scale"))] if not rs: return False @@ -5238,7 +5267,11 @@ def _same_pin(r): for r in rows: if not _same_pin(r): continue - key = (r.get("lane"), str(r.get("scale")), r.get("backend"), r.get("workload")) + # In-process ArcadeDB arms are censored per build (arm + build + size): a native out-of-memory or cap decision never takes + # the JVM entry off the page, and the other way round. The build rides on the backend element as "backend@build" and + # the consumers split it again; an old row with no build is the JVM build (arm_identity.build_of). + _b = _arm_build(r) + key = (r.get("lane"), str(r.get("scale")), f"{r.get('backend')}@{_b}" if _b else r.get("backend"), r.get("workload")) err = str(r.get("error") or "") if not err: clean.add(key) @@ -5445,6 +5478,7 @@ def _censored_entries(table): if not e.get("outcome")} out, marks = [], set() for (l, scale, backend, w), secs in sorted(_censored_cells().items(), key=str): + backend, _, build = str(backend).partition("@") if l != lane or (wl and w != wl): continue if scales and str(scale) not in scales: @@ -5461,7 +5495,8 @@ def _censored_entries(table): # "SurrealDB (embedded)" twice, one for documents with ten indexes and # one for the graph, and a reader could not tell which was which. _lc = lane == "lifecycle" - out.append({"backend": _lifecycle_row_label(backend, w) if _lc else display_name(backend), + out.append({"backend": _with_build(_lifecycle_row_label(backend, w) if _lc else display_name(backend), build), + "build": build or None, **({"engine": _lifecycle_engine(backend)} if _lc else {}), "backend_key": backend, "is_arcadedb": "arcadedb" in str(backend).lower(), @@ -5950,7 +5985,8 @@ def _censored_notes(table_id): notes = [] merged = {} scales = _table_scales(table_id) - for (l, scale, backend, w), secs in sorted(_censored_cells().items(), key=str): + for (l, scale, _qbackend, w), secs in sorted(_censored_cells().items(), key=str): + backend, _, _build = str(_qbackend).partition("@") if l != lane or (wl and w != wl): continue if scales and str(scale) not in scales: @@ -6002,7 +6038,7 @@ def _censored_notes(table_id): pins = [scale_label(lane, scale), scale_label(lane, secs.at)] else: budget = f"{secs / 3600:g} hour" if secs else "its" - _phase = _CENSORED_PHASE.get((lane, str(scale), backend, w)) + _phase = _CENSORED_PHASE.get((lane, str(scale), _qbackend, w)) _in = f" It was still in {_phase} when the budget ran out." if _phase else "" tail = (f" at {scale_label(lane, scale)}: {_cell} exceeded " f"its {budget} budget, the same budget every engine on this table had, on its first " @@ -6013,7 +6049,7 @@ def _censored_notes(table_id): f"inside its budget and was not retried, so there is no row. What it reported: " f"{secs}") pins = [scale_label(lane, scale), str(secs)] - merged.setdefault((kind, tail, tuple(pins)), []).append(display_name(backend)) + merged.setdefault((kind, tail, tuple(pins)), []).append(_with_build(display_name(backend), _build)) # ONE SENTENCE FOR THE ENGINES THAT HAVE THE SAME THING TO SAY. Three # engines were censored at TPC-H SF10 and the page printed three sentences @@ -6890,7 +6926,7 @@ def _finish_table(table: dict) -> dict: _per_situation = table.get("id") == "lifecycle" def _cell_of(m): - return (str(m.get("backend_key")), str(m.get("scale")), str(m.get("backend")) if _per_situation else "") + return (str(m.get("backend_key")), str(m.get("scale")), str(m.get("backend")) if _per_situation else "", m.get("build") or "") _cens = {_cell_of(m) for m in _marked} table["entries"] = [e for e in table["entries"] if e.get("outcome") or _cell_of(e) not in _cens] entries = table["entries"] + _marked diff --git a/benchmarks/experiments/make_2611_stages.py b/benchmarks/experiments/make_2611_stages.py index c8309bf1e1e..aa404ba0b60 100644 --- a/benchmarks/experiments/make_2611_stages.py +++ b/benchmarks/experiments/make_2611_stages.py @@ -124,6 +124,20 @@ # The sparse overlay names its files per scale; the dense one uses the runner's default mp_{label}_b{rep}.json, which is what # the dense resolver looks for. OVERLAY_ENV = {"l3s": 'BENCH_DRIVER_OUT_FMT=sp_{label}_{scale}.json'} +# THE NATIVE CHAIN'S OVERLAY FILES carry the build in their name (and every pass carries engine_backend), so the two builds of one pin +# share one directory without one chain overwriting the other: mp_fp32_native_b1.json beside mp_fp32_b1.json (arm_identity.py, export_web). +OVERLAY_ENV_NATIVE = {"l3s": 'BENCH_DRIVER_OUT_FMT=sp_{label}_native_{scale}.json', + "l3d": 'BENCH_DRIVER_OUT_FMT=mp_{label}_native_b{rep}.json'} +CHAIN_BUILD = "" # "native" while generating the native chain (set by main from the pinned wheel), else "" + + +def overlay_env(lane): + return (OVERLAY_ENV_NATIVE if CHAIN_BUILD == "native" else OVERLAY_ENV).get(lane, "") + + +def served_arcadedb_arms(): + """The served ArcadeDB arms (a client_server topology): the same Java server image whichever wheel is installed, so ONE chain runs them.""" + return sorted(b for b in runner.BACKENDS if engine_of(b) == "arcadedb" and runner.BACKENDS[b].get("topology") == "client_server") # THE OVERLAY LANES. The dense and sparse TABLES do not read the lane's own search rows: they read a multipass overlay # produced by a bespoke driver run through `runner.py --driver`, which builds once and then times five passes, so cold and @@ -519,8 +533,10 @@ def profile_problems(profile, stages, label=""): out.append(f"{tag}{st[0]}: {b} is a Neo4j/Memgraph dense cell, dropped by the {profile} profile") if DROP_QUANTIZED_DENSE in drops: dense = {b for st in stages if st[2] == "l3d" for b in stage_backends(st) if engine_of(b) == "arcadedb"} - if dense and dense != set(ARCADEDB_FP32_DENSE): - out.append(f"{tag}ArcadeDB's dense arms are {sorted(dense)}, expected exactly its fp32 pair {sorted(ARCADEDB_FP32_DENSE)}") + # the native chain runs the embedded half of the pair only: the served arm is the JVM chain's (served_arcadedb_arms) + want = set(ARCADEDB_FP32_DENSE) - (set(served_arcadedb_arms()) if CHAIN_BUILD == "native" else set()) + if dense and dense != want: + out.append(f"{tag}ArcadeDB's dense arms are {sorted(dense)}, expected exactly its fp32 {'arm' if len(want) == 1 else 'pair'} {sorted(want)}") for why in drops: if why == DROP_SECOND_LIST: continue @@ -800,6 +816,31 @@ def instrument_problems(env=None): _BACKENDS = re.compile(r'^BACKENDS="([^"]*)"', re.M) +_NATIVE_WHEEL = re.compile(r"dist/arcadedb-\d") + + +def served_arcadedb_in(text): + """The served ArcadeDB arms a stage script runs.""" + m = _BACKENDS.search(text) + served = set(served_arcadedb_arms()) + return sorted(b for b in (m.group(1).split() if m else []) if b in served) + + +def chain_overlap_problems(*dirs): + """A served ArcadeDB arm may be in ONE chain only. Given the draft folders of the chains of one release (native and JVM), a served + arm that appears in more than one would run its cells twice against the same server image. Also: a native chain runs none.""" + out, seen = [], {} + for d in dirs: + for f in sorted(glob.glob(os.path.join(d, "*.sh"))): + text = open(f, encoding="utf-8").read() + for b in served_arcadedb_in(text): + seen.setdefault(b, set()).add(d) + for b, where in sorted(seen.items()): + if len(where) > 1: + out.append(f"served ArcadeDB arm {b} is in {len(where)} chains ({', '.join(sorted(where))}); the server is one image, so one chain runs it") + return out + + def lint_draft(text, name, engines=None, profile=None): """Problems in one emitted stage script: it must export the instrument, name no other, write the 2611 rows file and run only subset arms (of `engines` when given), and, under a `profile`, no lane and no dense arm it drops. The pair-install @@ -824,6 +865,9 @@ def lint_draft(text, name, engines=None, profile=None): out.append(f"{name}: no BACKENDS line") else: arcade = has_arcadedb_arm(m.group(1).split()) + if _NATIVE_WHEEL.search(text) and served_arcadedb_in(text): + out.append(f"{name}: a native-build chain runs served ArcadeDB arms {served_arcadedb_in(text)}; the server is one image and " + f"the JVM chain runs it") # every stage builds images with the pinned wheel baked in, so every stage guards the wheel's sha256 if "is not the wheel this chain was generated for" not in text: out.append(f"{name}: no guard for the pinned wheel's sha256, which the image build bakes into every image") @@ -1532,7 +1576,7 @@ def stage_script(idx, total, spec, after, pins, params): if lane in OVERLAY: drv, oreps, dirs = OVERLAY[lane] outdir, orf = (rows_name(x) for x in dirs[scale]) - oenv = " ".join(x for x in [senv, OVERLAY_ENV.get(lane, "")] if x) + oenv = " ".join(x for x in [senv, overlay_env(lane)] if x) body += (f' run_overlay "{lane}/{scale}/$BE/{wl} multipass" {scale} {cap} ' f'"$BE" {wl} {drv} "{outdir}" "{orf}" {oreps} "{oenv}"\n') body += 'done\n' @@ -1867,6 +1911,9 @@ def main(argv=None) -> int: ap.add_argument("--project-reps", type=int, choices=(3, 5), help="with --project: price this many repetitions (default: the profile's target)") ap.add_argument("--check", action="store_true", help="selection, coverage and instrument proofs only; no pins needed") ap.add_argument("--drafts", metavar="DIR", help="with --check: also lint the scripts in DIR") + ap.add_argument("--also-drafts", metavar="DIR", action="append", default=[], + help="with --check --drafts: the other chain of the same release (the native chain beside the JVM one); fails when a " + "served ArcadeDB arm is in both") ap.add_argument("--arcadedb-release", metavar="X.Y.Z", help="ArcadeDB alone on this release (needs the five pin arguments)") ap.add_argument("--engine-commit") ap.add_argument("--image-digest") @@ -1895,7 +1942,12 @@ def main(argv=None) -> int: return 1 print(f"jpype pin: {bench_common.JPYPE_PIN}; instrument {INSTRUMENT}; mode {mode}; reps {a.reps}; profile {a.profile}") - skip = tuple(profile_skip(a.profile)) + ((TIMESCALEDB,) if a.without_timescaledb else ()) + # ONE CHAIN RUNS THE SERVED ARCADEDB CELLS. The server is the ARCADEDB_SERVER_IMAGE container, the same Java server whichever wheel + # is installed, so the JVM chain runs them and the native chain does not (a second run would double the cells under one key). + global CHAIN_BUILD + CHAIN_BUILD = engine_pin.backend_from_wheel(a.wheel_name) if mode == "arcadedb" else "" + skip = (tuple(profile_skip(a.profile)) + ((TIMESCALEDB,) if a.without_timescaledb else ()) + + (tuple(served_arcadedb_arms()) if CHAIN_BUILD == "native" else ())) specs = subset_specs(engines, lanes, a.order, skip, a.tiers) pair_id = f"{a.id_prefix}00" if mode == "arcadedb" else None stages = number(specs, a.id_prefix) @@ -1921,6 +1973,8 @@ def main(argv=None) -> int: print(" schedule: " + ", ".join(f"T{t} p{p}" for t, p in schedule_order({stage_tier(s) for s in stages}, a.reps))) if a.check and a.drafts: problems += lint_drafts(a.drafts, set(engines), status_file, a.profile) + if a.also_drafts: + problems += chain_overlap_problems(a.drafts, *a.also_drafts) for p in problems: print(f" PROBLEM {p}") if problems: diff --git a/benchmarks/experiments/make_paper_tables.py b/benchmarks/experiments/make_paper_tables.py index f0c41e8c4d0..56f9c753db8 100644 --- a/benchmarks/experiments/make_paper_tables.py +++ b/benchmarks/experiments/make_paper_tables.py @@ -800,14 +800,19 @@ def _pinned_sparse_rows(arm="arcadedb_sparse_embedded"): # partial or mixed, and still refuses. if not out: return {} - # Both builds of one release at one pin: this overlay path holds ONE build (arm_identity.select_build when the freeze names one); - # it is not yet build-aware, so with both and no choice it still stops here. + # Both builds of one release at one pin: the page shows both (export_web reads the frozen rows itself); this path feeds the paper's + # LaTeX table, which is one build, so it validates each build and returns the chosen one (below). _builds = {_AI.build_of(r) for t in out for r in out[t]} if len(_builds) > 1: - _want = _AI.wanted_build() - if not _want: - sys.stderr.write(f"pinned sparse rows hold both builds {sorted(map(str, _builds))}; set {_AI.ENV}\n") - return None + # Both builds measured: each is validated on its own below (five reps at every tier, one identity). The paper's LaTeX table is + # ONE build, so with no choice made it takes the JVM build, the build every earlier row is; BENCH_ENGINE_BACKEND picks the other. + # The page reads the frozen rows through export_web and shows both builds. + _want = _AI.wanted_build() or "jvm" + for _b in sorted(_builds - {_want}): + _other = {t: [r for r in rs if _AI.build_of(r) == _b] for t, rs in out.items()} + if any(len(_other.get(t, [])) < 5 for t in tiers) or len({_AI.arm_identity(r) for t in tiers for r in _other[t]}) != 1: + sys.stderr.write(f"pinned sparse rows of the {_b} build are incomplete or span two identities\n") + return None out = {t: [r for r in rs if _AI.build_of(r) == _want] for t, rs in out.items()} if any(len(out.get(t, [])) < 5 for t in tiers): missing = [t for t in tiers if len(out.get(t, [])) < 5] @@ -939,6 +944,20 @@ def mp_arms_present(small=False): return base + _optional_arms_present(cand, os.path.basename(cand), optional) +# The in-process ArcadeDB arms of the dense overlay (the others are served or comparators, which have no native build). +MP_INPROCESS_ARMS = ("fp32", "int8") + + +def _refuse_partial_native(cand, label, arms=MP_INPROCESS_ARMS, prefix="mp"): + """The native build's overlay files (mp__native_b.json) are all there or none: a partial native run is a run in progress, + and publishing it beside a complete JVM overlay would print a native arm built from fewer builds than its twin.""" + have = {a: [b for b in range(1, MP_BUILDS + 1) if os.path.isfile(os.path.join(cand, f"{prefix}_{a}_native_b{b}.json"))] + for a in arms} + if any(have.values()) and any(len(v) != MP_BUILDS for v in have.values()): + part = {a: len(v) for a, v in have.items() if len(v) != MP_BUILDS} + raise SystemExit(f"{label}: the native build's overlay is partial {part} of {MP_BUILDS} builds per arm; no fallback, re-run the arm") + + def dense_mp_dir(): """results/dense_mp5_: complete, or absent, or refuse. PINNED ONLY: a fallback to an older overlay would publish an older engine's numbers @@ -969,6 +988,7 @@ def dense_mp_dir(): if missing: raise SystemExit(f"dense_mp5_{pin}: {len(missing)} of {len(MP_ARMS) * MP_BUILDS} files missing " f"(e.g. {missing[0]}); no fallback, re-run the arm") + _refuse_partial_native(cand, f"dense_mp5_{pin}") return cand @@ -1020,6 +1040,7 @@ def _dense_multipass(): # could not decide whether an 85% cold-latency move between two engine # builds was a regression or noise. It is now five, one per build. stats = {} + _both_builds = bool(glob.glob(os.path.join(dense_mp_dir(), "mp_*_native_b*.json"))) and not _AI.wanted_build() for fp in sorted(glob.glob(os.path.join(dense_mp_dir(), "mp_*_b*.json"))): try: passes = json.load(open(fp)) @@ -1029,6 +1050,10 @@ def _dense_multipass(): continue p0 = passes[0] be = p0.get("backend") + _ob = _AI.build_of(p0) + _want = _AI.wanted_build() + if _want and _ob not in (None, _want): + continue # a one-build freeze reads that build's overlay files only label = NAMES.get(be, be) if be == "arcadedb_dense_embedded": q = str(p0.get("quantization", "")).lower() @@ -1040,6 +1065,8 @@ def _dense_multipass(): # this; the server arm was the one row still asserting. q = str(p0.get("quantization", "")).lower() label = "ArcadeDB (srv, int8)" if "int8" in q else "ArcadeDB (srv, fp32)" + if _ob and _both_builds: + label = f"{label} [{_ob}]" # both builds in the directory: two arms, never one pooled label s = stats.setdefault(label, {"build": [], "cold": [], "warm": [], "cold99": [], "recall": []}) # One value per BUILD for anything that is a property of the build. diff --git a/benchmarks/experiments/runner.py b/benchmarks/experiments/runner.py index 741da96e5d4..a035986562c 100644 --- a/benchmarks/experiments/runner.py +++ b/benchmarks/experiments/runner.py @@ -1802,6 +1802,7 @@ def _at_image_jvm_defaults(cfg): # THE STAGE'S PINNED ARCADEDB VERSION, and the image tag that names it (engine_pin.py, 2026-10-22). BACKENDS says `dbbench:arcadedb`, the # logical build target; the image a cell runs is `dbbench:arcadedb-`, swapped in below, so no cell can start on a floating tag. import engine_pin # noqa: E402 +import arm_identity as _AI # noqa: E402 _ENGINE_PIN = engine_pin.pin_from_wheel(_LOCAL_WHEEL) # The build the wheel is ("native" / "jvm", "" for the older arcadedb_embedded wheel): part of the image tag and read back from the image. _ENGINE_BACKEND = engine_pin.backend_from_wheel(_LOCAL_WHEEL) @@ -4480,6 +4481,7 @@ def worker(shard): active_backends.discard(job["backend"]) cv.notify_all() row["manifest"] = ts + _AI.stamp_build(row, _ENGINE_BACKEND) # a killed cell's row names the build its wheel is, not the JVM default # `or` rather than a dict default: an EMPTY error string is what # made thirty OOM kills look like blank successes. status = (row.get("error") diff --git a/benchmarks/experiments/test_build_pair_page.py b/benchmarks/experiments/test_build_pair_page.py index f5afa9dde69..b082c79d09a 100644 --- a/benchmarks/experiments/test_build_pair_page.py +++ b/benchmarks/experiments/test_build_pair_page.py @@ -165,6 +165,27 @@ def _with_native(results, keep=lambda r: True, commit=None): w.writerows(rows + twins) +def _native_overlays(results): + """The native chain's overlay files: a native copy of each in-process arm's files, every pass stamped engine_backend=native.""" + n = 0 + for d in results.glob("dense_mp5_*"): + for arm in ("fp32", "int8"): + for fp in sorted(d.glob(f"mp_{arm}_b*.json")): + passes = json.loads(fp.read_text()) + (d / fp.name.replace(f"mp_{arm}_b", f"mp_{arm}_native_b")).write_text( + json.dumps([dict(p, engine_backend="native") for p in passes])) + n += 1 + for d in results.glob("sparse_mp_*"): + for arm in ("arc_fp32", "arc_int8"): + for fp in sorted(d.glob(f"sp_{arm}_*.json")): + tier = fp.stem.rsplit("_", 1)[1] + passes = json.loads(fp.read_text()) + (d / f"sp_{arm}_native_{tier}.json").write_text(json.dumps([dict(p, engine_backend="native") for p in passes])) + n += 1 + assert n + return n + + @pytest.fixture(scope="module") def pages(tmp_path_factory): """(payload of the old freeze, payload with native twins everywhere, payload with native twins in every lane but l2, the dir).""" @@ -174,15 +195,18 @@ def pages(tmp_path_factory): base = _stage(root / "base") assert _run("make_paper_tables.py", base).returncode == 0 out = [] - for name, keep in (("old", None), ("both", lambda r: True), ("partial", lambda r: r.get("lane") != "l2")): + for name, keep in (("old", None), ("both", lambda r: True), ("partial", lambda r: r.get("lane") != "l2"), ("bare", lambda r: True)): d = root / name shutil.copytree(base, d) if keep: _with_native(d, keep) + if name != "bare": # "bare": native rows but no native overlay files (the overlay is unstamped, so the JVM build) + if keep: + _native_overlays(d) done = _run("export_web.py", d) assert done.returncode == 0, done.stderr[-2000:] out.append(json.loads((d / "web_benchmarks.json").read_text())) - return (*out, root) + return (out[0], out[1], out[2], root, out[3]) def _t(p, tid): @@ -228,11 +252,129 @@ def test_the_served_arm_stays_one_row_and_both_builds_name_one_engine(pages): assert PC._check_build_pair({"tables": [t for t in both["tables"] if t["id"] != "l3d"]}, rows) == 0 -def test_the_dense_overlay_arms_name_no_build_and_the_gate_says_so(pages): - """OPEN: the dense multipass overlay (files, not frozen rows) carries no engine_backend, so its fp32 and int8 arms print without a - build. On a page that names builds the gate refuses that, which is the loud behaviour wanted until the overlay says its build.""" +def test_a_stamped_overlay_stands_per_build_and_the_gate_passes_the_dense_table(pages): + """An overlay belongs to the build that produced it: the native files (engine_backend=native on every pass) and the JVM files + (no field: the JVM build) give each in-process dense arm two entries, and l3d no longer fails the build gate.""" both = pages[1] - assert PC._check_build_pair({"tables": [_t(both, "l3d")]}, []) > 0 + emb = [e["backend"] for e in _t(both, "l3d")["entries"] if e.get("is_arcadedb") and e.get("deployment") == "embedded"] + for prec in ("fp32", "int8"): + assert f"ArcadeDB (embedded, {prec}, native build)" in emb and f"ArcadeDB (embedded, {prec}, JVM build)" in emb, emb + assert PC._check_build_pair({"tables": [_t(both, "l3d")]}, []) == 0 + assert PC._check_build_pair(both, list(csv.DictReader((pages[3] / "both" / "runs_paper.csv").open()))) == 0 + + +def test_an_unstamped_overlay_is_the_jvm_build_and_the_native_build_is_declared_not_measured(pages): + """The default, chosen on purpose: an overlay file with no engine_backend is the JVM build (what every earlier overlay was). With + native rows on the page and no native overlay, the dense embedded arms read JVM build, and the native twin is a declared absence.""" + bare = pages[4] + t = _t(bare, "l3d") + emb = [e for e in t["entries"] if e.get("is_arcadedb") and e.get("deployment") == "embedded" and e.get("precision") in ("fp32", "int8")] + assert emb and {e["build"] for e in emb} == {"jvm"} # (the PQ arm comes from frozen rows, which have native twins) + assert any("native build" in a["backend"] and a["kind"] == "not measured" and ("fp32" in a["backend"] or "int8" in a["backend"]) + for a in t["declared_absences"]) + assert PC._check_build_pair({"tables": [t]}, []) == 0 + + +def test_the_sparse_second_pass_table_is_per_build_too(pages, monkeypatch): + """The second-pass table reads sp__.json (JVM, or unstamped) and sp__native_.json (native), per build.""" + pytest.importorskip("numpy") + pytest.importorskip("pandas") + import export_web as X + d = pages[3] / "both" + monkeypatch.setattr(X, "RESULTS", d) + monkeypatch.setenv("BENCH_ENGINE_COMMIT", PIN) + monkeypatch.setattr(X, "_SPLIT", True) + t = X._sparse_multipass_table() + assert t is not None + emb = [e for e in t["entries"] if e.get("is_arcadedb") and e.get("deployment") == "embedded"] + assert {e["build"] for e in emb} == {"native", "jvm"} + assert all(AI.build_in_label(e["backend"]) == e["build"] for e in emb) + monkeypatch.setattr(X, "_SPLIT", False) + plain = X._sparse_multipass_table() + assert {e["build"] for e in plain["entries"]} == {None} # an unsplit page reads the plain files only + + +def test_the_rows_behind_an_entry_are_that_builds_rows(): + """Notes that read the frozen rows behind an entry (heap, memory) read one build's rows for one build's entry.""" + pytest.importorskip("numpy") + pytest.importorskip("pandas") + import export_web as X + rows = [row("native", workload="oltp", heap_size="4g"), row("jvm", workload="oltp", heap_size="24g"), + row(None, backend="arcadedb_graph_server", workload="oltp")] + old = (X._FROZEN_ROWS, X._SPLIT) + try: + X._FROZEN_ROWS, X._SPLIT = rows, True + nat = X._entry_rows({"backend": "ArcadeDB (embedded, native build)", "build": "native", "scale": "sf1"}, "l2", "oltp") + jvm = X._entry_rows({"backend": "ArcadeDB (embedded, JVM build)", "build": "jvm", "scale": "sf1"}, "l2", "oltp") + assert [r["heap_size"] for r in nat] == ["4g"] and [r["heap_size"] for r in jvm] == ["24g"] + finally: + X._FROZEN_ROWS, X._SPLIT = old + + +def test_a_native_cap_or_oom_never_censors_the_jvm_entry(monkeypatch): + """Censored cells are keyed arm + build + size on a split page; an old run without a build is the JVM build.""" + pytest.importorskip("numpy") + pytest.importorskip("pandas") + import export_web as X + base = {"lane": "l2", "scale": "sf1", "workload": "oltp", "backend": "arcadedb_graph_embedded", "engine_commit": "abc123def"} + log = [dict(base, engine_backend="native", error="timeout_after_900s"), + dict(base, engine_backend="jvm", error=None, rc=0), + dict(base, scale="sf10", error="timeout_after_900s"), # an old row: no field + dict(base, scale="sf10", engine_backend="native", error=None, rc=0)] + monkeypatch.delenv("BENCH_ENGINE_COMMIT", raising=False) # no pin: every row is in scope + monkeypatch.setattr(X, "_run_log_rows", lambda: log) + monkeypatch.setattr(X, "_rule_censored", lambda taken, clean: {}) + monkeypatch.setattr(X, "_CENSORED_CACHE", None) + monkeypatch.setattr(X, "_SPLIT", True) + keys = X._censored_cells() + assert ("l2", "sf1", "arcadedb_graph_embedded@native", "oltp") in keys + assert ("l2", "sf1", "arcadedb_graph_embedded@jvm", "oltp") not in keys # the JVM twin ran clean + assert ("l2", "sf10", "arcadedb_graph_embedded@jvm", "oltp") in keys # unstamped old row: the JVM build + assert ("l2", "sf10", "arcadedb_graph_embedded@native", "oltp") not in keys + monkeypatch.setattr(X, "_CENSORED_CACHE", None) + monkeypatch.setattr(X, "_SPLIT", False) + assert all("@" not in k[2] for k in X._censored_cells()) # an unsplit page keys as it always did + + +def test_a_killed_cell_row_names_the_build_of_its_wheel(): + killed = {"lane": "l2", "backend": "arcadedb_graph_embedded", "error": "timeout_after_900s"} + assert AI.stamp_build(dict(killed), "native")["engine_backend"] == "native" + assert AI.build_of(AI.stamp_build(dict(killed), "native")) == "native" + assert AI.build_of(killed) == "jvm" # unstamped: the default + served = dict(killed, backend="arcadedb_graph_server") + assert "engine_backend" not in AI.stamp_build(served, "native") # a served arm has no build + assert AI.stamp_build(dict(killed, engine_backend="jvm"), "native")["engine_backend"] == "jvm" + + +def test_a_partial_native_overlay_is_refused_and_a_complete_or_absent_one_is_not(tmp_path): + import make_paper_tables as M + M._refuse_partial_native(str(tmp_path), "x") # none: nothing to refuse + for arm in M.MP_INPROCESS_ARMS: + for b in range(1, M.MP_BUILDS + 1): + (tmp_path / f"mp_{arm}_native_b{b}.json").write_text("[]") + M._refuse_partial_native(str(tmp_path), "x") # all of them + (tmp_path / "mp_int8_native_b2.json").unlink() + with pytest.raises(SystemExit) as e: + M._refuse_partial_native(str(tmp_path), "dense_mp5_x") + assert "partial" in str(e.value) + + +def test_the_pinned_sparse_path_validates_each_build_and_takes_the_jvm_one_unless_told(monkeypatch): + import make_paper_tables as M + + def rows(build, n=5): + return [dict(row(build, backend="arcadedb_sparse_embedded", lane="l3s", scale=t, rep=i), engine_commit="abc123def0") + for t in ("tiny", "small", "medium") for i in range(n)] + monkeypatch.setenv("BENCH_ENGINE_COMMIT", "abc123def0") + monkeypatch.delenv(AI.ENV, raising=False) + monkeypatch.setattr(M, "load_canonical", lambda *a, **k: rows("native") + rows("jvm")) + got = M._pinned_sparse_rows() + assert got and {AI.build_of(r) for rs in got.values() for r in rs} == {"jvm"} + monkeypatch.setenv(AI.ENV, "native") + assert {AI.build_of(r) for rs in M._pinned_sparse_rows().values() for r in rs} == {"native"} + monkeypatch.delenv(AI.ENV) + monkeypatch.setattr(M, "load_canonical", lambda *a, **k: rows("native", 3) + rows("jvm")) + assert M._pinned_sparse_rows() is None # a short native build is not published beside a full JVM one def test_neither_build_is_a_median_over_both(pages): @@ -261,3 +403,22 @@ def test_a_freeze_whose_builds_name_two_commits_is_not_exported(pages): _with_native(d, commit="fffffffff") done = _run("export_web.py", d) assert done.returncode != 0 and "engine_commit" in (done.stderr + done.stdout) + + +def test_a_censored_native_cell_becomes_a_native_row_and_leaves_the_jvm_entry(monkeypatch): + pytest.importorskip("numpy") + pytest.importorskip("pandas") + import export_web as X + base = {"lane": "l2", "scale": "sf1", "workload": "oltp", "backend": "arcadedb_graph_embedded", "engine_commit": "abc123def"} + log = [dict(base, engine_backend="native", error="timeout_after_900s"), dict(base, engine_backend="jvm", error=None, rc=0)] + monkeypatch.delenv("BENCH_ENGINE_COMMIT", raising=False) + monkeypatch.setattr(X, "_run_log_rows", lambda: log) + monkeypatch.setattr(X, "_rule_censored", lambda taken, clean: {}) + monkeypatch.setattr(X, "_CENSORED_CACHE", None) + monkeypatch.setattr(X, "_SPLIT", True) + measured = {"backend": "ArcadeDB (embedded, JVM build)", "build": "jvm", "backend_key": "arcadedb_graph_embedded", "scale": "sf1", + "is_arcadedb": True, "metrics": {"point p50 ms": {"median": 1.0}}} + table = {"id": "l2", "entries": [measured], "columns": ["point p50 ms"]} + marked, _marks = X._censored_entries(table) + assert [(m["backend"], m["build"]) for m in marked] == [("ArcadeDB (embedded, native build)", "native")] + assert table["entries"] == [measured] # the JVM entry is untouched diff --git a/benchmarks/experiments/test_chain_served_arms.py b/benchmarks/experiments/test_chain_served_arms.py new file mode 100644 index 00000000000..4ab93b0c0b3 --- /dev/null +++ b/benchmarks/experiments/test_chain_served_arms.py @@ -0,0 +1,90 @@ +"""The served ArcadeDB cells run in ONE chain: the JVM chain. The native chain skips them, and a check fails when both have them. + +The server is the ARCADEDB_SERVER_IMAGE container, the same Java server whichever wheel is installed; a second run would put two sets of +rows under one key. Run with `python -m pytest test_chain_served_arms.py -q` from this directory. Nothing is run: only drafts are written. +""" +import re +import sys +from pathlib import Path + +import pytest + +HERE = Path(__file__).resolve().parent +sys.path.insert(0, str(HERE)) + +import test_stage_generator_2611 as T # noqa: E402 + +G = T.G +NATIVE_WHEEL = T.REL_WHEEL.replace("arcadedb_embedded-", "arcadedb-") +JVM_WHEEL = T.REL_WHEEL.replace("arcadedb_embedded-", "arcadedb_jvm-") + + +def _chain(tmp_path, name, wheel, prefix): + out = tmp_path / name + r = T.run(T._release_args(**{"--wheel-name": wheel, "--wheel-url": "https://files.pythonhosted.org/packages/x/" + wheel}) + + ["--after", "none", "--id-prefix", prefix, "--out", str(out)]) + assert r.returncode == 0, r.stdout[-800:] + r.stderr[-800:] + return out + + +def _arms(directory): + found = set() + for f in Path(directory).glob("*.sh"): + for m in re.finditer(r'^BACKENDS="([^"]*)"', f.read_text(), re.M): + found.update(m.group(1).split()) + return found + + +@pytest.fixture(scope="module") +def chains(tmp_path_factory): + root = tmp_path_factory.mktemp("chains") + return _chain(root, "native", NATIVE_WHEEL, "qN"), _chain(root, "jvm", JVM_WHEEL, "qJ"), _chain(root, "old", T.REL_WHEEL, "qO") + + +def test_the_served_arcadedb_arms_are_the_client_server_ones(): + served = G.served_arcadedb_arms() + assert "arcadedb_server" in served and "arcadedb_graph_server" in served + assert not [b for b in served if "embedded" in b and "server" not in b] + assert "arcadedb_graph_embedded" not in served + + +def test_the_native_chain_runs_no_served_arcadedb_cell_and_keeps_the_embedded_ones(chains): + native, jvm, _old = chains + served = set(G.served_arcadedb_arms()) + assert _arms(native) and not (_arms(native) & served) + assert any(b.startswith("arcadedb") for b in _arms(native)) + assert _arms(jvm) & served # the JVM chain is the one that runs the server + + +def test_the_jvm_chain_and_a_chain_of_an_old_wheel_are_unchanged_by_the_native_rule(chains): + _native, jvm, old = chains + assert _arms(jvm) == _arms(old) # arcadedb_embedded wheels are the JVM build + + +def test_the_two_chains_of_one_release_pass_the_overlap_check(chains): + native, jvm, _old = chains + assert G.chain_overlap_problems(str(native), str(jvm)) == [] + + +def test_the_overlap_check_fails_when_a_served_arm_is_in_both_chains(chains): + _native, jvm, old = chains + problems = G.chain_overlap_problems(str(jvm), str(old)) + assert problems and all("one image" in p for p in problems) + assert any("arcadedb_graph_server" in p for p in problems) + + +def test_a_native_draft_that_runs_a_served_arm_fails_the_lint(chains): + native, jvm, _old = chains + served_stage = next(f for f in sorted(jvm.glob("*.sh")) if G.served_arcadedb_in(f.read_text())) + forged = served_stage.read_text().replace("dist/" + JVM_WHEEL, "dist/" + NATIVE_WHEEL) + assert "dist/" + NATIVE_WHEEL in forged + assert any("native-build chain runs served ArcadeDB arms" in p for p in G.lint_draft(forged, "qN01.sh")) + clean = next(f for f in sorted(native.glob("*.sh")) if 'BACKENDS="' in f.read_text()) + assert not [p for p in G.lint_draft(clean.read_text(), clean.name) if "native-build chain" in p] + + +def test_the_native_chain_names_its_overlay_files_by_build(chains): + native, jvm, _old = chains + text = lambda d: "".join(f.read_text() for f in d.glob("*.sh")) # noqa: E731 + assert "mp_{label}_native_b{rep}.json" in text(native) + assert "mp_{label}_native_b{rep}.json" not in text(jvm) From f0990f0b2da2eabc66f0cc714acf3e9bf1aad93c Mon Sep 17 00:00:00 2001 From: Taewoon Kim Date: Fri, 30 Oct 2026 10:56:20 +0900 Subject: [PATCH 07/23] benchmarks: a native server arm (upstream's native-image server) beside the JVM server, four ArcadeDB arms from one engine commit DECISIONS #222 (user, 2026-10-30): the page and the paper show ArcadeDB's native and JVM builds side by side in embedded AND server mode. Upstream publishes an official native-image server (arcadedata/arcadedb:-native, native-image.yml, native/pom.xml), so the native server arm is that image, addressed exactly like the Docker JVM server: no launcher, same client, cpuset, memory cap and split. 1. runner: a chain whose wheel is the native build runs every served ArcadeDB arm on ARCADEDB_NATIVE_SERVER_IMAGE (refused when unset: F5, wheel + JVM image + native image are one commit). native_server_arm derives the container command (the scratch image has no shell and no JAVA_OPTS: its entrypoint takes -D and -Xmx arguments) FROM the JVM arm's own JAVA_OPTS, so both servers are configured from one list; -XX flags have no native meaning and are recorded in native_dropped_flags. The image-defaults sensitivity arm and e4 have no native counterpart and stay in the JVM chain. 2. Explicit settings for both servers: the JVM image left server mode, bind host, port and (relaxed class) txWalFlush to its defaults; they are now written out for every served arm, and the strict class replaces the txWalFlush token (one token: the JVM takes the last, the native image the first). Left at the engine default on both and only read back: arcadedb.maxPageRAM. arcadedb_settings_readback asks each running engine over HTTP (server?mode=default, schema:database, SELECT version()), stamps server_settings / server_engine_commit on the row and refuses a cell whose engine runs a requested setting otherwise or whose commit is not the pin. observe_server reads -Xmx and the -D tokens from the container's Cmd too. 3. Rows: server rows carry engine_backend native|jvm (old server rows without it are the JVM server); the client container reports the build of the server it talks to (BENCH_ARCADEDB_SERVER_BUILD), the runner stamps killed cells from server_build. arm_identity.built() = in-process or served; labels "ArcadeDB (server, native build)" / "(server, JVM build)"; the build-pair gates, not-measured declarations and per-build notes apply to server arms through the same code. pair_problems also compares server settings between the builds of one cell and the servers' engine commit. cpu_baseline is stamped per artifact (native server -march=x86-64-v3 from native.march, native wheel -march=compatibility, JVM at run time) and disclosed under the server tables from the rows; neither baseline is changed. 4. Stage generator: the native chain no longer skips served arms; it exports the native image pin and BENCH_SERVER_BUILD (the runner refuses a value that is not its wheel's build) and checks the native server's commit in the pre-flight (verify_native_server_image.sh). chain_served_problems: each chain runs the served arms of its own build once, the same ones in both; the lint refuses a script that names the other build's server. Output of the JVM chain and of old-wheel chains is byte for byte unchanged (stage_drafts_2611.sha256 holds). 5. make_paper_tables: no single-build default (the sparse path takes a build or "all"; the T2 and sparse tables print both builds). 6. smoke_native_server.sh (not run here): one served cell of l2 and one of docs_oltp on the published native image, rows checked; PAGE-SPEC and READING-RESULTS describe the four arms and the CPU baseline disclosure. Tests: test_native_server_arm.py (new, 15), test_chain_served_arms.py rewritten, test_build_pair_page.py and test_native_jvm_arms.py updated for server rows carrying a build; on the pre-change code 25 tests fail and the 10 page-fixture tests error. Full benchmarks/experiments suite: 1131 passed. Co-Authored-By: Claude Opus 5.5 --- benchmarks/experiments/PAGE-SPEC.md | 4 +- benchmarks/experiments/READING-RESULTS.md | 2 +- benchmarks/experiments/arm_identity.py | 106 ++++++++- benchmarks/experiments/bench_common.py | 8 +- benchmarks/experiments/export_web.py | 27 ++- benchmarks/experiments/make_2611_stages.py | 106 +++++++-- benchmarks/experiments/make_paper_tables.py | 57 +++-- benchmarks/experiments/page_check.py | 4 +- benchmarks/experiments/runner.py | 206 +++++++++++++++++- benchmarks/experiments/smoke_native_server.sh | 71 ++++++ .../experiments/test_build_pair_page.py | 45 ++-- .../experiments/test_chain_served_arms.py | 76 +++++-- .../experiments/test_native_jvm_arms.py | 10 +- .../experiments/test_native_server_arm.py | 152 +++++++++++++ .../experiments/verify_native_server_image.sh | 46 ++++ 15 files changed, 821 insertions(+), 99 deletions(-) create mode 100755 benchmarks/experiments/smoke_native_server.sh create mode 100644 benchmarks/experiments/test_native_server_arm.py create mode 100755 benchmarks/experiments/verify_native_server_image.sh diff --git a/benchmarks/experiments/PAGE-SPEC.md b/benchmarks/experiments/PAGE-SPEC.md index 0a23a041957..78e95ecab58 100644 --- a/benchmarks/experiments/PAGE-SPEC.md +++ b/benchmarks/experiments/PAGE-SPEC.md @@ -57,8 +57,8 @@ ArcadeDB is pinned by **upstream commit SHA**, not by a release number. The buil - Every row carries `engine_commit` (short SHA of the upstream commit the wheel and server image were both built from). - The page prints it as the engine identity, linked to the commit on GitHub. Every ArcadeDB row is named from its own engine string and commit, never from an image tag; served rows carry the row's `server_image_ref` (BUGS.md F16, F32). The page header lists the identities actually present. - Embedded and server arms in one table are built from the SAME commit, paired, and run the same JVM (DECISIONS #54). -- **Two builds, two arms (26.11.1).** ArcadeDB has two builds of one release: `arcadedb` (the native image, the default install) and `arcadedb-jvm` (JPype and a bundled JDK), both built from one engine commit. A page whose frozen rows hold the native build shows every in-process ArcadeDB arm twice, side by side, with the build last in the label: `ArcadeDB (embedded, native build)` and `ArcadeDB (embedded, JVM build)`, `ArcadeDB (embedded, fp32, native build)` and so on, in every table that has the arm (the lifecycle table puts the build after the situation: `Graph (embedded, native build)`). The native build comes first. The build is `engine_backend` on the row; a row without it, or with it empty, is the JVM build (`arm_identity.build_of`). The served ArcadeDB arm stays one arm: the server is the same Java server whichever package drives it, and its rows carry no build. A freeze of rows from before 26.11.1 has no native row, so it prints no build and renders as it always did. `BENCH_ENGINE_BACKEND=native|jvm` still freezes one build on purpose; the labels then still name it. -- **Both builds, one engine commit, or no page.** `export_web` refuses a freeze whose native and JVM rows name different `engine_version` or `engine_commit` values, and `page_check` fails the same page again (`_check_build_pair`). The gate also fails an in-process ArcadeDB arm that names no build on a page that names builds, two builds of one arm and size whose `version_name` differ, and a build that has a row at a size where the other has none. That last case is not left out: the exporter prints the missing build as a declared absence of kind `not measured` (a whole-row declaration with one generated sentence per table and build, naming the arms and sizes), which the roster and the pair gate read as accounted for. A table never takes a median over both builds: the exporter groups on the build and `_one_version` refuses an entry that spans two. Where the lifecycle table holds both builds its start column is `engine start ms`, the `start_jvm()` call on the JVM build and the library load with the isolate creation on the native build, and a sentence under the table says so. **Overlays belong to the build that produced them.** The dense and sparse multipass drivers stamp `engine_backend` on every pass they write (`run_conditions`), the native chain names its files `mp__native_b.json` and `sp__native_.json` beside the JVM build's, and the exporter reads the build from the file's own content, so the in-process dense and sparse arms stand once per build like any other arm. An overlay file without the field is the JVM build, as an old row is; a page with native rows and no native overlay therefore prints the JVM arm and declares the native one `not measured`. A partial native overlay (not every build of every in-process arm) refuses the freeze (`make_paper_tables._refuse_partial_native`). The paper's pinned-sparse path validates each build on its own and, with no `BENCH_ENGINE_BACKEND`, takes the JVM build. **Censored cells and the notes read per build.** A cap, timeout or out-of-memory outcome is keyed arm, build and size, so a native outcome never removes the JVM entry; the runner stamps `engine_backend` on the row of a cell it killed, because that row comes from the runner and not from the engine process, and an old outcome without the field is the JVM build. The heap and memory notes read the frozen rows behind each build's entry. **The server runs in one chain.** The served ArcadeDB arm is the `ARCADEDB_SERVER_IMAGE` container, the same Java server whichever wheel is installed, so the JVM chain runs the served cells and the native chain skips them (`make_2611_stages.served_arcadedb_arms`); `--check --drafts DIR --also-drafts DIR2` fails when a served ArcadeDB arm is in both chains, and the lint fails a native draft that runs one. +- **Two builds, two arms (26.11.1).** ArcadeDB has two builds of one release: `arcadedb` (the native image, the default install) and `arcadedb-jvm` (JPype and a bundled JDK), both built from one engine commit. A page whose frozen rows hold the native build shows every in-process ArcadeDB arm twice, side by side, with the build last in the label: `ArcadeDB (embedded, native build)` and `ArcadeDB (embedded, JVM build)`, `ArcadeDB (embedded, fp32, native build)` and so on, in every table that has the arm (the lifecycle table puts the build after the situation: `Graph (embedded, native build)`). The native build comes first. The build is `engine_backend` on the row; a row without it, or with it empty, is the JVM build (`arm_identity.build_of`). The served ArcadeDB arm stands twice as well: `ArcadeDB (server, native build)` and `ArcadeDB (server, JVM build)`, so four ArcadeDB arms in all, from one engine commit. The JVM server is the official Docker image built at the pair's commit (`ARCADEDB_SERVER_IMAGE`); the native server is upstream's official native-image server, `arcadedata/arcadedb:-native` (`ARCADEDB_NATIVE_SERVER_IMAGE`; built by upstream's `native-image.yml` from `native/pom.xml`), started and addressed exactly like the JVM one: same client, cpuset, memory cap and split. Both get the same explicit settings (server mode, bind host and port, `txWalFlush` per durability class, the HTTP body limit and query cap where the arm sets them), the native one as `-D` arguments derived from the JVM arm's own `JAVA_OPTS`; the runner reads each value back from the running engine (`server_settings`, with `server_engine_commit` from `SELECT version()`) and refuses a cell whose engine answers otherwise. The page discloses the one difference under the server tables: the CPU baseline each artifact was compiled for (`cpu_baseline` on the row: the native server image `-march=x86-64-v3` from `native.march`, the native wheel `-march=compatibility`, the JVM compiling for the host CPU); neither baseline is changed. A server row carries `engine_backend` (a row without it is the JVM server, the only one before 26.11.1); the JVM image-defaults sensitivity arm has no native counterpart and stays one arm. A freeze of rows from before 26.11.1 has no native row, so it prints no build and renders as it always did. `BENCH_ENGINE_BACKEND=native|jvm` still freezes one build on purpose; the labels then still name it. +- **Both builds, one engine commit, or no page.** `export_web` refuses a freeze whose native and JVM rows name different `engine_version` or `engine_commit` values, and `page_check` fails the same page again (`_check_build_pair`). The gate also fails an in-process ArcadeDB arm that names no build on a page that names builds, two builds of one arm and size whose `version_name` differ, and a build that has a row at a size where the other has none. That last case is not left out: the exporter prints the missing build as a declared absence of kind `not measured` (a whole-row declaration with one generated sentence per table and build, naming the arms and sizes), which the roster and the pair gate read as accounted for. A table never takes a median over both builds: the exporter groups on the build and `_one_version` refuses an entry that spans two. Where the lifecycle table holds both builds its start column is `engine start ms`, the `start_jvm()` call on the JVM build and the library load with the isolate creation on the native build, and a sentence under the table says so. **Overlays belong to the build that produced them.** The dense and sparse multipass drivers stamp `engine_backend` on every pass they write (`run_conditions`), the native chain names its files `mp__native_b.json` and `sp__native_.json` beside the JVM build's, and the exporter reads the build from the file's own content, so the in-process dense and sparse arms stand once per build like any other arm. An overlay file without the field is the JVM build, as an old row is; a page with native rows and no native overlay therefore prints the JVM arm and declares the native one `not measured`. A partial native overlay (not every build of every in-process arm) refuses the freeze (`make_paper_tables._refuse_partial_native`). The paper's pinned-sparse path validates each build on its own and, with no `BENCH_ENGINE_BACKEND`, takes the JVM build. **Censored cells and the notes read per build.** A cap, timeout or out-of-memory outcome is keyed arm, build and size, so a native outcome never removes the JVM entry; the runner stamps `engine_backend` on the row of a cell it killed, because that row comes from the runner and not from the engine process, and an old outcome without the field is the JVM build. The heap and memory notes read the frozen rows behind each build's entry. **Each chain runs the server of its own build.** The native chain (native wheel) runs every served ArcadeDB arm on the native server image and the JVM chain on the Docker JVM image, each arm once; the JVM-only arms (the image-defaults sensitivity arm, e4) stay in the JVM chain (`make_2611_stages.build_less_served_arms`). The stage exports `BENCH_SERVER_BUILD` for its build and the runner refuses one that is not its wheel's; `--check --drafts DIR --also-drafts DIR2` fails when the two chains do not run the same served arms once each, and the lint fails a script that names the other build's server. The pre-flight checks the native server's engine commit (`verify_native_server_image.sh`, `SELECT version()`) against the pin, like the JVM pair. **The pair gate covers the servers.** The same gates (one commit, a row per build or a declared absence, equal server settings between the builds of one cell: `arm_identity.server_settings_problems`) apply to the server arms. - A campaign FREEZES one commit and holds it start to finish. Upstream landing a fix mid-run does not restart the campaign; it becomes a dated changelog entry and a candidate for the next re-pin. - Re-pinning is a deliberate, dated act recorded in the changelog with what moved. No table falls back to an older pin, anywhere (DECISIONS #62). - **One ArcadeDB version until the final DBBench release, then history (DECISIONS #189).** Until the final DBBench page and repo are released (after the late-November 2026 paper), a new ArcadeDB release's pass REPLACES the old one: after one scripted old-against-new comparison (a regression is filed before the next cut), the old version's ArcadeDB rows, page data, pin-specific code, pinned test fixtures and wording are removed. Comparator rows are kept across it and carry their own engine's version, never ArcadeDB's. From the final release on, a new engine version ADDS rows and the page keeps the history: diff --git a/benchmarks/experiments/READING-RESULTS.md b/benchmarks/experiments/READING-RESULTS.md index d2ddc3af37d..ed80cdbc346 100644 --- a/benchmarks/experiments/READING-RESULTS.md +++ b/benchmarks/experiments/READING-RESULTS.md @@ -26,7 +26,7 @@ The single rule, from which most of the rest follows: **`engine_version` is deliberately null for non-ArcadeDB rows in L4.** Use `backend_version`. `run_conditions()` stamps `engine_version` from the installed wheel, which is right for the ArcadeDB row and wrong for DuckDB and QuestDB, so `l4_tsbs.py` asks each backend for its own version instead. A wrong version that passes a provenance audit is worse than none. -**`engine_backend` says which ArcadeDB build produced a row: `native` (the `arcadedb` wheel) or `jvm` (`arcadedb-jvm`, the older `arcadedb_embedded`).** `engine_version` is the same string for both wheels of one release, and `backend` is the arm name, so neither can tell them apart. It is read from `arcadedb.__backend__` by `bench_common.engine_backend()` and is null where the process never loaded ArcadeDB (every comparator, and a served ArcadeDB arm, whose engine runs in a container). The page shows both builds as two arms, `native build` and `JVM build`, one next to the other (PAGE-SPEC section 1); a row without the field is the JVM build. +**`engine_backend` says which ArcadeDB build produced a row: `native` (the `arcadedb` wheel) or `jvm` (`arcadedb-jvm`, the older `arcadedb_embedded`).** `engine_version` is the same string for both wheels of one release, and `backend` is the arm name, so neither can tell them apart. It is read from `arcadedb.__backend__` by `bench_common.engine_backend()` and for a served ArcadeDB arm it is the build of the server container the cell talked to (the official JVM server image or upstream's official native-image server, named by `server_build`); it is null for every comparator and for the image-defaults sensitivity arm. A server row also carries the settings the engine reported at start (`server_settings`), its own commit (`server_engine_commit`) and the CPU baseline of the artifact (`cpu_baseline`). The page shows both builds, embedded and server, as four arms, `native build` and `JVM build`, one next to the other (PAGE-SPEC section 1); a row without the field is the JVM build. **Arms are selected by flag, not by filename.** `claims_check.ts_arm()` takes `primitive=` and `numpy_cols=` because the published row is one specific arm. Assert the flags rather than globbing the directory. diff --git a/benchmarks/experiments/arm_identity.py b/benchmarks/experiments/arm_identity.py index db59a90d081..ec58c8d85c7 100644 --- a/benchmarks/experiments/arm_identity.py +++ b/benchmarks/experiments/arm_identity.py @@ -9,8 +9,10 @@ * `engine_backend` is "native" or "jvm": that value. * a row of an ArcadeDB in-process arm WITHOUT the field, or with it empty (every row before 26.11.1, and those rows read back from the frozen CSV): "jvm", because that is what it ran. - * a served arm, the HTTP client container and every comparator: None. The served engine is a JVM server whichever Python package - drives it, and the field is null there on purpose, so it never selects or splits them. + * a served ArcadeDB arm (`arcadedb_server`, `arcadedb_graph_server`, ...) carries the build of the SERVER it talked to since + #222: "jvm" for the official Docker server image, "native" for the native wheel's in-process server (native_server_launcher.py). + A served row without the field (every row before it) is "jvm", the only server there was. The image-defaults sensitivity arm + (`arcadedb_imgdefaults_server`), the HTTP client container and every comparator carry None: they have no native counterpart. `select_build(rows, want)` is the one place a freeze picks a build: with BENCH_ENGINE_BACKEND=native|jvm the other build's in-process ArcadeDB rows leave the freeze (they stay in runs.jsonl); without it BOTH builds stay, and the page shows them side by side as two arms @@ -36,13 +38,24 @@ def in_process(r): return b.startswith("arcadedb") and not any(t in b for t in _NOT_IN_PROCESS) +def served(r): + """True for a served ArcadeDB arm that stands once per build: the HTTP server arms, not the image-defaults sensitivity arm.""" + b = str(r.get("backend") or "") + return b.startswith("arcadedb") and "server" in b and not any(t in b for t in ("http_client", "imgdefaults")) + + +def built(r): + """True for an ArcadeDB arm that stands once per build: an in-process arm or a served one (#222).""" + return in_process(r) or served(r) + + def build_of(r): """"native", "jvm" or None for the row (see the module docstring).""" eb = r.get("engine_backend") if eb in BUILDS: return eb # No usable field: an old row, or a row read back from the frozen CSV, where a row from before 26.11.1 has the column empty. - if in_process(r) and not eb: + if built(r) and not eb: return "jvm" return None @@ -54,7 +67,7 @@ def arm_identity(r): def identities(rows): """The set of (engine_version, build) over the in-process ArcadeDB rows.""" - return {arm_identity(r) for r in rows if in_process(r)} + return {arm_identity(r) for r in rows if built(r)} def wanted_build(env=None): @@ -76,25 +89,51 @@ def select_build(rows, want=None): """ want = wanted_build() if want is None else want if want: - return [r for r in rows if not (in_process(r) and build_of(r) not in (None, want))] + return [r for r in rows if not (built(r) and build_of(r) not in (None, want))] return rows def stamp_build(row, wheel_build): - """Give an in-process ArcadeDB row the build of the wheel its runner was pinned to, when the row names none. + """Give an ArcadeDB row (in-process or served) its build when the row names none: the build of the wheel its runner was pinned to, or, + for a served row, the server's own (`server_build`, written by the runner at launch). A cell killed by its cap or by the kernel writes a row from the runner, not from the engine process, so the row has no engine_backend of its own; without this a native timeout would read as the JVM build (the default for an unstamped row). Returns the row. """ - if wheel_build in BUILDS and in_process(row) and not row.get("engine_backend"): - row["engine_backend"] = wheel_build + if built(row) and not row.get("engine_backend"): + if served(row) and row.get("server_build") in BUILDS: + row["engine_backend"] = row["server_build"] + elif in_process(row) and wheel_build in BUILDS: + row["engine_backend"] = wheel_build + if built(row) and row.get("engine_backend") in BUILDS and not row.get("cpu_baseline"): + row["cpu_baseline"] = cpu_baseline(row["engine_backend"], served(row)) return row +# THE CPU BASELINE OF EACH ARTIFACT, disclosed on the page and left as built. The native wheel is compiled with `-march=compatibility` +# (runs on any x86-64 CPU); upstream's official native server image is compiled for `native.march` (native/pom.xml: x86-64-v3 on amd64, i.e. +# AVX2/BMI2/FMA, set by ArcadeData/arcadedb#10581; override with ARCADEDB_NATIVE_SERVER_MARCH if a different image is pinned); the JVM +# server and the JVM wheel compile at run time for the CPU they run on. +NATIVE_WHEEL_MARCH = "compatibility" +NATIVE_SERVER_MARCH_DEFAULT = "x86-64-v3" + + +def native_server_march(env=None): + return ((os.environ if env is None else env).get("ARCADEDB_NATIVE_SERVER_MARCH", "").strip() or NATIVE_SERVER_MARCH_DEFAULT) + + +def cpu_baseline(build, is_server, env=None): + """The CPU baseline sentence stamped on a row of `build` ("native" / "jvm").""" + if build == "native": + return (f"native server image: -march={native_server_march(env)}" if is_server + else f"native wheel: -march={NATIVE_WHEEL_MARCH}") + return "JVM: compiled at run time for the host CPU" + + def builds_in(rows): """The builds the in-process ArcadeDB rows of `rows` ran on, as a set of "native" / "jvm".""" - return {build_of(r) for r in rows if in_process(r)} - {None} + return {build_of(r) for r in rows if built(r)} - {None} def splits_page(rows): @@ -142,15 +181,60 @@ def pair_problems(rows): out = [] by = {} for r in rows: - if in_process(r) and build_of(r): + if built(r) and build_of(r): by.setdefault(build_of(r), []).append(r) if len(by) < 2: return out for field in ("engine_version", "engine_commit"): - seen = {b: sorted({str(r.get(field) or "") for r in rs} - {""}) for b, rs in by.items()} + # engine_version of a served row is the server's own build string ("server: (build ...)"), not the wheel's: compare the wheels' versions + # on the in-process rows; the commit is one value for every ArcadeDB row, served ones included. + seen = {b: sorted({str(r.get(field) or "") for r in rs if field != "engine_version" or in_process(r)} - {""}) for b, rs in by.items()} flat = {v for vs in seen.values() for v in vs} if len(flat) > 1: out.append(f"the two ArcadeDB builds name different {field} values {seen}; one release is built from one engine commit") + out += server_settings_problems(rows) + return out + + +# THE SETTINGS A SERVER ARM IS HELD TO (#222 (3)). Both servers (the Docker JVM image, the native wheel's in-process server) are launched with +# the same explicit settings and the runner reads each back from the engine at start (`server_settings`, key -> value in force). The keys +# below must be present on every served ArcadeDB row and equal between the two builds of one cell (same arm, scale, durability class); +# other keys the runner requested (the HTTP body limit, the query cap) are compared when both builds list them. +SERVER_SETTING_KEYS = ("arcadedb.server.mode", "arcadedb.server.httpIncomingHost", "arcadedb.server.httpIncomingPort", "arcadedb.txWalFlush") + + +def server_settings_problems(rows): + """Sentences for a served-arm settings mismatch between the builds (empty: fine). A native server row with no read-back is a problem + too: a setting written is not a setting in force.""" + out, cells = [], {} + for r in rows: + if not served(r) or build_of(r) not in BUILDS: + continue + got = r.get("server_settings") + if isinstance(got, str): + try: + import json + got = json.loads(got) + except ValueError: + got = None + if build_of(r) == "native" and not isinstance(got, dict): + out.append(f"{r.get('backend')}: a native server row has no server_settings read-back") + continue + sc, pin = str(r.get("server_engine_commit") or ""), str(r.get("engine_commit") or "") + if sc and pin and not (sc.startswith(pin[:7]) or pin.startswith(sc[:7])): + out.append(f"{r.get('backend')}: the {build_of(r)} server reports engine commit {sc[:9]} and the row names {pin[:9]}") + if isinstance(got, dict): + key = (r.get("backend"), str(r.get("scale")), str(r.get("durability_class") or "")) + cells.setdefault(key, {}).setdefault(build_of(r), []).append(got) + for (arm, scale, cls), by in sorted(cells.items(), key=str): + if len(by) < 2: + continue + a, b = (by[x][0] for x in BUILDS) + keys = set(SERVER_SETTING_KEYS) | (set(a) & set(b)) + for k in sorted(keys): + if str(a.get(k)) != str(b.get(k)): + out.append(f"{arm} at {scale} ({cls or 'any class'}): server setting {k} is {a.get(k)!r} on the native server and " + f"{b.get(k)!r} on the JVM server") return out diff --git a/benchmarks/experiments/bench_common.py b/benchmarks/experiments/bench_common.py index 19b1b33edd8..8eccb8eb3f8 100644 --- a/benchmarks/experiments/bench_common.py +++ b/benchmarks/experiments/bench_common.py @@ -824,12 +824,14 @@ def _host_identity(): def engine_backend(): - """"native" or "jvm" when this process loaded the ArcadeDB package, else None. Lanes that set engine_version by hand stamp it too.""" + """"native" or "jvm": the package this process loaded, else the build of the ArcadeDB server it talks to (BENCH_ARCADEDB_SERVER_BUILD, set by the runner on a served ArcadeDB arm's client), else None. Lanes that set engine_version by hand stamp it too.""" try: import _arcadedb_import - return _arcadedb_import.loaded_backend() + got = _arcadedb_import.loaded_backend() except Exception: - return None + got = None + # A served ArcadeDB arm's client container never imports the package: the runner names the build of the SERVER it started (#222). + return got or os.environ.get("BENCH_ARCADEDB_SERVER_BUILD", "").strip() or None def run_conditions(**extra): diff --git a/benchmarks/experiments/export_web.py b/benchmarks/experiments/export_web.py index 4f9bd7932a4..60b9702a557 100644 --- a/benchmarks/experiments/export_web.py +++ b/benchmarks/experiments/export_web.py @@ -1095,7 +1095,7 @@ def _overlay_files_by_build(root, stem, tail, backend_key): "jvm" read from the first pass of each file (arm_identity.build_of; an unstamped ArcadeDB file is the JVM build). The native files carry `_native` after the stem; a glob on the plain stem never matches them.""" names = sorted(root.glob(f"{stem}{tail}")) + sorted(root.glob(f"{stem}_native{tail}")) - if not (_SPLIT and _AI.in_process({"backend": backend_key})): + if not (_SPLIT and _AI.built({"backend": backend_key})): return [("", sorted(root.glob(f"{stem}{tail}")))] if names else [] got = {} for fp in names: @@ -1444,7 +1444,7 @@ def _agg(rows, field, across_scales=False): def _arm_build(r): """"native" or "jvm" for an in-process ArcadeDB row of a split page, else "" (served arms, comparators, an unsplit page).""" - return (_AI.build_of(r) or "") if (_SPLIT and _AI.in_process(r)) else "" + return (_AI.build_of(r) or "") if (_SPLIT and _AI.built(r)) else "" def _with_build(label, build): @@ -1875,6 +1875,28 @@ def _declare_missing_builds(tables): t["conditions"].append(note) +def _disclose_server_builds(tables): + """Under every table that holds a native-build server arm: both server arms are upstream's official images (the JVM server image and the + native-image server), run with the same server settings that each server reports back at start, and the one disclosed difference is the + CPU baseline each artifact was compiled for (read from the rows' `cpu_baseline`, never typed here).""" + if not _SPLIT or _AI.wanted_build(): + return + nat = next((r.get("cpu_baseline") for r in _FROZEN_ROWS if _AI.served(r) and _AI.build_of(r) == "native" and r.get("cpu_baseline")), None) + wheel = next((r.get("cpu_baseline") for r in _FROZEN_ROWS if _AI.in_process(r) and _AI.build_of(r) == "native" and r.get("cpu_baseline")), None) + if not nat: + return + wheel_part = f" and the native wheel (the embedded arm) as {wheel}" if wheel else "" + for t in tables: + if not any(e.get("build") == "native" and "server" in str(e.get("backend")) for e in t.get("entries", [])): + continue + note = _gen(f"Both server arms are the official ArcadeDB server images, one built for the JVM and one compiled to a native executable, " + f"started with the same server settings, which each server reports back when it starts. The CPU baselines differ: " + f"the native server image is {nat}{wheel_part}, and the JVM server compiles for the CPU it runs on.", nat, *([wheel] if wheel else [])) + t.setdefault("conditions", []) + if note not in t["conditions"]: + t["conditions"].append(note) + + def _withhold_cells(tables): """Take out the cells whose answer was wrong, and say so on the table.""" notes = collections.defaultdict(list) @@ -7726,6 +7748,7 @@ def main() -> int: if _note not in _t["conditions"]: _t["conditions"].append(_note) _declare_missing_builds(tables) + _disclose_server_builds(tables) # THE OVERRIDES A READER MEETS (CAMPAIGN section 7 row 21). PROTOCOL.md # section 7 lists every default this benchmark overrides, and the rows its # last column marked NOWHERE were named in no sentence under any table. One diff --git a/benchmarks/experiments/make_2611_stages.py b/benchmarks/experiments/make_2611_stages.py index aa404ba0b60..cd35ad01ab1 100644 --- a/benchmarks/experiments/make_2611_stages.py +++ b/benchmarks/experiments/make_2611_stages.py @@ -86,6 +86,7 @@ import re import sys +import arm_identity as _AI import bench_common import cap_rule import runner @@ -136,9 +137,15 @@ def overlay_env(lane): def served_arcadedb_arms(): - """The served ArcadeDB arms (a client_server topology): the same Java server image whichever wheel is installed, so ONE chain runs them.""" + """The served ArcadeDB arms (a client_server topology). Both chains run them (#222): the JVM chain on the Docker JVM server image, the + native chain on upstream's native server image, each arm once per chain.""" return sorted(b for b in runner.BACKENDS if engine_of(b) == "arcadedb" and runner.BACKENDS[b].get("topology") == "client_server") + +def build_less_served_arms(): + """The served ArcadeDB arms with no native counterpart (the JVM image-defaults sensitivity arm): the JVM chain runs them, the native one does not.""" + return [b for b in served_arcadedb_arms() if not _AI.served({"backend": b})] + # THE OVERLAY LANES. The dense and sparse TABLES do not read the lane's own search rows: they read a multipass overlay # produced by a bespoke driver run through `runner.py --driver`, which builds once and then times five passes, so cold and # warm exist for every engine under one protocol. The lane row is still needed (it carries ingest, build, memory and disk), @@ -534,7 +541,7 @@ def profile_problems(profile, stages, label=""): if DROP_QUANTIZED_DENSE in drops: dense = {b for st in stages if st[2] == "l3d" for b in stage_backends(st) if engine_of(b) == "arcadedb"} # the native chain runs the embedded half of the pair only: the served arm is the JVM chain's (served_arcadedb_arms) - want = set(ARCADEDB_FP32_DENSE) - (set(served_arcadedb_arms()) if CHAIN_BUILD == "native" else set()) + want = set(ARCADEDB_FP32_DENSE) - (set(build_less_served_arms()) if CHAIN_BUILD == "native" else set()) if dense and dense != want: out.append(f"{tag}ArcadeDB's dense arms are {sorted(dense)}, expected exactly its fp32 {'arm' if len(want) == 1 else 'pair'} {sorted(want)}") for why in drops: @@ -826,18 +833,38 @@ def served_arcadedb_in(text): return sorted(b for b in (m.group(1).split() if m else []) if b in served) -def chain_overlap_problems(*dirs): - """A served ArcadeDB arm may be in ONE chain only. Given the draft folders of the chains of one release (native and JVM), a served - arm that appears in more than one would run its cells twice against the same server image. Also: a native chain runs none.""" - out, seen = [], {} +def _chain_of(text): + """"native" for a stage script of a native-wheel chain, else "jvm".""" + return "native" if _NATIVE_WHEEL.search(text) else "jvm" + + +def chain_served_problems(*dirs): + """Each chain runs exactly the served ArcadeDB arms of its own build, each once (#222 (5)). Given the draft folders of the chains of one + release (native and JVM): one folder per build, and every served arm runs in the same lanes in both. A folder that runs the other + build's server (a script exporting the other build) fails in lint_draft.""" + out, per = [], {} for d in dirs: + counts = collections.Counter() + builds = set() for f in sorted(glob.glob(os.path.join(d, "*.sh"))): text = open(f, encoding="utf-8").read() + if _BACKENDS.search(text): + builds.add(_chain_of(text)) + lane = re.search(r"--lanes (\w+) ", text) for b in served_arcadedb_in(text): - seen.setdefault(b, set()).add(d) - for b, where in sorted(seen.items()): - if len(where) > 1: - out.append(f"served ArcadeDB arm {b} is in {len(where)} chains ({', '.join(sorted(where))}); the server is one image, so one chain runs it") + if b not in build_less_served_arms(): # the JVM-only arms (image defaults, e4) belong to the JVM chain alone + counts[(b, lane.group(1) if lane else "?")] += 1 + if len(builds) > 1: + out.append(f"{d} mixes native and JVM stage scripts: one chain, one build") + per[d] = (next(iter(builds), ""), counts) + if len({b for b, _c in per.values()}) < len(per): + out.append("two chains of the same build were given: one release is a native chain and a JVM chain") + base = next(iter(per.values()), ("", {}))[1] + for d, (_b, counts) in per.items(): + for key in sorted(set(counts) | set(base)): + if counts.get(key, 0) != base.get(key, 0): + out.append(f"served ArcadeDB arm {key[0]} (lane {key[1]}) runs {counts.get(key, 0)}x in {d} and {base.get(key, 0)}x in the other " + f"chain; each chain runs it once") return out @@ -865,9 +892,19 @@ def lint_draft(text, name, engines=None, profile=None): out.append(f"{name}: no BACKENDS line") else: arcade = has_arcadedb_arm(m.group(1).split()) - if _NATIVE_WHEEL.search(text) and served_arcadedb_in(text): - out.append(f"{name}: a native-build chain runs served ArcadeDB arms {served_arcadedb_in(text)}; the server is one image and " - f"the JVM chain runs it") + # THE SERVED ARM IS THE SERVER OF THE CHAIN'S OWN BUILD (#222 (5)): a native-wheel chain runs the native server image, a JVM chain the + # Docker JVM image; the runner refuses a BENCH_SERVER_BUILD that is not the wheel's build, and this lint refuses a script that names the other. + if has_arcadedb_arm(m.group(1).split()): + native_chain = _chain_of(text) == "native" + want_build = f'BENCH_SERVER_BUILD="{"native" if native_chain else "jvm"}"' + other = "jvm" if native_chain else "native" + if f'BENCH_SERVER_BUILD="{other}"' in text or (not native_chain and "ARCADEDB_NATIVE_SERVER_IMAGE" in text): + out.append(f"{name}: a {'native' if native_chain else 'JVM'}-build chain runs the {other} build's server arm") + if native_chain and served_arcadedb_in(text): + for needle, what in ((want_build, "BENCH_SERVER_BUILD for its own build"), ("export ARCADEDB_NATIVE_SERVER_IMAGE=", "the native server image pin"), + ("ABORT: native server image", "the native server's engine-commit check")): + if needle not in text: + out.append(f"{name}: a native-build chain runs served ArcadeDB arms but has no {what}") # every stage builds images with the pinned wheel baked in, so every stage guards the wheel's sha256 if "is not the wheel this chain was generated for" not in text: out.append(f"{name}: no guard for the pinned wheel's sha256, which the image build bakes into every image") @@ -1532,9 +1569,28 @@ def stage_params(reps): "rep_helpers": REP_HELPERS.lstrip("\n"), "temp_gate_helpers": TEMP_GATE_HELPERS.lstrip("\n"), "pass_no": 1} +NATIVE_EXPORT = ('export ARCADEDB_NATIVE_SERVER_IMAGE="{img}" BENCH_SERVER_BUILD="native"\n' + 'docker image inspect "$ARCADEDB_NATIVE_SERVER_IMAGE" >/dev/null 2>&1 || {{ say "$ID ABORT: no native server image $ARCADEDB_NATIVE_SERVER_IMAGE"; exit 1; }}\n') +NATIVE_VERIFY = ('bash ./verify_native_server_image.sh "$ARCADEDB_NATIVE_SERVER_IMAGE" "$SHA" >> "$S" 2>&1 || {{ say "$ID ABORT: native server image is not at the pin"; exit 1; }}\n') + + def _pair_part(text, pins, params): - return text.format(wheel_name=pins["wheel_name"], wheel_sha256=pins["wheel_sha256"], - server_image=pins["ARCADEDB_SERVER_IMAGE"], jpype_version=params["jpype_version"]) + out = text.format(wheel_name=pins["wheel_name"], wheel_sha256=pins["wheel_sha256"], + server_image=pins["ARCADEDB_SERVER_IMAGE"], jpype_version=params["jpype_version"]) + # A native-wheel chain also pins upstream's native server image, and says which build its server arms are (#222). Only that chain's text + # changes: the JVM chain, and a chain of an old wheel, are emitted byte for byte as before. + if CHAIN_BUILD == "native": + img = pins.get("ARCADEDB_NATIVE_SERVER_IMAGE") + if not img: + raise SystemExit("REFUSING: a native-wheel chain needs the native server image (--native-image-digest with --arcadedb-release, or " + "ARCADEDB_NATIVE_SERVER_IMAGE): wheel, JVM server image and native server image are one engine commit (F5)") + anchor = f'ARCADEDB_SERVER_IMAGE="{pins["ARCADEDB_SERVER_IMAGE"]}"\n' + if anchor in out: + out = out.replace(anchor, anchor + NATIVE_EXPORT.format(img=img).replace("{{", "{").replace("}}", "}"), 1) + tail = 'ABORT: pair unverified at the pin"; exit 1; }\n' + if tail in out: + out = out.replace(tail, tail + NATIVE_VERIFY.replace("{{", "{").replace("}}", "}"), 1) + return out def stage_script(idx, total, spec, after, pins, params): @@ -1791,15 +1847,22 @@ def release_pins(a): raise SystemExit(f"REFUSING: --wheel-name {a.wheel_name} is not a {a.arcadedb_release} arcadedb, arcadedb_jvm or arcadedb_embedded wheel") if not a.wheel_url.startswith("https://") or not a.wheel_url.endswith(a.wheel_name): raise SystemExit("REFUSING: --wheel-url must be an https URL ending in the wheel's file name") + nd = getattr(a, "native_image_digest", None) + if nd and not re.fullmatch(r"sha256:[0-9a-f]{64}", nd): + raise SystemExit("REFUSING: --native-image-digest must be sha256:<64 hex>, upstream's -native image by digest") return {"ARCADEDB_WHEEL": os.path.join("bindings", "python", "dist", a.wheel_name), "ARCADEDB_WHEEL_SHA256": a.wheel_sha256, "ARCADEDB_SERVER_IMAGE": f"arcadedb-c25:{a.engine_commit[:9]}", + # upstream's published native server image: by digest when given, else its release tag + "ARCADEDB_NATIVE_SERVER_IMAGE": (f"arcadedata/arcadedb@{nd}" if nd else f"arcadedata/arcadedb:{a.arcadedb_release}-native"), "ARCADEDB_ENGINE_COMMIT": a.engine_commit} def installed_pins(): """The comparator mode's pins: the release pair already installed, from the environment.""" pins = {k: os.environ.get(k, "").strip() for k in ("ARCADEDB_WHEEL", "ARCADEDB_SERVER_IMAGE", "ARCADEDB_ENGINE_COMMIT")} + if os.environ.get("ARCADEDB_NATIVE_SERVER_IMAGE", "").strip(): + pins["ARCADEDB_NATIVE_SERVER_IMAGE"] = os.environ["ARCADEDB_NATIVE_SERVER_IMAGE"].strip() missing = [k for k, v in pins.items() if not v] if missing: raise SystemExit(f"REFUSING to emit: {', '.join(missing)} unset. The stages are generated for one release pair: " @@ -1913,8 +1976,10 @@ def main(argv=None) -> int: ap.add_argument("--drafts", metavar="DIR", help="with --check: also lint the scripts in DIR") ap.add_argument("--also-drafts", metavar="DIR", action="append", default=[], help="with --check --drafts: the other chain of the same release (the native chain beside the JVM one); fails when a " - "served ArcadeDB arm is in both") + "served ArcadeDB arm does not run once in each") ap.add_argument("--arcadedb-release", metavar="X.Y.Z", help="ArcadeDB alone on this release (needs the five pin arguments)") + ap.add_argument("--native-image-digest", help="the digest of upstream's arcadedata/arcadedb:-native image (the native chain's server); " + "without it the release tag is pinned") ap.add_argument("--engine-commit") ap.add_argument("--image-digest") ap.add_argument("--wheel-name") @@ -1942,12 +2007,13 @@ def main(argv=None) -> int: return 1 print(f"jpype pin: {bench_common.JPYPE_PIN}; instrument {INSTRUMENT}; mode {mode}; reps {a.reps}; profile {a.profile}") - # ONE CHAIN RUNS THE SERVED ARCADEDB CELLS. The server is the ARCADEDB_SERVER_IMAGE container, the same Java server whichever wheel - # is installed, so the JVM chain runs them and the native chain does not (a second run would double the cells under one key). + # EACH CHAIN RUNS THE SERVED ARCADEDB CELLS OF ITS OWN BUILD (#222): the JVM chain on the Docker JVM server image, the native chain on + # upstream's native server image (ARCADEDB_NATIVE_SERVER_IMAGE). Only the image-defaults sensitivity arm, which is a JVM-image arm, stays out of + # the native chain. global CHAIN_BUILD CHAIN_BUILD = engine_pin.backend_from_wheel(a.wheel_name) if mode == "arcadedb" else "" skip = (tuple(profile_skip(a.profile)) + ((TIMESCALEDB,) if a.without_timescaledb else ()) - + (tuple(served_arcadedb_arms()) if CHAIN_BUILD == "native" else ())) + + (tuple(build_less_served_arms()) if CHAIN_BUILD == "native" else ())) specs = subset_specs(engines, lanes, a.order, skip, a.tiers) pair_id = f"{a.id_prefix}00" if mode == "arcadedb" else None stages = number(specs, a.id_prefix) @@ -1974,7 +2040,7 @@ def main(argv=None) -> int: if a.check and a.drafts: problems += lint_drafts(a.drafts, set(engines), status_file, a.profile) if a.also_drafts: - problems += chain_overlap_problems(a.drafts, *a.also_drafts) + problems += chain_served_problems(a.drafts, *a.also_drafts) for p in problems: print(f" PROBLEM {p}") if problems: diff --git a/benchmarks/experiments/make_paper_tables.py b/benchmarks/experiments/make_paper_tables.py index 56f9c753db8..198e7a38fdc 100644 --- a/benchmarks/experiments/make_paper_tables.py +++ b/benchmarks/experiments/make_paper_tables.py @@ -685,10 +685,24 @@ def write(name, body): print("wrote", name) +def _builds_of_rows(rows): + """The builds present in `rows`, native first.""" + got = {_AI.build_of(r) for r in rows} - {None} + return [b for b in ("native", "jvm") if b in got] + + +def _build_name(label, build, split): + """The arm's label with its build when the table names builds ("ArcadeDB (srv)" -> "ArcadeDB (srv, native build)").""" + return _AI.build_label(label, build) if (build and split) else label + + def tabular_table(rows): l1 = [r for r in rows if r["lane"] == "l1"] tpc = [r for r in rows if r["lane"] == "l1tpc"] - order = ["arcadedb_embedded", "arcadedb_server", "duckdb", "postgres"] + # Both ArcadeDB builds, embedded and server, stand as their own rows (#222 (4)); a freeze of old rows has no native row and renders as before. + split = _AI.splits_page(rows) + order = [(be, b) for be in ("arcadedb_embedded", "arcadedb_server") + for b in (("native", "jvm") if split else (None,))] + [("duckdb", None), ("postgres", None)] # Peak memory (owned: anonymous plus shared, issue #353) from the OLTP # rows only. This lane runs OLTP and OLAP over one corpus, and memory is # recorded for BOTH, so pooling them would report a median straddling two @@ -698,9 +712,9 @@ def tabular_table(rows): lines = [r"\begin{tabular}{lrrrrr}", r"\toprule", r"System & OLTP (ops/s) & Ins.\ p99 (ms) & " r"Q1 (ms) & Q6 (ms) & Peak memory (GiB) \\", r"\midrule"] - for be in order: - oltp = [r for r in l1 if r["backend"] == be and r["workload"] == "oltp"] - tq = [r for r in tpc if r["backend"] == be and r["workload"] == "olap"] + for be, bld in order: + oltp = [r for r in l1 if r["backend"] == be and r["workload"] == "oltp" and (bld is None or _AI.build_of(r) == bld)] + tq = [r for r in tpc if r["backend"] == be and r["workload"] == "olap" and (bld is None or _AI.build_of(r) == bld)] # MEDIAN ONLY for Q1/Q6, ranges kept for OLTP and insert p99. # # Not a space-saving dodge: it puts dispersion where dispersion is @@ -713,7 +727,7 @@ def tabular_table(rows): # as it took last time. The caption states the bound they now carry # implicitly, and the artifact has every repetition. lines.append(" & ".join([ - NAMES[be], + _build_name(NAMES[be], bld, split), mmm(oltp, "oltp_ops_per_s"), mmm(oltp, "insert_p99_ms"), fmt(st.median([r["q1_ms"] for r in tq if isinstance(r.get("q1_ms"), (int, float))])) @@ -769,7 +783,7 @@ def graph_table(rows): # would keep the failure mode alive, since the next missing directory would # re-arm it. _sparse_2681_rows below refuses instead of falling back. -def _pinned_sparse_rows(arm="arcadedb_sparse_embedded"): +def _pinned_sparse_rows(arm="arcadedb_sparse_embedded", build=None): """T4's ArcadeDB rows from the PINNED campaign, when it is complete. The paper corpus rows (8,841,823 x 30,109; load_canonical's PAPER_CORPUS @@ -805,30 +819,34 @@ def _pinned_sparse_rows(arm="arcadedb_sparse_embedded"): _builds = {_AI.build_of(r) for t in out for r in out[t]} if len(_builds) > 1: # Both builds measured: each is validated on its own below (five reps at every tier, one identity). The paper's LaTeX table is - # ONE build, so with no choice made it takes the JVM build, the build every earlier row is; BENCH_ENGINE_BACKEND picks the other. - # The page reads the frozen rows through export_web and shows both builds. - _want = _AI.wanted_build() or "jvm" - for _b in sorted(_builds - {_want}): + # ONE build per call and NO DEFAULT (#222 (4)): the caller names the build (or "all" for every build, each validated), and + # BENCH_ENGINE_BACKEND names it for a one-build freeze. The tables loop over the builds and print both. + _want = build or _AI.wanted_build() + if _want is None: + sys.stderr.write("pinned sparse rows hold both builds: name the build (there is no single-build default)\n") + return None + for _b in sorted(_builds - ({_want} if _want != "all" else set())): _other = {t: [r for r in rs if _AI.build_of(r) == _b] for t, rs in out.items()} if any(len(_other.get(t, [])) < 5 for t in tiers) or len({_AI.arm_identity(r) for t in tiers for r in _other[t]}) != 1: sys.stderr.write(f"pinned sparse rows of the {_b} build are incomplete or span two identities\n") return None - out = {t: [r for r in rs if _AI.build_of(r) == _want] for t, rs in out.items()} + if _want != "all": + out = {t: [r for r in rs if _AI.build_of(r) == _want] for t, rs in out.items()} if any(len(out.get(t, [])) < 5 for t in tiers): missing = [t for t in tiers if len(out.get(t, [])) < 5] sys.stderr.write(f"pinned sparse rows incomplete at {missing}\n") return None versions = {_AI.arm_identity(r) for t in tiers for r in out[t]} # (engine_version, build): one engine, one build - if len(versions) != 1: + if len(versions) != (len(_builds) if build == "all" else 1): sys.stderr.write(f"pinned sparse rows span {sorted(versions, key=str)}\n") return None return out -def _sparse_2681_rows(arm="arcadedb_sparse_embedded"): +def _sparse_2681_rows(arm="arcadedb_sparse_embedded", build=None): """The pinned sparse rows, or refuse. There is no overlay fallback: incomplete or mixed pinned rows stop the generation.""" - pinned = _pinned_sparse_rows(arm) + pinned = _pinned_sparse_rows(arm, build) if pinned is None: raise SystemExit(f"pinned sparse rows for {arm} incomplete or mixed (see stderr); no fallback") return pinned # {} when the arm has not run at this pin (absent, not partial) @@ -840,7 +858,9 @@ def sparse_table(rows): # runs.jsonl and are untouched: they are not ArcadeDB runs, their own # versions did not change, and the F9 control re-runs one of them to # license keeping the other two. - arc = _sparse_2681_rows() + arc_all = _sparse_2681_rows(build="all") + arc_builds = _builds_of_rows([r for rs in arc_all.values() for r in rs]) + arc_by_build = {b: {t: [r for r in rs if _AI.build_of(r) == b] for t, rs in arc_all.items()} for b in arc_builds} order = ["arcadedb_sparse_embedded", "qdrant_sparse", "milvus_sparse", "elasticsearch_sparse"] tiers = (("tiny", "100k"), ("small", "1M"), ("medium", "8.84M")) @@ -860,7 +880,8 @@ def sparse_table(rows): # which is the side of the trade the sparse narrative uses less. lines = [r"\begin{tabular}{llrrr}", r"\toprule", r"System & Tier & p50 (ms) & p99 (ms) & R@10 \\", r"\midrule"] - for i, be in enumerate(order): + blocks = [(be, b) for be in order for b in (arc_builds if be == "arcadedb_sparse_embedded" and arc_builds else [None])] + for i, (be, bld) in enumerate(blocks): if i: lines.append(r"\addlinespace") for j, (sc, lab) in enumerate(tiers): @@ -875,10 +896,10 @@ def sparse_table(rows): # has not run at this pin, and the `if not g` branch four # lines down already renders that as "--". Indexing raised # KeyError instead and took the whole publish with it. - g = arc.get(sc) or [] + g = arc_by_build.get(bld, {}).get(sc) or [] # The system name labels its block once; repeating it down three # rows is noise the eye has to subtract. - head = NAMES[be] if j == 0 else "" + head = _build_name(NAMES[be], bld, len(arc_builds) > 1 or "native" in arc_builds) if j == 0 else "" if not g: lines.append(" & ".join([head, lab, "--", "--", "--"]) + r" \\") continue diff --git a/benchmarks/experiments/page_check.py b/benchmarks/experiments/page_check.py index 5e4845395a9..90f7f55ea54 100644 --- a/benchmarks/experiments/page_check.py +++ b/benchmarks/experiments/page_check.py @@ -1356,7 +1356,7 @@ def _check_build_pair(payload, rows): arms = collections.defaultdict(dict) absent = {str(a.get("backend")) for a in (t.get("declared_absences") or []) if not a.get("column")} for e in t.get("entries", []): - if not (e.get("is_arcadedb") and _AI.in_process({"backend": e.get("backend_key") or ""})): + if not (e.get("is_arcadedb") and _AI.built({"backend": e.get("backend_key") or ""})): continue b = e.get("build") if b not in _AI.BUILDS: @@ -1446,7 +1446,7 @@ def _check_lane_roster(payload): continue # TWO BUILDS, TWO ARMS: on a split page an in-process ArcadeDB arm is owed one row per build, and a build with none is a # declared absence like any other ("not measured"), under the arm's label with the build. - if _page_split(payload) and _AI.in_process({"backend": key}) and key in keyed: + if _page_split(payload) and _AI.built({"backend": key}) and key in keyed: have = {e.get("build") for e in keyed[key]} for b in _AI.BUILDS: checked += 1 diff --git a/benchmarks/experiments/runner.py b/benchmarks/experiments/runner.py index a035986562c..c3652ef9a52 100644 --- a/benchmarks/experiments/runner.py +++ b/benchmarks/experiments/runner.py @@ -1714,6 +1714,41 @@ def _with_http_body_limit(cfg): BACKENDS[_be] = _with_http_body_limit(BACKENDS[_be]) +# THE SERVER SETTINGS BOTH SERVER ARMS RUN WITH, EXPLICIT (#222 (3)). The JVM server image left four of them to its own defaults: the server +# mode (development), the HTTP bind host (0.0.0.0), the HTTP port (2480) and, at the relaxed durability class, txWalFlush (0). The native +# server image takes its settings as the same -D tokens on its command line, so the defaults of two different builds would have been the +# only thing keeping the servers equal. They are written out here for every served ArcadeDB arm, the strict class replaces the txWalFlush +# token (durability_server_patch), and the runner reads each one back from the running engine (arcadedb_settings_readback) and refuses a cell +# whose engine answers otherwise. NOT made explicit: arcadedb.maxPageRAM (the page cache). Neither build is given a value, because no arm +# of the campaign sets one; it is read back and stamped on both servers, so a difference is visible (a gate in arm_identity compares it). +EXPLICIT_SERVER_SETTINGS = ("arcadedb.server.mode=development", "arcadedb.server.httpIncomingHost=0.0.0.0", + "arcadedb.server.httpIncomingPort=2480", "arcadedb.txWalFlush=0") +PAGE_CACHE_KEY = "arcadedb.maxPageRAM" +STAMPED_SETTING_KEYS = tuple(k.split("=")[0] for k in EXPLICIT_SERVER_SETTINGS) + (PAGE_CACHE_KEY,) + + +def _with_explicit_settings(cfg): + """A copy of a served ArcadeDB arm's launch configuration whose JAVA_OPTS also carries each EXPLICIT_SERVER_SETTINGS entry it lacks.""" + out = dict(cfg) + env, hits = list(cfg["server_env"]), 0 + for i, e in enumerate(env): + if isinstance(e, str) and e.startswith("JAVA_OPTS="): + for tok in EXPLICIT_SERVER_SETTINGS: + if f"-D{tok.split('=')[0]}=" not in e: + e = e + f" -D{tok}" + env[i] = e + hits += 1 + if hits != 1: + raise SystemExit(f"a served ArcadeDB arm must carry exactly one JAVA_OPTS entry (found {hits})") + out["server_env"] = env + return out + + +for _be, _cfg in list(BACKENDS.items()): + if "arcadedb" in _be and _cfg.get("topology") == "client_server" and str(_cfg.get("server_image", "")).startswith("arcadedata/arcadedb"): + BACKENDS[_be] = _with_explicit_settings(_cfg) + + # THE QUERY HEAP BUDGET OVERRIDE (DECISIONS #175; bench_common.QUERY_RAM_ENV): OFF unless the launching environment sets the variable. The # repair stage for the l1tpc tpch10 olap cells sets it, and then the served documents arm is launched with the flag; with the variable unset # BACKENDS is exactly what it was, so the running chain is unchanged by construction (test_overrides holds that). @@ -1888,6 +1923,64 @@ def _require_local_server_image(): f"server={_LOCAL_SERVER_IMAGE} ({_swapped} server arms swapped)") +# THE NATIVE SERVER ARM (#222). Upstream publishes an official native-image server, arcadedata/arcadedb:-native (built by its +# native-image.yml from native/pom.xml). It is addressed exactly like the JVM Docker server: same client code, cpuset, memory cap and split, +# same explicit settings. It differs in how it is configured: the entrypoint is the native executable and takes `-D=` and +# `-Xmx` as command-line arguments (there is no shell, no JAVA_OPTS, no server.sh). native_server_arm derives that command line FROM the +# JVM arm's own JAVA_OPTS, so the two servers are configured from one list by construction. A chain whose wheel is the native build runs +# the native server image for every served ArcadeDB arm; the JVM chain runs the Docker JVM image. Same engine commit for all four arms. +_LOCAL_NATIVE_SERVER_IMAGE = os.environ.get("ARCADEDB_NATIVE_SERVER_IMAGE", "").strip() + + +_SERVER_BUILD_ENV = os.environ.get("BENCH_SERVER_BUILD", "").strip() +if _SERVER_BUILD_ENV and _SERVER_BUILD_ENV != (_ENGINE_BACKEND or "jvm"): + raise SystemExit(f"REFUSING: BENCH_SERVER_BUILD={_SERVER_BUILD_ENV} but the pinned wheel is the {_ENGINE_BACKEND or 'jvm'} build. A chain runs the " + "served ArcadeDB arm of its own build: the native wheel with the native server image, the JVM wheel with the Docker JVM server.") + + +def native_server_arm(cfg, image): + """A copy of a served ArcadeDB arm's launch configuration that starts upstream's native server image: the arm's `-D` tokens (from its + JAVA_OPTS) and `-Xmx{heap}` (from ARCADEDB_OPTS_MEMORY) as the container command, no environment. JVM-only flags (-XX:...) have no native + meaning and are dropped; they are recorded in `native_dropped_flags`, which lands on the row.""" + out = dict(cfg) + toks, dropped, heap = [], [], None + for i, e in enumerate(cfg["server_env"]): + if not (isinstance(e, str) and e not in ("-e",)): + continue + if e.startswith("ARCADEDB_OPTS_MEMORY="): + m = re.findall(r"-Xmx\S+", e) + heap = m[-1] if m else None + elif e.startswith("JAVA_OPTS="): + for t in e.split("=", 1)[1].split(): + (toks if t.startswith("-D") else dropped).append(t) + if not toks or not heap: + raise SystemExit("native_server_arm: the JVM arm has no -D settings or no -Xmx to carry over") + seen, uniq = set(), [] + for t in reversed(toks): # the JVM takes the last of two tokens for one setting; the native image the first + if t.split("=", 1)[0] not in seen: + seen.add(t.split("=", 1)[0]) + uniq.append(t) + out["server_env"] = [] + out["server_cmd"] = list(cfg.get("server_cmd", [])) + [heap] + list(reversed(uniq)) + out["server_image"] = image + out["server_build"] = "native" + out["native_dropped_flags"] = dropped + return out + + +if _ENGINE_BACKEND == "native": + if not _LOCAL_NATIVE_SERVER_IMAGE: + raise SystemExit("REFUSING: the pinned wheel is the native build, so the served ArcadeDB arms run upstream's native server image, and\n" + "ARCADEDB_NATIVE_SERVER_IMAGE is unset (F5: wheel, JVM server image and native server image are ONE engine commit).") + _n_native = 0 + for _name, _cfg in list(BACKENDS.items()): + if (_AI.served({"backend": _name}) and _cfg.get("topology") == "client_server" + and _cfg.get("server_image") == _LOCAL_SERVER_IMAGE and not _cfg.get("jvm_defaults")): + BACKENDS[_name] = native_server_arm(_cfg, _LOCAL_NATIVE_SERVER_IMAGE) + _n_native += 1 + print(f"[engine] native chain: served ArcadeDB arms on {_LOCAL_NATIVE_SERVER_IMAGE} ({_n_native} arms)") + + if _ENGINE_PIN: for _cfg in BACKENDS.values(): if _cfg.get("image") == engine_pin.LOGICAL_IMAGE: @@ -2097,6 +2190,14 @@ def _cpuset_size(cpuset): return n +def _set_tx_wal_flush(text, value): + """`text` with its -Darcadedb.txWalFlush= token set to `value` (replaced where present, appended otherwise): one token, never two, + because the JVM takes the last and the native image the first.""" + if re.search(r"-Darcadedb\.txWalFlush=\S+", text): + return re.sub(r"-Darcadedb\.txWalFlush=\S+", f"-Darcadedb.txWalFlush={value}", text) + return text + f" -Darcadedb.txWalFlush={value}" + + def durability_server_patch(cfg, cls): """The SERVED half of the durability axis (DECISIONS #90). @@ -2125,8 +2226,12 @@ def durability_server_patch(cfg, cls): # own JVM, appended to JAVA_OPTS. for i, e in enumerate(env): if isinstance(e, str) and e.startswith("JAVA_OPTS="): - env[i] = e + f" -Darcadedb.txWalFlush={bench_common.ARCADE_STRICT_TX_WAL_FLUSH}" + env[i] = _set_tx_wal_flush(e, bench_common.ARCADE_STRICT_TX_WAL_FLUSH) notes.append(f"txWalFlush={bench_common.ARCADE_STRICT_TX_WAL_FLUSH}") + # ArcadeDB served from the native image: the same token, on the command line. + if any(isinstance(c, str) and c.startswith("-Darcadedb.txWalFlush=") for c in cmd): + cmd = [_set_tx_wal_flush(c, bench_common.ARCADE_STRICT_TX_WAL_FLUSH) if isinstance(c, str) else c for c in cmd] + notes.append(f"txWalFlush={bench_common.ARCADE_STRICT_TX_WAL_FLUSH}") # QuestDB: its commit mode is a server setting. if "questdb" in str(cfg.get("server_image", "")): env += ["-e", "QDB_CAIRO_COMMIT_MODE=sync"] @@ -2254,6 +2359,8 @@ def _arcadedb_images(jobs): if str(cfg.get("image") or "").startswith(engine_pin.LOGICAL_IMAGE): emb.add(cfg["image"]) img = str(cfg.get("server_image") or "") + if _LOCAL_NATIVE_SERVER_IMAGE and img == _LOCAL_NATIVE_SERVER_IMAGE: + continue # a scratch image: no jars and no shell to read; its engine commit is read from SELECT version() at every cell start if img.startswith("arcadedata/arcadedb") or (_LOCAL_SERVER_IMAGE and img == _LOCAL_SERVER_IMAGE): srv.add(img) return emb, srv @@ -2762,6 +2869,11 @@ def observe_server(cid): envs = " ".join(json.loads(env)) except Exception: envs = "" + # the native server image takes its settings and heap as the container's command, not its environment + try: + envs += " " + " ".join(json.loads(sh(["docker", "inspect", "-f", "{{json .Config.Cmd}}", cid]) or "null") or []) + except Exception: # noqa: BLE001 + pass # TWO SPELLINGS, because -Xmx is not how every JVM image takes its heap. # This used to match -Xmx only, so ArcadeDB and Elasticsearch (ES_JAVA_OPTS # carries -Xmx) had a witness and NEO4J DID NOT: it takes @@ -2859,6 +2971,78 @@ def server_jvm_readback(cid): return out +_REQ_RE = re.compile(r"-D(arcadedb\.[A-Za-z0-9_.]+)=(\S+)") + + +def requested_settings(text): + """{key: value} of the `-Darcadedb.*` tokens in a launch text (environment and command); the last of two for one key wins. No password.""" + got = {k: v for k, v in _REQ_RE.findall(text or "")} + got.pop("arcadedb.server.rootPassword", None) + return got + + +def settings_problems(got, requested): + """Requested settings the engine LISTS and reports with another value (empty: fine). A key the engine does not list is not a problem here: + `server_settings_unlisted` names it on the row.""" + out = [] + for k, want in sorted((requested or {}).items()): + have = (got or {}).get(k) + if have is not None and str(have) != str(want): + out.append(f"setting {k} was requested as {want} and the engine runs {have}") + return out + + +def engine_commit_problems(got, pinned): + """The server's `SELECT version()` commit against the pin (ARCADEDB_ENGINE_COMMIT); [] when no pin is claimed.""" + if not pinned: + return [] + if not got: + return ["the server's engine commit could not be read from SELECT version()"] + return [] if (got.startswith(pinned[:7]) or pinned.startswith(got[:7])) else [f"the server runs engine commit {got[:9]}, the pin is {pinned[:9]}"] + + +def arcadedb_settings_readback(cid, text): + """What a served ArcadeDB (either build) runs with, asked of the engine over HTTP, before the client starts and outside every timer: + `GET /api/v1/server?mode=default` (server-scoped settings with the value in force), `SELECT FROM schema:database` (database-scoped ones, + txWalFlush among them) and `SELECT version()` (the engine's own build stamp, so the commit is evidence). Returns server_settings (the + requested keys plus STAMPED_SETTING_KEYS), server_settings_requested, server_settings_unlisted, server_engine_version and + server_engine_commit. Never ends a cell; what it could not read is named in server_settings_source.""" + requested = requested_settings(text) + pw = re.search(r"-Darcadedb\.server\.rootPassword=(\S+)", text) + db = re.search(r"-Darcadedb\.server\.defaultDatabases=(\w+)", text) + auth = {"Authorization": "Basic " + base64.b64encode(f"root:{pw.group(1) if pw else 'dbbenchpass'}".encode()).decode()} + ips = (sh(["docker", "inspect", "-f", "{{range .NetworkSettings.Networks}}{{.IPAddress}} {{end}}", cid]) or "").split() + listed, source, version = {}, "the engine could not be asked from the host", None + for ip in ips[:1]: + base = f"http://{ip}:2480/api/v1" + for attempt in range(3): + try: + req = urllib.request.Request(base + "/server?mode=default", headers=auth) + with urllib.request.urlopen(req, timeout=15) as resp: + listed.update({x["key"]: x["value"] for x in json.load(resp)["settings"]}) + for cmd in ("SELECT FROM schema:database", "SELECT version() AS v"): + req = urllib.request.Request(f"{base}/query/{db.group(1) if db else 'bench'}", headers={**auth, "Content-Type": "application/json"}, + data=json.dumps({"language": "sql", "command": cmd}).encode()) + with urllib.request.urlopen(req, timeout=15) as resp: + res = json.load(resp)["result"][0] + if "settings" in res: + listed.update({x["key"]: x["value"] for x in res["settings"]}) + elif "v" in res: + version = res["v"] + source = "read from the engine over HTTP (server?mode=default, schema:database, SELECT version())" + break + except Exception: # noqa: BLE001 - recorded in the source field + time.sleep(1) + keys = sorted(set(requested) | set(STAMPED_SETTING_KEYS)) + m = engine_pin.BUILD_RE.search(str(version or "")) + return {"server_settings": {k: listed.get(k) for k in keys}, + "server_settings_requested": requested, + "server_settings_unlisted": [k for k in keys if k not in listed], + "server_settings_source": source, + "server_engine_version": version, + "server_engine_commit": m.group(1) if m else None} + + def image_defaults_findings(row, server_mem): """What the image-defaults arm must show for the cell to be the one asked for: a heap of 75% of the container limit, generational ZGC, and no -Xms. Returns @@ -3301,7 +3485,7 @@ def run_cell(job, rep, scale, cpuset, tier, net_name): if os.environ.get("BENCH_PG_FIT", "").lower() == "off": # PostgreSQL's own defaults, for a same-run A/B only (PG_FIT_CMD). _fit.update(pg_workers=2, pg_par=8, pg_procs=8, pg_work_mem="4MB") - server_cmd = [c.format(sb=f"{max(1, srv_gb // 4)}GB", + server_cmd = [c.format(heap=heap, sb=f"{max(1, srv_gb // 4)}GB", ecs=f"{max(1, srv_gb * 3 // 4)}GB", mwm=f"{max(1, srv_gb // 2)}GB", **_fit) for c in be.get("server_cmd", [])] @@ -3334,6 +3518,10 @@ def run_cell(job, rep, scale, cpuset, tier, net_name): if be.get("sidecars"): row["sidecars"] = {_s["name"]: {"image": _s["image"], "mem": _s["mem"]} for _s in be["sidecars"]} row.update(be.get("stamp", {})) + if "arcadedb" in job["backend"]: + row["server_build"] = be.get("server_build", "jvm") # the build of the SERVER this cell talks to (#222) + if be.get("native_dropped_flags") is not None: + row["native_dropped_flags"] = " ".join(be["native_dropped_flags"]) server_cid = sh(["docker", "run", "-d", "--network", net_name, *RUNNER_LABELS, "--name", f"srv-{run_id}", @@ -3365,8 +3553,18 @@ def run_cell(job, rep, scale, cpuset, tier, net_name): # THE JVM FLAGS THE SERVED ARCADEDB RUNS, read from the process (CAMPAIGN 7 # row 69): on every served ArcadeDB arm, so the main arms' G1 and half-memory -Xmx # are evidence beside the image-defaults arm's ZGC and 75%. - if "arcadedb" in job["backend"]: + if "arcadedb" in job["backend"] and be.get("server_build") != "native": row.update(server_jvm_readback(server_cid)) + if "arcadedb" in job["backend"]: + # THE SETTINGS IN FORCE, asked of the engine on both servers; a setting written is not a setting in force (#222 (3)). + _rb = arcadedb_settings_readback(server_cid, " ".join( + [str(x) for x in be.get("server_env", [])] + [str(x) for x in server_cmd])) + row.update(_rb) + _bad = settings_problems(_rb.get("server_settings"), _rb.get("server_settings_requested")) + _bad += engine_commit_problems(_rb.get("server_engine_commit"), _LOCAL_ENGINE_COMMIT) + if _bad: + row["error"] = "; ".join(_bad) + "; the cell is not the one we specified" + return row if be.get("jvm_defaults"): row["server_jvm_defaults"] = True _bad = image_defaults_findings(row, server_mem) @@ -3420,6 +3618,8 @@ def run_cell(job, rep, scale, cpuset, tier, net_name): client_caps = ["--memory", str(client_mem), "--memory-swap", str(client_mem)] bench_env = ["-e", f"BENCH_SERVER_HOST=srv-{run_id}", "-e", f"BENCH_SERVER_PORT={be['server_port']}"] + if "arcadedb" in job["backend"]: + bench_env += ["-e", f"BENCH_ARCADEDB_SERVER_BUILD={row['server_build']}"] # where the client reaches a sidecar (Milvus bulk import uploads its files to MinIO) bench_env += [x for _n, _h in _sc.items() for x in ("-e", f"BENCH_SIDECAR_{_n.upper()}={_h}")] else: diff --git a/benchmarks/experiments/smoke_native_server.sh b/benchmarks/experiments/smoke_native_server.sh new file mode 100755 index 00000000000..ab25a0b182b --- /dev/null +++ b/benchmarks/experiments/smoke_native_server.sh @@ -0,0 +1,71 @@ +#!/bin/bash +# Smoke for the native server arm (#222): plumbing only, at the smallest scale. Run from the measurement lane, never from a tool shell on the laptop. +# +# Starts upstream's native-image ArcadeDB server image as the served arm (the runner swaps it in because the pinned wheel is the native build), +# runs ONE served cell of l2 (arcadedb_graph_server) and ONE of docs_oltp (l1 oltp, arcadedb_server), then checks the rows: build, settings +# stamps read back from the engine, engine commit, CPU baseline. Exits non-zero on any miss. The engine commit is READ from SELECT version() and +# REPORTED, not asserted equal to the wheel's: this checks the plumbing, not the pair (the stage pre-flight checks the pair). +# +# usage: smoke_native_server.sh [] [] +set -uo pipefail +HERE="$(cd "$(dirname "$(readlink -f "${BASH_SOURCE[0]}")")" && pwd)" +IMG="${1:-arcadedata/arcadedb:26.11.1-SNAPSHOT-native}" +WHEEL="${2:-/home/tk/build-scratch/native-e353330c5c/dist/arcadedb-26.11.1.dev0-py3-none-manylinux_2_28_x86_64.whl}" +OUT="${SMOKE_OUT:-/home/tk/build-scratch/smoke-native-server}" +TAG="smoke_native_server_$(date +%s)" +[ -f "$WHEEL" ] || { echo "SMOKE FAIL: no native wheel $WHEEL"; exit 1; } +case "$(basename "$WHEEL")" in arcadedb-[0-9]*) ;; *) echo "SMOKE FAIL: $WHEEL is not the native (arcadedb-) wheel"; exit 1 ;; esac +mkdir -p "$OUT" || exit 1 +docker image inspect "$IMG" >/dev/null 2>&1 || docker pull -q "$IMG" >/dev/null || { echo "SMOKE FAIL: cannot pull $IMG"; exit 1; } +# the image answers on its own before the runner starts it (reports the commit; no sha asserted) +bash "$HERE/verify_native_server_image.sh" "$IMG" > "$OUT/verify.txt" 2>&1 || { cat "$OUT/verify.txt"; echo "SMOKE FAIL: the native server image did not answer"; exit 1; } +COMMIT=$(sed -n 's/^NATIVE_SERVER_COMMIT=//p' "$OUT/verify.txt") +[ -n "$COMMIT" ] || { echo "SMOKE FAIL: no commit read from SELECT version()"; exit 1; } +echo "native server commit (reported, not asserted): $COMMIT" +# ARCADEDB_SERVER_IMAGE is the JVM image the F5 rule wants set beside the wheel; the native chain never starts it, so a name is enough here. +export ARCADEDB_WHEEL="$WHEEL" ARCADEDB_SERVER_IMAGE="arcadedb-c25:${COMMIT:0:9}" ARCADEDB_NATIVE_SERVER_IMAGE="$IMG" BENCH_SERVER_BUILD=native +export BENCH_CPUSET="${BENCH_CPUSET:-4,5}" BENCH_INSTRUMENT=2026-11 +cd "$HERE" || exit 1 +rc=0 +for CELL in "l2 arcadedb_graph_server oltp" "l1 arcadedb_server oltp"; do + set -- $CELL + uv run --no-project --python 3.12 --with numpy python runner.py --lanes "$1" --backends "$2" --workloads "$3" --scale tiny --reps 1 --tier sweep \ + --results-file "$TAG.jsonl" > "$OUT/run_$1.log" 2>&1 || { tail -20 "$OUT/run_$1.log"; echo "SMOKE FAIL: runner exited non-zero for $1"; rc=1; } +done +[ "$rc" = 0 ] || exit 1 +RESULTS_FILE="$HERE/results/$TAG.jsonl" EXPECT_COMMIT="$COMMIT" uv run --no-project --python 3.12 python -I - <<'PY' +import json, os, sys +sys.path.insert(0, os.getcwd()) +import arm_identity as AI +rows = [json.loads(l) for l in open(os.environ["RESULTS_FILE"]) if l.strip()] +bad = [] +want = {"l2": "arcadedb_graph_server", "l1": "arcadedb_server"} +for lane, arm in want.items(): + got = [r for r in rows if r.get("lane") == lane and r.get("backend") == arm] + if len(got) != 1: + bad.append(f"{lane}/{arm}: {len(got)} rows, expected 1") + continue + r = got[0] + def need(cond, msg): + if not cond: + bad.append(f"{lane}/{arm}: {msg}") + need(not r.get("error") and r.get("rc") == 0, f"error={r.get('error')!r} rc={r.get('rc')!r}") + need(r.get("engine_backend") == "native" and r.get("server_build") == "native", f"engine_backend={r.get('engine_backend')!r} server_build={r.get('server_build')!r}") + need(AI.build_of(r) == "native" and AI.served(r), "arm_identity does not read it as a native served row") + need(r.get("cpu_baseline", "").startswith("native server image: -march="), f"cpu_baseline={r.get('cpu_baseline')!r}") + s = r.get("server_settings") or {} + for k, v in (("arcadedb.server.mode", "development"), ("arcadedb.server.httpIncomingHost", "0.0.0.0"), + ("arcadedb.server.httpIncomingPort", "2480"), ("arcadedb.txWalFlush", "0")): + need(str(s.get(k)) == v, f"server_settings[{k}]={s.get(k)!r}, expected {v!r} (unlisted: {r.get('server_settings_unlisted')})") + need("read from the engine over HTTP" in str(r.get("server_settings_source")), f"settings source: {r.get('server_settings_source')!r}") + need(str(r.get("server_engine_commit") or "").startswith(os.environ["EXPECT_COMMIT"][:7]), f"server_engine_commit={r.get('server_engine_commit')!r}") + need("UseCompactObjectHeaders" in str(r.get("native_dropped_flags")), f"native_dropped_flags={r.get('native_dropped_flags')!r}") + need("-Xmx" in str(r.get("server_cmd")) and "-Darcadedb.txWalFlush=0" in str(r.get("server_cmd")), f"server_cmd={r.get('server_cmd')!r}") + print(f"{lane}/{arm}: build={r.get('engine_backend')} commit={r.get('server_engine_commit')} heap={r.get('server_heap')} cpu_baseline={r.get('cpu_baseline')!r} settings={s}") +pair = AI.server_settings_problems(rows) +bad += pair +if bad: + print("SMOKE FAIL:\n " + "\n ".join(bad)) + sys.exit(1) +print("SMOKE PASS native server arm") +PY diff --git a/benchmarks/experiments/test_build_pair_page.py b/benchmarks/experiments/test_build_pair_page.py index b082c79d09a..43b870eba11 100644 --- a/benchmarks/experiments/test_build_pair_page.py +++ b/benchmarks/experiments/test_build_pair_page.py @@ -40,12 +40,13 @@ def test_a_freeze_of_old_rows_is_not_split_and_a_native_row_splits_it(): assert AI.builds_in(old) == {"jvm"} assert AI.splits_page(old + [row("native")]) and AI.builds_in(old + [row("native")]) == {"native", "jvm"} assert AI.splits_page([row("native")]) # native only: an unlabelled arm would read as the JVM build - assert not AI.splits_page([row("native", backend="arcadedb_graph_server"), row(backend="qdrant_dense")]) + assert AI.splits_page([row("native", backend="arcadedb_graph_server"), row(backend="qdrant_dense")]) # a native server row splits it too + assert not AI.splits_page([row(backend="arcadedb_graph_server"), row(backend="qdrant_dense")]) -def test_an_empty_engine_backend_from_the_csv_is_the_jvm_build_for_in_process_arms_only(): +def test_an_empty_engine_backend_from_the_csv_is_the_jvm_build_for_in_process_and_served_arms_only(): assert AI.build_of(row("")) == "jvm" and AI.build_of(row(None)) == "jvm" - assert AI.build_of(row("", backend="arcadedb_graph_server")) is None + assert AI.build_of(row("", backend="arcadedb_graph_server")) == "jvm" assert AI.build_of(row("", backend="qdrant_dense")) is None @@ -129,7 +130,8 @@ def test_the_gate_fails_an_in_process_arm_that_names_no_build_on_a_page_that_doe assert PC._check_build_pair(payload([NATIVE, JVM, entry("ArcadeDB (embedded, fp32)", None)]), []) == 1 assert "NOBUILD" in capsys.readouterr().out served = entry("ArcadeDB (server)", None, key="arcadedb_graph_server") - assert PC._check_build_pair(payload([NATIVE, JVM, served]), []) == 0 # one served arm, no build + assert PC._check_build_pair(payload([NATIVE, JVM, dict(served, backend="ArcadeDB (server, native build)", build="native"), + dict(served, backend="ArcadeDB (server, JVM build)", build="jvm")]), []) == 0 # server arms stand per build too # ------------------------------------------------------------------ the exporter, on the publish fixture @@ -150,11 +152,13 @@ def _run(script, results): def _with_native(results, keep=lambda r: True, commit=None): """Rewrite the freeze: every in-process ArcadeDB row gains a native twin (for the rows `keep` accepts).""" rows = list(csv.DictReader((results / "runs_paper.csv").open())) - fields = list(rows[0]) + (["engine_backend"] if "engine_backend" not in rows[0] else []) + fields = list(rows[0]) + [f for f in ("engine_backend", "server_settings", "cpu_baseline") if f not in rows[0]] twins = [] for r in rows: - if AI.in_process(r) and keep(r): - t = dict(r, engine_backend="native") + if AI.built(r) and keep(r): + t = dict(r, engine_backend="native", cpu_baseline=AI.cpu_baseline("native", AI.served(r))) + if AI.served(r): # a native server row carries the settings its engine reported at start, equal to the JVM server's + t["server_settings"] = json.dumps({k: "x" for k in AI.SERVER_SETTING_KEYS}) if commit: t["engine_commit"] = commit twins.append(t) @@ -240,18 +244,30 @@ def test_every_in_process_arm_of_a_lane_table_stands_twice_native_first(pages): "ArcadeDB (embedded, native build)", "ArcadeDB (embedded, JVM build)"] -def test_the_served_arm_stays_one_row_and_both_builds_name_one_engine(pages): +def test_the_served_arm_stands_once_per_build_and_both_builds_name_one_engine(pages): both = pages[1] for tid in ("docs_oltp", "l2"): t = _t(both, tid) - served = [e for e in t["entries"] if e.get("is_arcadedb") and e.get("deployment") == "server"] - assert served and all(e["build"] is None and "build" not in e["backend"] for e in served) + served = [e for e in t["entries"] if e.get("is_arcadedb") and e.get("deployment") == "server" and "imgdefaults" not in e["backend_key"]] + assert {e["backend"] for e in served} >= {"ArcadeDB (server, native build)", "ArcadeDB (server, JVM build)"} + assert {e["build"] for e in served} == {"native", "jvm"} + assert all(e["build"] is None for e in t["entries"] if "imgdefaults" in e.get("backend_key", "")) names = {e["version_name"] for e in _inproc(t)} assert len(names) == 1, names rows = list(csv.DictReader((pages[3] / "both" / "runs_paper.csv").open())) assert PC._check_build_pair({"tables": [t for t in both["tables"] if t["id"] != "l3d"]}, rows) == 0 +def test_the_server_tables_disclose_the_cpu_baselines_read_from_the_rows(pages): + both = pages[1] + t = _t(both, "docs_oltp") + note = [c for c in t["conditions"] if "official ArcadeDB server images" in c] + assert len(note) == 1 and "native server image: -march=x86-64-v3" in note[0] and "native wheel: -march=compatibility" in note[0] + assert "launcher" not in note[0] and "\u2014" not in note[0] + old = _t(pages[0], "docs_oltp") + assert not [c for c in old.get("conditions", []) if "official ArcadeDB server images" in c] # a freeze of old rows says nothing new + + def test_a_stamped_overlay_stands_per_build_and_the_gate_passes_the_dense_table(pages): """An overlay belongs to the build that produced it: the native files (engine_backend=native on every pass) and the JVM files (no field: the JVM build) give each in-process dense arm two entries, and l3d no longer fails the build gate.""" @@ -359,7 +375,7 @@ def test_a_partial_native_overlay_is_refused_and_a_complete_or_absent_one_is_not assert "partial" in str(e.value) -def test_the_pinned_sparse_path_validates_each_build_and_takes_the_jvm_one_unless_told(monkeypatch): +def test_the_pinned_sparse_path_validates_each_build_and_has_no_single_build_default(monkeypatch): import make_paper_tables as M def rows(build, n=5): @@ -368,13 +384,16 @@ def rows(build, n=5): monkeypatch.setenv("BENCH_ENGINE_COMMIT", "abc123def0") monkeypatch.delenv(AI.ENV, raising=False) monkeypatch.setattr(M, "load_canonical", lambda *a, **k: rows("native") + rows("jvm")) - got = M._pinned_sparse_rows() + assert M._pinned_sparse_rows() is None # no default build: the caller names one + got = M._pinned_sparse_rows(build="jvm") assert got and {AI.build_of(r) for rs in got.values() for r in rs} == {"jvm"} + both = M._pinned_sparse_rows(build="all") + assert {AI.build_of(r) for rs in both.values() for r in rs} == {"native", "jvm"} monkeypatch.setenv(AI.ENV, "native") assert {AI.build_of(r) for rs in M._pinned_sparse_rows().values() for r in rs} == {"native"} monkeypatch.delenv(AI.ENV) monkeypatch.setattr(M, "load_canonical", lambda *a, **k: rows("native", 3) + rows("jvm")) - assert M._pinned_sparse_rows() is None # a short native build is not published beside a full JVM one + assert M._pinned_sparse_rows(build="all") is None # a short native build is not published beside a full JVM one def test_neither_build_is_a_median_over_both(pages): diff --git a/benchmarks/experiments/test_chain_served_arms.py b/benchmarks/experiments/test_chain_served_arms.py index 4ab93b0c0b3..4e745ea756d 100644 --- a/benchmarks/experiments/test_chain_served_arms.py +++ b/benchmarks/experiments/test_chain_served_arms.py @@ -1,7 +1,5 @@ -"""The served ArcadeDB cells run in ONE chain: the JVM chain. The native chain skips them, and a check fails when both have them. - -The server is the ARCADEDB_SERVER_IMAGE container, the same Java server whichever wheel is installed; a second run would put two sets of -rows under one key. Run with `python -m pytest test_chain_served_arms.py -q` from this directory. Nothing is run: only drafts are written. +"""Each chain runs exactly the served ArcadeDB arm of its own build, each once: the native chain on upstream's native server image, the JVM +chain on the Docker JVM server (#222). Run with `python -m pytest test_chain_served_arms.py -q` from this directory. Nothing is run: only drafts are written. """ import re import sys @@ -48,39 +46,77 @@ def test_the_served_arcadedb_arms_are_the_client_server_ones(): assert "arcadedb_graph_embedded" not in served -def test_the_native_chain_runs_no_served_arcadedb_cell_and_keeps_the_embedded_ones(chains): +def test_the_native_chain_runs_the_served_arms_of_its_build_and_keeps_the_embedded_ones(chains): native, jvm, _old = chains served = set(G.served_arcadedb_arms()) - assert _arms(native) and not (_arms(native) & served) + per_build = served - set(G.build_less_served_arms()) + assert "arcadedb_server" in per_build and "arcadedb_graph_server" in per_build + ran = _arms(jvm) & per_build # the arms this profile runs at all + assert {"arcadedb_server", "arcadedb_graph_server"} <= ran + assert _arms(native) & per_build == ran # each chain runs the served arm of its own build, the same ones assert any(b.startswith("arcadedb") for b in _arms(native)) - assert _arms(jvm) & served # the JVM chain is the one that runs the server + # the arms with no native counterpart (the JVM image-defaults arm; e4 mixes the wheel with the JVM image) run in the JVM chain only + assert set(G.build_less_served_arms()) >= {"arcadedb_imgdefaults_server", "arcadedb_e4"} + assert not (_arms(native) & set(G.build_less_served_arms())) + assert set(G.build_less_served_arms()) & _arms(jvm) def test_the_jvm_chain_and_a_chain_of_an_old_wheel_are_unchanged_by_the_native_rule(chains): _native, jvm, old = chains assert _arms(jvm) == _arms(old) # arcadedb_embedded wheels are the JVM build + text = lambda d: "".join(f.read_text() for f in d.glob("*.sh")) # noqa: E731 + assert "ARCADEDB_NATIVE_SERVER_IMAGE" not in text(jvm) and "ARCADEDB_NATIVE_SERVER_IMAGE" not in text(old) + assert "BENCH_SERVER_BUILD" not in text(jvm) + + +def test_the_native_chain_pins_upstreams_native_server_image_and_checks_its_commit(chains): + native, _jvm, _old = chains + text = "".join(f.read_text() for f in native.glob("*.sh")) + assert 'export ARCADEDB_NATIVE_SERVER_IMAGE="arcadedata/arcadedb:26.10.2-native" BENCH_SERVER_BUILD="native"' in text \ + or re.search(r'export ARCADEDB_NATIVE_SERVER_IMAGE="arcadedata/arcadedb:[\d.]+-native" BENCH_SERVER_BUILD="native"', text) + assert 'verify_native_server_image.sh "$ARCADEDB_NATIVE_SERVER_IMAGE" "$SHA"' in text + assert "ABORT: native server image is not at the pin" in text -def test_the_two_chains_of_one_release_pass_the_overlap_check(chains): +def test_the_two_chains_of_one_release_pass_the_served_arm_check(chains): native, jvm, _old = chains - assert G.chain_overlap_problems(str(native), str(jvm)) == [] + assert G.chain_served_problems(str(native), str(jvm)) == [] -def test_the_overlap_check_fails_when_a_served_arm_is_in_both_chains(chains): +def test_the_served_arm_check_fails_for_two_chains_of_one_build(chains): _native, jvm, old = chains - problems = G.chain_overlap_problems(str(jvm), str(old)) - assert problems and all("one image" in p for p in problems) - assert any("arcadedb_graph_server" in p for p in problems) + problems = G.chain_served_problems(str(jvm), str(old)) + assert any("same build" in p for p in problems) + + +def test_the_served_arm_check_fails_when_a_chain_lacks_a_served_arm(chains, tmp_path): + native, jvm, _old = chains + cut = tmp_path / "native_cut" + cut.mkdir() + victim = next(f for f in sorted(native.glob("*.sh")) if G.served_arcadedb_in(f.read_text())) + for f in native.glob("*.sh"): + if f != victim: + (cut / f.name).write_text(f.read_text()) + problems = G.chain_served_problems(str(cut), str(jvm)) + assert any("each chain runs it once" in p for p in problems) -def test_a_native_draft_that_runs_a_served_arm_fails_the_lint(chains): +def test_a_chain_that_runs_the_other_builds_server_arm_fails_the_lint(chains): native, jvm, _old = chains - served_stage = next(f for f in sorted(jvm.glob("*.sh")) if G.served_arcadedb_in(f.read_text())) - forged = served_stage.read_text().replace("dist/" + JVM_WHEEL, "dist/" + NATIVE_WHEEL) - assert "dist/" + NATIVE_WHEEL in forged - assert any("native-build chain runs served ArcadeDB arms" in p for p in G.lint_draft(forged, "qN01.sh")) - clean = next(f for f in sorted(native.glob("*.sh")) if 'BACKENDS="' in f.read_text()) - assert not [p for p in G.lint_draft(clean.read_text(), clean.name) if "native-build chain" in p] + nat = next(f for f in sorted(native.glob("*.sh")) if G.served_arcadedb_in(f.read_text())) + jv = next(f for f in sorted(jvm.glob("*.sh")) if G.served_arcadedb_in(f.read_text())) + # a native chain naming the JVM server build + forged = nat.read_text().replace('BENCH_SERVER_BUILD="native"', 'BENCH_SERVER_BUILD="jvm"') + assert any("native-build chain runs the jvm build's server arm" in p for p in G.lint_draft(forged, nat.name)) + # a JVM chain naming the native one + forged = jv.read_text().replace("export ARCADEDB_WHEEL=", 'export BENCH_SERVER_BUILD="native"\nexport ARCADEDB_WHEEL=') + assert any("JVM-build chain runs the native build's server arm" in p for p in G.lint_draft(forged, jv.name)) + # a native chain with no pin or no check + missing = nat.read_text().replace('export ARCADEDB_NATIVE_SERVER_IMAGE=', 'export SOMETHING_ELSE=') + assert any("no the native server image pin" in p for p in G.lint_draft(missing, nat.name)) + clean = G.lint_draft(nat.read_text(), nat.name) + assert not [p for p in clean if "server" in p and "native" in p], clean + assert not [p for p in G.lint_draft(jv.read_text(), jv.name) if "server arm" in p] def test_the_native_chain_names_its_overlay_files_by_build(chains): diff --git a/benchmarks/experiments/test_native_jvm_arms.py b/benchmarks/experiments/test_native_jvm_arms.py index f827572541c..db8d217b591 100644 --- a/benchmarks/experiments/test_native_jvm_arms.py +++ b/benchmarks/experiments/test_native_jvm_arms.py @@ -30,13 +30,15 @@ def row(build="__absent__", backend="arcadedb_sparse_embedded", rep=1, ts="2026- # ------------------------------------------------------------------ arm_identity -def test_build_of_old_rows_are_the_jvm_build_and_served_arms_and_comparators_have_none(): +def test_build_of_old_rows_are_the_jvm_build_for_in_process_and_served_arms_and_comparators_have_none(): assert AI.build_of(row()) == "jvm" # before the field existed assert AI.build_of(row("native")) == "native" and AI.build_of(row("jvm")) == "jvm" - assert AI.build_of(row(None, backend="arcadedb_sparse_server")) is None # served: null on purpose - assert AI.build_of(row(backend="arcadedb_sparse_server")) is None # an older served row is not "jvm" either + assert AI.build_of(row(None, backend="arcadedb_sparse_server")) == "jvm" # served rows carry the server's build since #222; old ones = JVM + assert AI.build_of(row(backend="arcadedb_sparse_server")) == "jvm" # an older served row is the JVM server, the only one there was + assert AI.build_of(row("native", backend="arcadedb_sparse_server")) == "native" assert AI.build_of(row(None, backend="qdrant_sparse")) is None assert AI.build_of(row(backend="arcadedb_http_client")) is None + assert AI.build_of(row(backend="arcadedb_imgdefaults_server")) is None # the JVM image-defaults arm has no native counterpart def test_the_same_release_on_two_builds_is_two_identities(): @@ -51,7 +53,7 @@ def test_both_builds_stay_unless_one_is_chosen(): assert AI.select_build(rows, None) == rows nat = AI.select_build(rows, "native") assert [AI.build_of(r) for r in nat if AI.in_process(r)] == ["native"] - assert len(nat) == 3 # comparators and served arms stay + assert len(nat) == 2 # comparators stay; the served arm of the other build leaves with it assert [r["backend"] for r in AI.select_build(rows, "jvm")].count("arcadedb_sparse_embedded") == 1 diff --git a/benchmarks/experiments/test_native_server_arm.py b/benchmarks/experiments/test_native_server_arm.py new file mode 100644 index 00000000000..0365f24f2fc --- /dev/null +++ b/benchmarks/experiments/test_native_server_arm.py @@ -0,0 +1,152 @@ +"""The native server arm (#222): upstream's native-image server next to the Docker JVM server, same client, same explicit settings. + +Nothing is started here: only the launch configuration, the identity rules and the read-back parsing are exercised. +Run with `python -m pytest test_native_server_arm.py -q` from this directory. +""" +import json +import os +import sys +from pathlib import Path + +import pytest + +sys.path.insert(0, str(Path(__file__).resolve().parent)) +os.environ.setdefault("BENCH_INSTRUMENT", "2026-11") + +import arm_identity as AI # noqa: E402 +import runner as R # noqa: E402 + +V = "26.11.1.dev0" +COMMIT = "e353330c5c" + "0" * 30 + + +def row(build="__absent__", backend="arcadedb_server", **kw): + r = {"lane": "l1", "scale": "small", "workload": "oltp", "backend": backend, "engine_version": V, "engine_commit": COMMIT, + "durability_class": "relaxed"} + if build != "__absent__": + r["engine_backend"] = build + r.update(kw) + return r + + +# ---------------------------------------------------------------- the launch configuration +def test_the_jvm_arm_states_the_settings_the_image_used_to_default(): + env = " ".join(x for x in R.BACKENDS["arcadedb_server"]["server_env"] if isinstance(x, str)) + for tok in R.EXPLICIT_SERVER_SETTINGS: + assert f"-D{tok}" in env + assert env.count("-Darcadedb.txWalFlush=") == 1 + for arm in ("arcadedb_graph_server", "arcadedb_sparse_server", "arcadedb_e2_server"): + assert "-Darcadedb.server.mode=development" in " ".join(R.BACKENDS[arm]["server_env"]) + + +def test_the_strict_class_replaces_the_flush_token_instead_of_adding_a_second(): + cfg, note = R.durability_server_patch(R.BACKENDS["arcadedb_server"], "strict") + env = " ".join(x for x in cfg["server_env"] if isinstance(x, str)) + assert env.count("-Darcadedb.txWalFlush=") == 1 and "-Darcadedb.txWalFlush=1" in env and "txWalFlush=1" in note + relaxed, _ = R.durability_server_patch(R.BACKENDS["arcadedb_server"], "relaxed") + assert "-Darcadedb.txWalFlush=0" in " ".join(x for x in relaxed["server_env"] if isinstance(x, str)) + + +def test_the_native_arm_is_derived_from_the_jvm_arms_own_settings(): + jvm = R.BACKENDS["arcadedb_graph_server"] + nat = R.native_server_arm(jvm, "arcadedata/arcadedb:26.11.1-native") + cmd = nat["server_cmd"] + assert nat["server_image"].endswith("-native") and nat["server_build"] == "native" and nat["server_env"] == [] + assert cmd[0] == "-Xmx{heap}" # the heap the JVM arm gets from ARCADEDB_OPTS_MEMORY + jvm_d = {t for e in jvm["server_env"] if isinstance(e, str) and e.startswith("JAVA_OPTS=") for t in e.split("=", 1)[1].split() if t.startswith("-D")} + assert {t for t in cmd if t.startswith("-D")} == jvm_d # the same -D list, no more, no fewer + assert any(t.startswith("-Darcadedb.server.httpBodyContentMaxSize=") for t in cmd) # the graph arm's /batch limit is carried over + assert "-XX:+UseCompactObjectHeaders" in nat["native_dropped_flags"] and not any(t.startswith("-XX") for t in cmd) + assert jvm["server_image"] != nat["server_image"] and "server_build" not in jvm # the JVM arm is untouched + + +def test_the_strict_patch_reaches_the_native_command_line_too(): + nat = R.native_server_arm(R.BACKENDS["arcadedb_server"], "arcadedata/arcadedb:26.11.1-native") + cfg, note = R.durability_server_patch(nat, "strict") + assert [t for t in cfg["server_cmd"] if t.startswith("-Darcadedb.txWalFlush=")] == ["-Darcadedb.txWalFlush=1"] and "txWalFlush=1" in note + + +def test_a_native_arm_needs_a_heap_and_settings_to_carry(): + with pytest.raises(SystemExit): + R.native_server_arm({"server_env": ["-e", "JAVA_OPTS=-XX:+UseZGC"], "server_cmd": []}, "x") + + +# ---------------------------------------------------------------- read-back parsing and refusals +def test_requested_settings_are_read_from_env_and_command_without_the_password(): + text = "ARCADEDB_OPTS_MEMORY=-Xmx4g JAVA_OPTS=-Darcadedb.server.rootPassword=pw -Darcadedb.txWalFlush=0 -Darcadedb.txWalFlush=1" + assert R.requested_settings(text) == {"arcadedb.txWalFlush": "1"} + assert R.requested_settings("-Xmx4g -Darcadedb.server.mode=development") == {"arcadedb.server.mode": "development"} + + +def test_a_setting_the_engine_runs_otherwise_refuses_the_cell_and_an_unlisted_one_does_not(): + want = {"arcadedb.txWalFlush": "1", "arcadedb.server.mode": "development"} + assert R.settings_problems({"arcadedb.txWalFlush": "0", "arcadedb.server.mode": "development"}, want) == [ + "setting arcadedb.txWalFlush was requested as 1 and the engine runs 0"] + assert R.settings_problems({"arcadedb.txWalFlush": "1", "arcadedb.server.mode": "development"}, want) == [] + assert R.settings_problems({"arcadedb.txWalFlush": "1", "arcadedb.server.mode": None}, want) == [] + + +def test_the_engine_commit_must_match_the_pin_when_one_is_claimed(): + assert R.engine_commit_problems(COMMIT[:9], COMMIT) == [] and R.engine_commit_problems(COMMIT, COMMIT) == [] + assert R.engine_commit_problems("deadbeef0", COMMIT) and R.engine_commit_problems(None, COMMIT) + assert R.engine_commit_problems("deadbeef0", "") == [] + + +def test_a_bench_server_build_that_is_not_the_wheels_is_refused_at_import(): + import subprocess + env = dict(os.environ, BENCH_SERVER_BUILD="native", BENCH_INSTRUMENT="2026-11") + for k in ("ARCADEDB_WHEEL", "ARCADEDB_SERVER_IMAGE", "ARCADEDB_NATIVE_SERVER_IMAGE", "ARCADEDB_ENGINE_COMMIT"): + env.pop(k, None) + r = subprocess.run([sys.executable, "-c", "import runner"], cwd=Path(__file__).parent, env=env, capture_output=True, text=True) + assert r.returncode != 0 and "BENCH_SERVER_BUILD=native" in (r.stderr + r.stdout) + + +# ---------------------------------------------------------------- identity, labels, gates +def test_a_served_row_carries_the_server_build_and_an_old_one_is_jvm(): + assert AI.served(row()) and not AI.served(row(backend="arcadedb_imgdefaults_server")) and not AI.served(row(backend="arcadedb_sparse_embedded")) + assert AI.build_of(row()) == "jvm" and AI.build_of(row("native")) == "native" + stamped = AI.stamp_build(row(server_build="native", engine_backend=None), "native") + assert stamped["engine_backend"] == "native" + # a killed served cell is stamped by the SERVER's build, not the wheel's + assert AI.stamp_build(row(server_build="jvm"), "native")["engine_backend"] == "jvm" + + +def test_the_four_arms_are_labelled_by_mode_and_build(): + assert AI.build_label("ArcadeDB (server)", "native") == "ArcadeDB (server, native build)" + assert AI.build_label("ArcadeDB (server)", "jvm") == "ArcadeDB (server, JVM build)" + assert AI.build_label("ArcadeDB (embedded)", "native") == "ArcadeDB (embedded, native build)" + assert AI.strip_build("ArcadeDB (server, JVM build)") == "ArcadeDB (server)" + + +def test_the_cpu_baseline_of_each_artifact_is_stamped_and_not_changed(): + assert AI.stamp_build(row("native", backend="arcadedb_sparse_embedded"), "native")["cpu_baseline"] == "native wheel: -march=compatibility" + assert AI.stamp_build(row("native"), "native")["cpu_baseline"] == "native server image: -march=x86-64-v3" + assert AI.stamp_build(row("jvm"), "jvm")["cpu_baseline"].startswith("JVM") + assert AI.cpu_baseline("native", True, {"ARCADEDB_NATIVE_SERVER_MARCH": "x86-64-v2"}).endswith("-march=x86-64-v2") + + +SETTINGS = {"arcadedb.server.mode": "development", "arcadedb.server.httpIncomingHost": "0.0.0.0", + "arcadedb.server.httpIncomingPort": "2480", "arcadedb.txWalFlush": "0"} + + +def test_the_pair_gate_refuses_servers_whose_settings_differ_and_a_native_row_with_no_read_back(): + a, b = row("native", server_settings=dict(SETTINGS)), row("jvm", server_settings=dict(SETTINGS)) + assert AI.pair_problems([a, b]) == [] + b["server_settings"] = dict(SETTINGS, **{"arcadedb.txWalFlush": "1"}) + assert any("arcadedb.txWalFlush" in p and "native server" in p for p in AI.pair_problems([a, b])) + assert any("no server_settings" in p for p in AI.pair_problems([row("native"), row("jvm", server_settings=dict(SETTINGS))])) + # the rows read back from the CSV hold the dict as JSON text + assert AI.pair_problems([row("native", server_settings=json.dumps(SETTINGS)), row("jvm", server_settings=json.dumps(SETTINGS))]) == [] + # different durability classes are different cells + c = row("jvm", server_settings=dict(SETTINGS, **{"arcadedb.txWalFlush": "1"}), durability_class="strict") + assert AI.pair_problems([a, c]) == [] + + +def test_the_pair_gate_covers_the_servers_engine_commit(): + a, b = row("native", server_settings=SETTINGS), row("jvm", server_settings=SETTINGS, engine_commit="f" * 40) + assert any("engine_commit" in p for p in AI.pair_problems([a, b])) + + +def test_a_one_build_freeze_drops_the_other_builds_server_rows(): + rows = [row("native"), row("jvm"), row(backend="duckdb")] + assert [AI.build_of(r) for r in AI.select_build(rows, "native")] == ["native", None] diff --git a/benchmarks/experiments/verify_native_server_image.sh b/benchmarks/experiments/verify_native_server_image.sh new file mode 100755 index 00000000000..13b32bd02ec --- /dev/null +++ b/benchmarks/experiments/verify_native_server_image.sh @@ -0,0 +1,46 @@ +#!/bin/bash +# Verify upstream's native-image ArcadeDB server: its engine commit (SELECT version()) is the pin (#222). +# +# usage: verify_native_server_image.sh [] +# +# The native server image (arcadedata/arcadedb:-native, built by upstream's native-image.yml from native/pom.xml) is a scratch image: +# no shell, no jars, nothing to `docker exec`. Its commit is therefore read the only way it can be: start it as a server on a loopback port with +# the same settings the benchmark gives it, ask `SELECT version()` over HTTP, stop it. Prints one line, `NATIVE_SERVER_COMMIT=`, and +# exits non-zero when the sha is given and differs, or when the server does not answer. With no sha it only reports (the smoke uses this: +# it checks plumbing, not the pair). +# +# FOR A DEV-COMMIT PAIR the image must be built at the pair's commit: dispatch upstream's native-image.yml with `ref=` (and +# `publish_nightly` is NOT needed: build it locally with native/scripts/build-native-docker.sh at that checkout), tag the result +# `arcadedb-native-c25:`, and pass that tag as ARCADEDB_NATIVE_SERVER_IMAGE. For a RELEASE pass it is the published +# `arcadedata/arcadedb:-native` tag, pulled by the pair stage. Nothing here builds an image. +set -uo pipefail +IMG="${1:?usage: verify_native_server_image.sh []}" +SHA="${2:-}" +PW=dbbenchpass +fail() { echo "NATIVE SERVER UNVERIFIED: $*"; exit 1; } +command -v docker >/dev/null || fail "no docker" +command -v curl >/dev/null || fail "no curl" +docker image inspect "$IMG" >/dev/null 2>&1 || fail "no image $IMG" +NAME="nsv-$$" +CID=$(docker run -d --rm --name "$NAME" -p 127.0.0.1::2480 "$IMG" -Xmx2g -Darcadedb.server.rootPassword=$PW \ + -Darcadedb.server.defaultDatabases='bench[root]' -Darcadedb.server.mode=development -Darcadedb.server.httpIncomingHost=0.0.0.0 \ + -Darcadedb.server.httpIncomingPort=2480) || fail "docker run" +trap 'docker rm -f "$NAME" >/dev/null 2>&1' EXIT +PORT=$(docker port "$NAME" 2480/tcp | head -1 | sed 's/.*://') +[ -n "$PORT" ] || fail "no mapped port" +up=0 +for _ in $(seq 1 90); do + curl -sf -o /dev/null "http://127.0.0.1:$PORT/api/v1/ready" && { up=1; break; } + sleep 1 +done +[ "$up" = 1 ] || fail "the server did not answer /api/v1/ready in 90 s" +V=$(curl -sf -u "root:$PW" -H 'Content-Type: application/json' -d '{"language":"sql","command":"SELECT version() AS v"}' \ + "http://127.0.0.1:$PORT/api/v1/query/bench" | python3 -c 'import sys,json; print(json.load(sys.stdin)["result"][0]["v"])') || fail "SELECT version() failed" +GOT=$(printf '%s' "$V" | sed -n 's/.*(build \([0-9a-f]\{7,40\}\)\/.*/\1/p') +[ -n "$GOT" ] || fail "no build sha in '$V'" +echo "NATIVE_SERVER_VERSION=$V" +echo "NATIVE_SERVER_COMMIT=$GOT" +if [ -n "$SHA" ]; then + case "$SHA" in "$GOT"*) ;; *) case "$GOT" in "${SHA:0:7}"*) ;; *) fail "the native server runs ${GOT:0:9}, expected ${SHA:0:9}" ;; esac ;; esac +fi +exit 0 From e241301c8aac9054863d74edb1e3ab292cc5f3bd Mon Sep 17 00:00:00 2001 From: Taewoon Kim Date: Fri, 30 Oct 2026 11:02:20 +0900 Subject: [PATCH 08/23] benchmarks: read each server's own start line (mode, CPUs, MAXRAM) and gate on it; page cache compared; smoke at 2g Both builds log `ArcadeDB Server started in '' mode (CPUs= MAXRAM=)` (ArcadeDBServer.java), from Runtime.availableProcessors() and Runtime.maxMemory(). That is the witness that the requested -Xmx and the container's cpuset reached the engine, on the native image as on the JVM. - runner: parse_server_started / size_to_bytes / server_log_problems / arcadedb_server_log_readback; run_cell stamps server_mode, server_cpus, server_max_heap(_bytes) on the row and fails the cell when MAXRAM is not within 5% of the requested -Xmx (not asked for the image-defaults arm), when CPUs differs from the cpuset size, when the mode differs from the requested one, or when the line is missing. - arm_identity.pair_problems: a native server row must carry the three fields; the builds of one cell must agree on mode and CPUs and on MAXRAM within 5%; arcadedb.maxPageRAM joins SERVER_SETTING_KEYS (engine default on both, so a different effective page cache between builds refuses the page). - smoke_native_server.sh: --mem 3g --heap 2g (the runner's existing overrides, own results file) so it fits a 4 GB slot and checks the -Xmx path at 2g, asserts mode, CPUs and MAXRAM for the native server, logs into its own OUT dir (no mkdir or redirect in the queue line); no system python3 in either script. Tests: 3 new in test_native_server_arm.py fail on the previous code (the page-fixture tests error there, they use the new fields); suite 1134 passed. Co-Authored-By: Claude Opus 5.5 --- benchmarks/experiments/arm_identity.py | 17 +++++- benchmarks/experiments/runner.py | 56 +++++++++++++++++++ benchmarks/experiments/smoke_native_server.sh | 19 ++++++- .../experiments/test_build_pair_page.py | 3 +- .../experiments/test_native_server_arm.py | 46 ++++++++++++++- .../experiments/verify_native_server_image.sh | 2 +- 6 files changed, 133 insertions(+), 10 deletions(-) diff --git a/benchmarks/experiments/arm_identity.py b/benchmarks/experiments/arm_identity.py index ec58c8d85c7..b6f7370467c 100644 --- a/benchmarks/experiments/arm_identity.py +++ b/benchmarks/experiments/arm_identity.py @@ -200,7 +200,11 @@ def pair_problems(rows): # the same explicit settings and the runner reads each back from the engine at start (`server_settings`, key -> value in force). The keys # below must be present on every served ArcadeDB row and equal between the two builds of one cell (same arm, scale, durability class); # other keys the runner requested (the HTTP body limit, the query cap) are compared when both builds list them. -SERVER_SETTING_KEYS = ("arcadedb.server.mode", "arcadedb.server.httpIncomingHost", "arcadedb.server.httpIncomingPort", "arcadedb.txWalFlush") +SERVER_SETTING_KEYS = ("arcadedb.server.mode", "arcadedb.server.httpIncomingHost", "arcadedb.server.httpIncomingPort", "arcadedb.txWalFlush", + "arcadedb.maxPageRAM") # the page cache stays at the engine default on both servers; a different effective value refuses the page +# What each server's own start line reports (ArcadeDBServer.java: mode, CPUs, MAXRAM); a native row must carry all three, and the builds must agree. +SERVER_LOG_FIELDS = ("server_mode", "server_cpus", "server_max_heap_bytes") +MAX_HEAP_TOLERANCE = 0.05 def server_settings_problems(rows): @@ -223,15 +227,24 @@ def server_settings_problems(rows): sc, pin = str(r.get("server_engine_commit") or ""), str(r.get("engine_commit") or "") if sc and pin and not (sc.startswith(pin[:7]) or pin.startswith(sc[:7])): out.append(f"{r.get('backend')}: the {build_of(r)} server reports engine commit {sc[:9]} and the row names {pin[:9]}") + if build_of(r) == "native" and any(r.get(f) in (None, "") for f in SERVER_LOG_FIELDS): + out.append(f"{r.get('backend')}: a native server row has no start-line read-back ({', '.join(SERVER_LOG_FIELDS)})") if isinstance(got, dict): key = (r.get("backend"), str(r.get("scale")), str(r.get("durability_class") or "")) - cells.setdefault(key, {}).setdefault(build_of(r), []).append(got) + cells.setdefault(key, {}).setdefault(build_of(r), []).append(dict(got, **{f: r.get(f) for f in SERVER_LOG_FIELDS if r.get(f) not in (None, "")})) for (arm, scale, cls), by in sorted(cells.items(), key=str): if len(by) < 2: continue a, b = (by[x][0] for x in BUILDS) keys = set(SERVER_SETTING_KEYS) | (set(a) & set(b)) + for f in SERVER_LOG_FIELDS: # only when both builds report it (an old JVM row has none) + if a.get(f) is not None and b.get(f) is not None: + keys.add(f) for k in sorted(keys): + if k == "server_max_heap_bytes" and a.get(k) is not None and b.get(k) is not None: + if abs(float(a[k]) - float(b[k])) > MAX_HEAP_TOLERANCE * max(float(a[k]), float(b[k])): + out.append(f"{arm} at {scale} ({cls or 'any class'}): the native server reports MAXRAM {a[k]} bytes and the JVM server {b[k]}") + continue if str(a.get(k)) != str(b.get(k)): out.append(f"{arm} at {scale} ({cls or 'any class'}): server setting {k} is {a.get(k)!r} on the native server and " f"{b.get(k)!r} on the JVM server") diff --git a/benchmarks/experiments/runner.py b/benchmarks/experiments/runner.py index c3652ef9a52..2511ba83154 100644 --- a/benchmarks/experiments/runner.py +++ b/benchmarks/experiments/runner.py @@ -3001,6 +3001,58 @@ def engine_commit_problems(got, pinned): return [] if (got.startswith(pinned[:7]) or pinned.startswith(got[:7])) else [f"the server runs engine commit {got[:9]}, the pin is {pinned[:9]}"] +# THE SERVER'S OWN START LINE, on both builds (ArcadeDBServer.java): `ArcadeDB Server started in 'development' mode (CPUs=4 MAXRAM=2.00GB)`, with +# Runtime.availableProcessors() and Runtime.maxMemory(). It is the witness that the requested -Xmx and the container's cpuset reached the engine. +_STARTED_RE = re.compile(r"ArcadeDB Server started in '([^']*)' mode \(CPUs=(\d+) MAXRAM=([^)]*)\)") +_SIZE_RE = re.compile(r"\s*([\d.,]+)\s*([KMGT]?)i?B?\s*", re.I) +HEAP_TOLERANCE = 0.05 # the size string is rounded and a collector may reserve a little of -Xmx + + +def size_to_bytes(text): + """'2.00GB' / '512MB' / '1.5 GiB' -> bytes (base 1024), None when it is not a size.""" + m = _SIZE_RE.fullmatch(str(text or "")) + if not m: + return None + return int(float(m.group(1).replace(",", "")) * 1024 ** (" KMGT".index(m.group(2).upper() or " "))) + + +def parse_server_started(log): + """{server_mode, server_cpus, server_max_heap, server_max_heap_bytes} from the LAST start line of a server log, or {}.""" + hits = _STARTED_RE.findall(log or "") + if not hits: + return {} + mode, cpus, maxram = hits[-1] + return {"server_mode": mode, "server_cpus": int(cpus), "server_max_heap": maxram.strip(), "server_max_heap_bytes": size_to_bytes(maxram)} + + +def server_log_problems(got, want_heap_bytes, want_cpus, want_mode=None): + """The start line against what was asked: the heap within HEAP_TOLERANCE of the requested -Xmx (when one was requested), the CPUs equal to the + cpuset's size, the mode equal to the requested one. A missing line is a problem: the witness must exist.""" + if not got: + return ["the server's start line (CPUs, MAXRAM) is not in its log"] + out = [] + mb = got.get("server_max_heap_bytes") + if want_heap_bytes: + if not mb or abs(mb - want_heap_bytes) > HEAP_TOLERANCE * want_heap_bytes: + out.append(f"the server reports MAXRAM {got.get('server_max_heap')} and -Xmx asked for {want_heap_bytes} bytes") + if want_cpus and got.get("server_cpus") != want_cpus: + out.append(f"the server sees {got.get('server_cpus')} CPUs and the container's cpuset has {want_cpus}") + if want_mode and got.get("server_mode") != want_mode: + out.append(f"the server runs in '{got.get('server_mode')}' mode and '{want_mode}' was requested") + return out + + +def arcadedb_server_log_readback(cid, tries=15): + """parse_server_started over the container's log, waiting a few seconds for the line (it follows the HTTP ready line).""" + got = {} + for _ in range(tries): + got = parse_server_started(docker_logs(cid)) + if got: + break + time.sleep(1) + return got + + def arcadedb_settings_readback(cid, text): """What a served ArcadeDB (either build) runs with, asked of the engine over HTTP, before the client starts and outside every timer: `GET /api/v1/server?mode=default` (server-scoped settings with the value in force), `SELECT FROM schema:database` (database-scoped ones, @@ -3562,6 +3614,10 @@ def run_cell(job, rep, scale, cpuset, tier, net_name): row.update(_rb) _bad = settings_problems(_rb.get("server_settings"), _rb.get("server_settings_requested")) _bad += engine_commit_problems(_rb.get("server_engine_commit"), _LOCAL_ENGINE_COMMIT) + _lg = arcadedb_server_log_readback(server_cid) + row.update(_lg) + _bad += server_log_problems(_lg, None if be.get("jvm_defaults") else mem_bytes(heap), _cpuset_size(cpuset), + (_rb.get("server_settings_requested") or {}).get("arcadedb.server.mode")) if _bad: row["error"] = "; ".join(_bad) + "; the cell is not the one we specified" return row diff --git a/benchmarks/experiments/smoke_native_server.sh b/benchmarks/experiments/smoke_native_server.sh index ab25a0b182b..3c75f8aa3f4 100755 --- a/benchmarks/experiments/smoke_native_server.sh +++ b/benchmarks/experiments/smoke_native_server.sh @@ -16,6 +16,7 @@ TAG="smoke_native_server_$(date +%s)" [ -f "$WHEEL" ] || { echo "SMOKE FAIL: no native wheel $WHEEL"; exit 1; } case "$(basename "$WHEEL")" in arcadedb-[0-9]*) ;; *) echo "SMOKE FAIL: $WHEEL is not the native (arcadedb-) wheel"; exit 1 ;; esac mkdir -p "$OUT" || exit 1 +exec > >(tee -a "$OUT/smoke.log") 2>&1 # the script writes its own log into OUT, so the queue line needs no mkdir or redirect docker image inspect "$IMG" >/dev/null 2>&1 || docker pull -q "$IMG" >/dev/null || { echo "SMOKE FAIL: cannot pull $IMG"; exit 1; } # the image answers on its own before the runner starts it (reports the commit; no sha asserted) bash "$HERE/verify_native_server_image.sh" "$IMG" > "$OUT/verify.txt" 2>&1 || { cat "$OUT/verify.txt"; echo "SMOKE FAIL: the native server image did not answer"; exit 1; } @@ -25,15 +26,21 @@ echo "native server commit (reported, not asserted): $COMMIT" # ARCADEDB_SERVER_IMAGE is the JVM image the F5 rule wants set beside the wheel; the native chain never starts it, so a name is enough here. export ARCADEDB_WHEEL="$WHEEL" ARCADEDB_SERVER_IMAGE="arcadedb-c25:${COMMIT:0:9}" ARCADEDB_NATIVE_SERVER_IMAGE="$IMG" BENCH_SERVER_BUILD=native export BENCH_CPUSET="${BENCH_CPUSET:-4,5}" BENCH_INSTRUMENT=2026-11 +# SMALL ON PURPOSE: the runner's own diagnostic overrides, --mem (the server's cap) and --heap (-Xmx), at 3g and 2g, so the smoke fits a 4 GB +# queue slot and the MAXRAM read-back below checks the -Xmx path at 2g. Rows go to their own results file, never into runs.jsonl. +SMOKE_MEM=3g; SMOKE_HEAP=2g +NCPU=0 +IFS=, read -ra _parts <<< "$BENCH_CPUSET" +for _p in "${_parts[@]}"; do _a=${_p%-*}; _b=${_p#*-}; NCPU=$((NCPU + _b - _a + 1)); done cd "$HERE" || exit 1 rc=0 for CELL in "l2 arcadedb_graph_server oltp" "l1 arcadedb_server oltp"; do set -- $CELL - uv run --no-project --python 3.12 --with numpy python runner.py --lanes "$1" --backends "$2" --workloads "$3" --scale tiny --reps 1 --tier sweep \ + uv run --no-project --python 3.12 --with numpy python runner.py --lanes "$1" --backends "$2" --workloads "$3" --scale tiny --reps 1 --tier sweep --mem "$SMOKE_MEM" --heap "$SMOKE_HEAP" \ --results-file "$TAG.jsonl" > "$OUT/run_$1.log" 2>&1 || { tail -20 "$OUT/run_$1.log"; echo "SMOKE FAIL: runner exited non-zero for $1"; rc=1; } done [ "$rc" = 0 ] || exit 1 -RESULTS_FILE="$HERE/results/$TAG.jsonl" EXPECT_COMMIT="$COMMIT" uv run --no-project --python 3.12 python -I - <<'PY' +RESULTS_FILE="$HERE/results/$TAG.jsonl" EXPECT_COMMIT="$COMMIT" EXPECT_HEAP_BYTES=$((2 * 1024 * 1024 * 1024)) EXPECT_CPUS=$NCPU uv run --no-project --python 3.12 python -I - <<'PY' import json, os, sys sys.path.insert(0, os.getcwd()) import arm_identity as AI @@ -60,8 +67,14 @@ for lane, arm in want.items(): need("read from the engine over HTTP" in str(r.get("server_settings_source")), f"settings source: {r.get('server_settings_source')!r}") need(str(r.get("server_engine_commit") or "").startswith(os.environ["EXPECT_COMMIT"][:7]), f"server_engine_commit={r.get('server_engine_commit')!r}") need("UseCompactObjectHeaders" in str(r.get("native_dropped_flags")), f"native_dropped_flags={r.get('native_dropped_flags')!r}") + need(r.get("server_mode") == "development", f"server_mode={r.get('server_mode')!r} from the start line") + need(r.get("server_cpus") == int(os.environ["EXPECT_CPUS"]), f"server_cpus={r.get('server_cpus')!r}, the cpuset has {os.environ['EXPECT_CPUS']}") + hb = r.get("server_max_heap_bytes") + need(isinstance(hb, (int, float)) and abs(hb - int(os.environ["EXPECT_HEAP_BYTES"])) <= 0.05 * int(os.environ["EXPECT_HEAP_BYTES"]), + f"server_max_heap={r.get('server_max_heap')!r} ({hb!r} bytes), -Xmx2g asked for {os.environ['EXPECT_HEAP_BYTES']}") + need("-Xmx2g" in str(r.get("server_cmd")), f"server_cmd lacks -Xmx2g: {r.get('server_cmd')!r}") need("-Xmx" in str(r.get("server_cmd")) and "-Darcadedb.txWalFlush=0" in str(r.get("server_cmd")), f"server_cmd={r.get('server_cmd')!r}") - print(f"{lane}/{arm}: build={r.get('engine_backend')} commit={r.get('server_engine_commit')} heap={r.get('server_heap')} cpu_baseline={r.get('cpu_baseline')!r} settings={s}") + print(f"{lane}/{arm}: mode={r.get('server_mode')} cpus={r.get('server_cpus')} maxram={r.get('server_max_heap')} build={r.get('engine_backend')} commit={r.get('server_engine_commit')} heap={r.get('server_heap')} cpu_baseline={r.get('cpu_baseline')!r} settings={s}") pair = AI.server_settings_problems(rows) bad += pair if bad: diff --git a/benchmarks/experiments/test_build_pair_page.py b/benchmarks/experiments/test_build_pair_page.py index 43b870eba11..4ee0d6504ae 100644 --- a/benchmarks/experiments/test_build_pair_page.py +++ b/benchmarks/experiments/test_build_pair_page.py @@ -152,13 +152,14 @@ def _run(script, results): def _with_native(results, keep=lambda r: True, commit=None): """Rewrite the freeze: every in-process ArcadeDB row gains a native twin (for the rows `keep` accepts).""" rows = list(csv.DictReader((results / "runs_paper.csv").open())) - fields = list(rows[0]) + [f for f in ("engine_backend", "server_settings", "cpu_baseline") if f not in rows[0]] + fields = list(rows[0]) + [f for f in ("engine_backend", "server_settings", "cpu_baseline", *AI.SERVER_LOG_FIELDS) if f not in rows[0]] twins = [] for r in rows: if AI.built(r) and keep(r): t = dict(r, engine_backend="native", cpu_baseline=AI.cpu_baseline("native", AI.served(r))) if AI.served(r): # a native server row carries the settings its engine reported at start, equal to the JVM server's t["server_settings"] = json.dumps({k: "x" for k in AI.SERVER_SETTING_KEYS}) + t.update(server_mode="development", server_cpus=4, server_max_heap_bytes=2 << 30) if commit: t["engine_commit"] = commit twins.append(t) diff --git a/benchmarks/experiments/test_native_server_arm.py b/benchmarks/experiments/test_native_server_arm.py index 0365f24f2fc..4e055aaacbb 100644 --- a/benchmarks/experiments/test_native_server_arm.py +++ b/benchmarks/experiments/test_native_server_arm.py @@ -129,24 +129,64 @@ def test_the_cpu_baseline_of_each_artifact_is_stamped_and_not_changed(): "arcadedb.server.httpIncomingPort": "2480", "arcadedb.txWalFlush": "0"} +LOG = {"server_mode": "development", "server_cpus": 4, "server_max_heap_bytes": 2 << 30} + + def test_the_pair_gate_refuses_servers_whose_settings_differ_and_a_native_row_with_no_read_back(): - a, b = row("native", server_settings=dict(SETTINGS)), row("jvm", server_settings=dict(SETTINGS)) + a, b = row("native", server_settings=dict(SETTINGS), **LOG), row("jvm", server_settings=dict(SETTINGS), **LOG) assert AI.pair_problems([a, b]) == [] b["server_settings"] = dict(SETTINGS, **{"arcadedb.txWalFlush": "1"}) assert any("arcadedb.txWalFlush" in p and "native server" in p for p in AI.pair_problems([a, b])) assert any("no server_settings" in p for p in AI.pair_problems([row("native"), row("jvm", server_settings=dict(SETTINGS))])) # the rows read back from the CSV hold the dict as JSON text - assert AI.pair_problems([row("native", server_settings=json.dumps(SETTINGS)), row("jvm", server_settings=json.dumps(SETTINGS))]) == [] + assert AI.pair_problems([row("native", server_settings=json.dumps(SETTINGS), **LOG), row("jvm", server_settings=json.dumps(SETTINGS))]) == [] # different durability classes are different cells c = row("jvm", server_settings=dict(SETTINGS, **{"arcadedb.txWalFlush": "1"}), durability_class="strict") assert AI.pair_problems([a, c]) == [] def test_the_pair_gate_covers_the_servers_engine_commit(): - a, b = row("native", server_settings=SETTINGS), row("jvm", server_settings=SETTINGS, engine_commit="f" * 40) + a, b = row("native", server_settings=SETTINGS, **LOG), row("jvm", server_settings=SETTINGS, engine_commit="f" * 40) assert any("engine_commit" in p for p in AI.pair_problems([a, b])) def test_a_one_build_freeze_drops_the_other_builds_server_rows(): rows = [row("native"), row("jvm"), row(backend="duckdb")] assert [AI.build_of(r) for r in AI.select_build(rows, "native")] == ["native", None] + + +# ---------------------------------------------------------------- the server's own start line (mode, CPUs, MAXRAM) +LINE = "2026-10-30 10:00:00.123 INFO ArcadeDB Server started in 'development' mode (CPUs=4 MAXRAM=2.00GB) [ArcadeDBServer]" + + +def test_the_start_line_is_parsed_from_the_log_and_the_last_one_wins(): + got = R.parse_server_started("noise\n" + LINE + "\n" + LINE.replace("CPUs=4", "CPUs=6").replace("2.00GB", "512MB")) + assert got == {"server_mode": "development", "server_cpus": 6, "server_max_heap": "512MB", "server_max_heap_bytes": 512 << 20} + assert R.parse_server_started(LINE)["server_max_heap_bytes"] == 2 << 30 + assert R.parse_server_started("no such line") == {} + assert R.size_to_bytes("1.5 GiB") == 3 << 29 and R.size_to_bytes("garbage") is None + + +def test_the_cell_fails_when_maxram_cpus_or_mode_differ_from_the_request(): + ok = R.parse_server_started(LINE) + assert R.server_log_problems(ok, 2 << 30, 4, "development") == [] + assert R.server_log_problems(dict(ok, server_max_heap_bytes=int((2 << 30) * 0.97)), 2 << 30, 4) == [] # rounding is allowed + assert any("MAXRAM" in p for p in R.server_log_problems(dict(ok, server_max_heap_bytes=1 << 30), 2 << 30, 4)) # a -Xmx that did not take + assert any("CPUs" in p for p in R.server_log_problems(ok, 2 << 30, 2)) + assert any("mode" in p for p in R.server_log_problems(ok, 2 << 30, 4, "production")) + assert R.server_log_problems(ok, None, 4) == [] # the image-defaults arm names no -Xmx + assert R.server_log_problems({}, 2 << 30, 4) # no start line is a failure + + +def test_the_pair_gate_compares_the_start_line_and_the_page_cache(): + a, b = row("native", server_settings=dict(SETTINGS), **LOG), row("jvm", server_settings=dict(SETTINGS), **LOG) + assert AI.pair_problems([a, b]) == [] + assert AI.pair_problems([a, dict(b, server_max_heap_bytes=int((2 << 30) * 0.98))]) == [] + assert any("MAXRAM" in p for p in AI.pair_problems([a, dict(b, server_max_heap_bytes=1 << 30)])) + assert any("server_cpus" in p for p in AI.pair_problems([a, dict(b, server_cpus=8)])) + assert any("server_mode" in p for p in AI.pair_problems([a, dict(b, server_mode="production")])) + assert any("start-line" in p for p in AI.pair_problems([row("native", server_settings=dict(SETTINGS)), b])) + page_a = dict(SETTINGS, **{"arcadedb.maxPageRAM": "4096"}) + page_b = dict(SETTINGS, **{"arcadedb.maxPageRAM": "1024"}) + assert any("arcadedb.maxPageRAM" in p for p in AI.pair_problems([dict(a, server_settings=page_a), dict(b, server_settings=page_b)])) + assert AI.pair_problems([dict(a, server_settings=page_a), dict(b, server_settings=page_a)]) == [] diff --git a/benchmarks/experiments/verify_native_server_image.sh b/benchmarks/experiments/verify_native_server_image.sh index 13b32bd02ec..166d9453afc 100755 --- a/benchmarks/experiments/verify_native_server_image.sh +++ b/benchmarks/experiments/verify_native_server_image.sh @@ -35,7 +35,7 @@ for _ in $(seq 1 90); do done [ "$up" = 1 ] || fail "the server did not answer /api/v1/ready in 90 s" V=$(curl -sf -u "root:$PW" -H 'Content-Type: application/json' -d '{"language":"sql","command":"SELECT version() AS v"}' \ - "http://127.0.0.1:$PORT/api/v1/query/bench" | python3 -c 'import sys,json; print(json.load(sys.stdin)["result"][0]["v"])') || fail "SELECT version() failed" + "http://127.0.0.1:$PORT/api/v1/query/bench" | uv run --no-project python -c 'import sys,json; print(json.load(sys.stdin)["result"][0]["v"])') || fail "SELECT version() failed" GOT=$(printf '%s' "$V" | sed -n 's/.*(build \([0-9a-f]\{7,40\}\)\/.*/\1/p') [ -n "$GOT" ] || fail "no build sha in '$V'" echo "NATIVE_SERVER_VERSION=$V" From a749a4de1a7e7a0fcb24dc481455656f1000a5c6 Mon Sep 17 00:00:00 2001 From: Taewoon Kim Date: Fri, 30 Oct 2026 11:11:13 +0900 Subject: [PATCH 09/23] benchmarks: the native server smoke passes its cpuset as a range (4-5) The first smoke run stopped in runner.split_cpuset, which parses only the a-b form, on the default BENCH_CPUSET=4,5. The image check before it passed (the native image answered and reported its engine commit). The smoke's own CPU count already handled ranges. Co-Authored-By: Claude Opus 5.5 --- benchmarks/experiments/smoke_native_server.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/benchmarks/experiments/smoke_native_server.sh b/benchmarks/experiments/smoke_native_server.sh index 3c75f8aa3f4..e29f6c2c0e2 100755 --- a/benchmarks/experiments/smoke_native_server.sh +++ b/benchmarks/experiments/smoke_native_server.sh @@ -25,7 +25,7 @@ COMMIT=$(sed -n 's/^NATIVE_SERVER_COMMIT=//p' "$OUT/verify.txt") echo "native server commit (reported, not asserted): $COMMIT" # ARCADEDB_SERVER_IMAGE is the JVM image the F5 rule wants set beside the wheel; the native chain never starts it, so a name is enough here. export ARCADEDB_WHEEL="$WHEEL" ARCADEDB_SERVER_IMAGE="arcadedb-c25:${COMMIT:0:9}" ARCADEDB_NATIVE_SERVER_IMAGE="$IMG" BENCH_SERVER_BUILD=native -export BENCH_CPUSET="${BENCH_CPUSET:-4,5}" BENCH_INSTRUMENT=2026-11 +export BENCH_CPUSET="${BENCH_CPUSET:-4-5}" BENCH_INSTRUMENT=2026-11 # a range: runner.split_cpuset parses a-b only # SMALL ON PURPOSE: the runner's own diagnostic overrides, --mem (the server's cap) and --heap (-Xmx), at 3g and 2g, so the smoke fits a 4 GB # queue slot and the MAXRAM read-back below checks the -Xmx path at 2g. Rows go to their own results file, never into runs.jsonl. SMOKE_MEM=3g; SMOKE_HEAP=2g From 9c2250f65f03235148ac7c67c7baeb2c53507f3a Mon Sep 17 00:00:00 2001 From: Taewoon Kim Date: Fri, 30 Oct 2026 11:34:09 +0900 Subject: [PATCH 10/23] benchmarks: the native server smoke checks both client rows and each cell's own cpuset Third smoke run (laptop, 2026-10-30 11:32 KST, upstream arcadedata/arcadedb:26.11.1-SNAPSHOT-native, engine cb43e8dac): all four cells passed (l1 and l2, the arcadedb-driver headline client and the lean_http sensitivity twin), but the row check failed on two assumptions of its own: - it expected one row per served arm; every served ArcadeDB arm runs twice (headline + sensitivity), so it now requires exactly one row of each role and checks every row; - it compared server_cpus with the whole smoke cpuset (2); the runner shards the cpuset across its workers, so each server gets its shard (4-4, 5-5) and reports 1. It now compares server_cpus with the row's own server_cpuset, and that shard with the smoke's cpuset. Run against the third run's rows, the fixed check prints SMOKE PASS (mode development, MAXRAM 2.00GB from -Xmx2g, cpus 1 = shard, build native, cpu_baseline x86-64-v3). Co-Authored-By: Claude Opus 5.5 --- benchmarks/experiments/smoke_native_server.sh | 17 +++++++++++++---- 1 file changed, 13 insertions(+), 4 deletions(-) diff --git a/benchmarks/experiments/smoke_native_server.sh b/benchmarks/experiments/smoke_native_server.sh index e29f6c2c0e2..5db221cd449 100755 --- a/benchmarks/experiments/smoke_native_server.sh +++ b/benchmarks/experiments/smoke_native_server.sh @@ -47,12 +47,17 @@ import arm_identity as AI rows = [json.loads(l) for l in open(os.environ["RESULTS_FILE"]) if l.strip()] bad = [] want = {"l2": "arcadedb_graph_server", "l1": "arcadedb_server"} +# every served ArcadeDB arm runs twice: the headline client (the official arcadedb-driver) and its sensitivity twin (lean_http) +checks = [] for lane, arm in want.items(): got = [r for r in rows if r.get("lane") == lane and r.get("backend") == arm] - if len(got) != 1: - bad.append(f"{lane}/{arm}: {len(got)} rows, expected 1") + roles = sorted(str(r.get("arcadedb_http_client_role")) for r in got) + if roles != ["headline", "sensitivity"]: + bad.append(f"{lane}/{arm}: client roles {roles}, expected one headline and one sensitivity row") continue - r = got[0] + checks += [(lane, arm, r) for r in got] +for lane, arm, r in checks: + arm = f"{arm}[{r.get('arcadedb_http_client_role')}]" def need(cond, msg): if not cond: bad.append(f"{lane}/{arm}: {msg}") @@ -68,7 +73,11 @@ for lane, arm in want.items(): need(str(r.get("server_engine_commit") or "").startswith(os.environ["EXPECT_COMMIT"][:7]), f"server_engine_commit={r.get('server_engine_commit')!r}") need("UseCompactObjectHeaders" in str(r.get("native_dropped_flags")), f"native_dropped_flags={r.get('native_dropped_flags')!r}") need(r.get("server_mode") == "development", f"server_mode={r.get('server_mode')!r} from the start line") - need(r.get("server_cpus") == int(os.environ["EXPECT_CPUS"]), f"server_cpus={r.get('server_cpus')!r}, the cpuset has {os.environ['EXPECT_CPUS']}") + def ncpu(cs): # '4-5' or '4,5' or '4-4' -> number of CPUs + return sum(int(p.split("-")[-1]) - int(p.split("-")[0]) + 1 for p in str(cs).split(",") if p) + # the runner shards the smoke's cpuset across its workers, so a cell's server gets its own shard (server_cpuset), not the whole set + need(r.get("server_cpuset") and r.get("server_cpus") == ncpu(r.get("server_cpuset")), f"server_cpus={r.get('server_cpus')!r}, server_cpuset={r.get('server_cpuset')!r}") + need(ncpu(r.get("server_cpuset") or "") <= int(os.environ["EXPECT_CPUS"]), f"server_cpuset={r.get('server_cpuset')!r} is wider than the smoke's {os.environ['EXPECT_CPUS']} CPUs") hb = r.get("server_max_heap_bytes") need(isinstance(hb, (int, float)) and abs(hb - int(os.environ["EXPECT_HEAP_BYTES"])) <= 0.05 * int(os.environ["EXPECT_HEAP_BYTES"]), f"server_max_heap={r.get('server_max_heap')!r} ({hb!r} bytes), -Xmx2g asked for {os.environ['EXPECT_HEAP_BYTES']}") From 1b2d6a3aca7b4d07fc2a0793b2c3d6629a233c33 Mon Sep 17 00:00:00 2001 From: Taewoon Kim Date: Wed, 4 Nov 2026 00:54:34 +0900 Subject: [PATCH 11/23] benchmarks: the view lists Message from 26.11.1; the rep state is per build in a release chain l2_graph._gav_types() adds the abstract Message supertype when the engine version the adapter records (self.version, the row's engine_version) is 26.11.1 or later, and not before: since upstream PR 9320 the registry hands a view to a walk only when it covers every vertex type. A version the harness cannot read is refused. The release lives in bench_common.GAV_SUPERTYPE_LISTED_FROM, shared with export_web's refusal of a stale row. The stage generator's REPSTATE file carries the build (-jvm, -native) in an ArcadeDB release chain: both chains label cells identically, so one shared file made the second chain skip every cell as clean. Comparator stages and the old-wheel chain are byte for byte unchanged, so stage_drafts_2611.sha256 does not move. Co-Authored-By: Claude Opus 5.5 --- benchmarks/experiments/CAMPAIGN.md | 4 +- benchmarks/experiments/bench_common.py | 18 +++++++ benchmarks/experiments/export_web.py | 2 +- benchmarks/experiments/l2_graph.py | 18 +++++-- benchmarks/experiments/make_2611_stages.py | 13 ++++- .../experiments/test_chain_served_arms.py | 48 +++++++++++++++++++ benchmarks/experiments/test_gav_scope.py | 48 ++++++++++++++++++- 7 files changed, 141 insertions(+), 10 deletions(-) diff --git a/benchmarks/experiments/CAMPAIGN.md b/benchmarks/experiments/CAMPAIGN.md index 2f0b7479140..423d995d76c 100644 --- a/benchmarks/experiments/CAMPAIGN.md +++ b/benchmarks/experiments/CAMPAIGN.md @@ -202,7 +202,7 @@ Rules every campaign keeps: | 58 | Every write the documents and graph tables time is inside an answer check (new-order stock, payment count, KNOWS edges after the insert and after the delete) | docs_oltp and graph OLTP (l2) | built (DECISIONS #151, F172, within #150's remit): `l1_tpc.py`, `l2_graph.py` (`edge_scan`), `test_docs_post_state.py`, `test_graph_post_state.py`; `equivalence_check` names AGE's missing first edge as the open defect apache/age#2587 | BUGS F172 (`.notes/bench/repros/validity-20261004`) | | 59 | The answer profile (`res__profile`) and the degenerate-answer check (`degenerate_check.py`: ZERO, CONSTANT, SCALE, and EMPTY answers) | every lane (one field added to every row) | **built (DECISIONS #151, within #150's remit)**: `bench_common.py`, `degenerate_check.py`, `test_degenerate_check.py`, `test_result_digest.py` | `.notes/bench/repros/validity-20261004/SUMMARY.md`: agreeing digests hid both F146 and F169 | | 60 | LSQB's q5, q6, q8, and q9 run LSQB's own text on every engine; every graph analytics row records `lsqb_text` | every engine on l2olap | built (the user, DECISIONS #154 items 1 and 2), except the `sf1full` exclusion: `graph_common.py`, `l2_graph.py`, `test_lsqb_text.py`. **The exclusion of q6 and q9 at `sf1full` (`graph_common.TIER_EXCLUDED`, DECISIONS #109) is NOT lifted: it needs a one-cell first-touch probe on mini and a maintainer decision (DECISIONS #161 D2: lift it after the probe, at the end of the campaign).** | BUGS F174 | -| 61 | ArcadeDB's Graph Analytical View covers every vertex and edge type of the loaded graph, as upstream's LSQB runner builds it; the no-view arm (`BENCH_GAV=0`) stays as the like-for-like row | ArcadeDB only (l2olap with-view arms) | built (the user, DECISIONS #154 item 3): `l2_graph.py` (rows record `gav_types`, a narrower cover refuses the cell), `test_gav_scope.py` | `.notes/bench/repros/perf-lsqb-q9-20261004` | +| 61 | ArcadeDB's Graph Analytical View covers every vertex and edge type of the loaded graph, as upstream's LSQB runner builds it; the no-view arm (`BENCH_GAV=0`) stays as the like-for-like row | ArcadeDB only (l2olap with-view arms) | built (the user, DECISIONS #154 item 3): `l2_graph.py` (rows record `gav_types`, a narrower cover refuses the cell; from 26.11.1 the list also carries the abstract `Message` supertype, upstream PR 9320, chosen from the recorded engine version, `bench_common.GAV_SUPERTYPE_LISTED_FROM`), `test_gav_scope.py` | `.notes/bench/repros/perf-lsqb-q9-20261004` | | 62 | ArcadeDB's sparse arms return ids, as every comparator does (`SELECT id, score FROM (...)`); every ArcadeDB sparse row records `sparse_result` (`id`) | ArcadeDB only (l3s and its multipass warm passes) | built (the user, DECISIONS #153 item 1, adopted as parity): `l3_sparse.py`, `sparse_multipass_driver.py`, `test_sparse_rows.py` | `.notes/bench/repros/perf-sparse-20261004`: the ids and their order are identical on 50 of 50 queries | | 63 | The documents lane loads ArcadeDB embedded through the bindings' columnar insert (own issue #150) instead of `insert_many` | ArcadeDB embedded only | built (the user's choice over my recommendation, DECISIONS #153 item 2): `Database.insert_columns` (`7a36dc650e`), `ArcadeTPC.build` in `l1_tpc.py`, `test_insert_columns.py`, `test_docs_columnar_load.py`; the row records `columnar_insert`. **The measured wheel must contain the bindings commit** (row 74) | `.notes/bench/repros/perf-docs-ingest-20261004` | | 64 | The sparse multipass driver keeps each comparator's version, as the dense driver does: `sparse_multipass_driver.py` stores the adapter's own string as `lib_version` and restores `engine_version` for the ArcadeDB arms | every comparator's sparse multipass warm passes (`l3s_mp`); no timing moves | **built (DECISIONS #150's remit, own issue #158)**: `test_sparse_mp_versions.py`; the October files stay as recorded | BUGS F173 | @@ -244,7 +244,7 @@ The detail behind item 8: **The next campaign's stages are generated by `make_2611_stages.py`.** One template carries every guard the campaign learned (cap table, shadowed definitions, budgets against the cap, whole-cell cost, images present and current, comparator digests frozen at generation, the JPype pin read out of the image, the pinned wheel by sha256, the cap rule, the ALL-DONE trap); the instrument, `REPS` and the banner are its parameters. The generator selects the multi-model subset, narrowed by a profile (next paragraph). The subset is `ENGINES` in the file: ArcadeDB (embedded and served), SurrealDB (embedded and served), ArangoDB, Memgraph, PostgreSQL with pgvector, AGE and TimescaleDB (and plain PostgreSQL where a lane uses it), MongoDB with MongoDB Search, DuckDB with VSS and DuckPGQ, SQLite with sqlite-vec, the composed Qdrant plus Neo4j stack, Neo4j and Qdrant on their own (graph and dense; sparse on the second list), and LadybugDB on every lane it supports. Every registered arm is either in the subset or in `OUTSIDE`, so a newly registered arm forces a decision (`--check`). The core lanes are graph (interactive and analytics), cross-model, documents and dense; under `--profile full`, `--second-list` adds lifecycle, then sparse and time series, after them. TimescaleDB runs whenever the time-series lane does (so not under the default profile); `--without-timescaledb` leaves it out. **THE PAPER PROFILE (the owner, 2026-10-25: the paper's experiments must finish by 2026-11-10).** `--profile paper-2611` is the default and narrows the subset in the open (`PROFILES`, `DROP_WHY` in the file). (1) Core lanes only: graph, cross-model, documents and dense. The second list (lifecycle, sparse, time series) is out of scope for the paper, for the comparators and for the ArcadeDB chain, and `--second-list` is refused unless `--profile full` is given. (2) Neo4j and Memgraph stay in the graph lane (and cross-model where they are); their dense-lane cells (`neo4j_dense`, `neo4j_dense_int8`, `memgraph_dense`, `memgraph_dense_int8`) are out. The composed embedded stack (SQLite + LanceDB + LadybugDB in one process, `composed_embedded_sqlite_lance_ladybug`, the owner, 2026-10-25) is a Tier 1 cross-model arm: engine `composed-embedded`, right after the served composed stack, six lane cells per pass (hybrid in both durability classes and atomicity, at e2 and e2_500k), on the lane `e2` only; it runs in the dense image, so rebuild `dbbench:dense` (`./build_images.sh dense`) before the first pass. (3) The dense lane is fp32 only, for every engine: no `_int8` and no `_pq` arm, the comparators' and ArcadeDB's alike (`arangodb_dense_int8`, `arcadedb_dense_embedded_int8`, `arcadedb_dense_server_int8`, `mongodb_dense_int8`, `qdrant_dense_int8`, `sqlite_vec_dense_int8`, and the PQ arms when `BENCH_DENSE_PQ=1`). Quantized arms are not like-for-like across engines, and quantization is out of the paper's scope: fp32 for every engine, ArcadeDB's own INT8 and BINARY out by the owner's rule, jvector PQ, BQ and NVQ studied after the paper. ArcadeDB keeps its fp32 dense arms, embedded and served. (4) The target is N=3 (`TARGET_REPS`; `--project` prices 3 repetitions by default). The chain keeps the passes 4 and 5 after pass 3, in #503's order, so they run only if time is left; stopping after the first six groups is a complete result. `--check` prints every dropped cell with its reason and count (lane cells per pass: 148 second-list, 8 Neo4j and Memgraph dense, 12 quantized dense) and proves that none is scheduled, in both chains whatever the arguments select (`profile_problems`); `--check --drafts DIR` lints written scripts for the same. `--profile full` is the whole subset as it was before the profile. Every generated stage exports `BENCH_INSTRUMENT=2026-11` before its first Python call, reads the selection back from `bench_common` and aborts if the tree does not select it, writes `runs_page__2611.jsonl` (and overlay directories with the same suffix) so one rows file never holds two instruments, and fixes `REPS` at generation (`--reps`, default 5, at most 5; the target is 3). The generator re-executes itself with the variable set and refuses a different value in the environment. `land_stage.py` lands exactly these files (DECISIONS #196; the 2026-10-25 change that closed the gap between the pull and the freeze): it pulls `runs_page__2611.jsonl` and the overlay directories with the same suffix, and `screen_pulled_rows` refuses the file before anything is merged unless every row is stamped `2026-11` and names the pin's engine commit, so a retired 2026-10 file or a file that mixes instruments never reaches `results/runs.jsonl`. The first landing meets no store (the October one is retired) and `merge_campaign.py` creates it. `export_web` no longer demands the `e4decomp_` artifacts of a landing scoped away from e4, because the chain has no e4 stage and `--only-lanes l2` would otherwise wait for one. A rehearsal restores what it wrote, the site payload (removed when the run created it) and the PAGE-SPEC.md block included. Failed cells (`rc != 0` or an error) merge as failures and are not frozen; censored queries keep their flag and publish no number. No stage script calls `land_stage.py` (the generated `qC` scripts only name it in a comment about the rows file); a landing is run by hand on the laptop. -The comparators are the default mode and run on the release pair already installed (`ARCADEDB_WHEEL`, `ARCADEDB_SERVER_IMAGE`, `ARCADEDB_ENGINE_COMMIT`): `python3 make_2611_stages.py --after --status-file STATUS.txt --id-prefix qS --out DIR`. `--after` is required, with no default (a marker name, or `none`), and the first stage waits for `grep -qx " ALL-DONE"`, each later one for the stage before it. The marker must be the LAST item still to run, one that writes its line whether it runs or is skipped: a marker already in the machine's STATUS.txt makes the first stage start at once, beside the running jobs (2026-10-23: `qMA06 ALL-DONE`, written by a skipped study). So `--status-file` is required with a marker (a copy of that STATUS.txt, or `none` when there is no queue) and the generator refuses a marker the file already holds; `--check --drafts DIR --status-file STATUS.txt` makes the same test on drafts that exist. The subset is emitted in ENGINE TIERS in the user's exact pass order (2026-10-24; `--tiers`, on by default, `--no-tiers` for the plain rep-major chain). N defaults to 5, the target is 3 (the paper profile), and the order is `schedule_order`: n=1 T1, n=2 T1, n=3 T1, n=1 T2, n=2 T2, n=3 T2, n=4 T1, n=4 T2, n=5 T1, n=5 T2. Every prefix of it is a complete set of equal replicates for the tiers covered so far (each tier has run passes 1..k, none skipped, and a pass is whole before the next starts); `--reps 3` stops after the first six groups, which is the chain of Tier 1 passes 1-3 and then Tier 2 passes 1-3 as before. Tier 1 is ArcadeDB (the four 26.11.1 arms, when `--arcadedb-release` adds them), SurrealDB, ArangoDB, PostgreSQL with pgvector and AGE, MongoDB with Search, the composed Qdrant plus Neo4j stack, and the embedded references SQLite, LadybugDB and DuckDB; Tier 2 is Memgraph, Neo4j and Qdrant on their own, Redis 8, CrateDB and TimescaleDB (`TIER_ENGINES`, with `ARM_TIER` for TimescaleDB, an arm of the `postgresql` engine that the user lists in Tier 2). Every stage holds the arms of one tier, and stage ids stay one run of consecutive numbers (a stage that mixed both tiers is split in two, so the stage count grows, and each stage repeats its guards and image checks; past 99 stages the ids take three digits, so `qS100` cannot sort before `qS11`). `--check` prints the schedule and proves the order and the prefixes (the plan collapsed into (tier, pass) groups equals `schedule_order`, each group holds the whole pass), and `--check --drafts DIR` lints a draft folder for the same. An arm with no tier runs after Tier 2 in each block. The ArcadeDB arms are Tier 1 but not in this chain: DBBench keeps one ArcadeDB version per host, so they run only once the benchmark pin is 26.11.1 (next paragraph). Inside a tier the order is the one below: a pass is pass k of every stage of the tier (core lanes, then the second list), and each stage runs ONE repetition (`--only-reps $PASS`). With `--no-tiers` the chain is pass 1 of every stage, then pass 2, up to pass N. If time runs out, every engine of a tier has the same n; repetitions days apart show drift. A stage whose environment declares fewer reps (`REPS=k`) is left out of the later passes. Inside a lane the engines run in `ENGINE_PRIORITY` (SurrealDB, ArangoDB, PostgreSQL, MongoDB, Redis 8, composed Qdrant+Neo4j, Neo4j, Qdrant, DuckDB, SQLite, Memgraph, LadybugDB, CrateDB; ArcadeDB's arms lead it when they join; Redis and CrateDB are skipped until their adapters are in `ENGINES`). The rep-1-first rule and the cap rule hold across passes: a pass above 1 runs a cell only when rep 1 of it was clean, and only a timeout of rep 1 censors the larger sizes. WHICH REPS EXIST is `~/rep-state-2026-11-.tsv` on the host (one `label, rep, ok|fail` line per finished repetition; the rows themselves carry the rep number in the runner's results file): stop after any pass, and the next pass, or a restarted stage, picks up from it with no regeneration; N can go to 3 or stop at 2. Delete that file to start a chain over. The ArcadeDB pass is `--arcadedb-release X.Y.Z` with `--engine-commit`, `--image-digest sha256:...`, `--wheel-name`, `--wheel-sha256` and `--wheel-url`; it refuses unless all five are given, because the release may not exist when the drafts are made. It writes `00`, the pair install (official image by digest, wheel by sha256, `build_matched_pair.sh`, `verify_pair_c25.sh`), and ArcadeDB's lane stages behind it. THE ARCADEDB GATE: the ArcadeDB chain is its own chain (Tier 1 only, passes 1..N in order), generated with `--after` the last comparator stage, so the comparators run the schedule first. `00` waits after its `--after` for the marker file `$HOME/conf/go-arcadedb-2611` on the host (it never creates it; it sits before the ALL-DONE trap, so a killed waiter hands nothing on), and every ArcadeDB lane stage aborts without it. The user or the agent touches that file when `CURRENT_PIN` moves to 26.11.1, and not before; `--check --drafts` lints that the gate is in the pair install and in every ArcadeDB stage and in no comparator stage. +The comparators are the default mode and run on the release pair already installed (`ARCADEDB_WHEEL`, `ARCADEDB_SERVER_IMAGE`, `ARCADEDB_ENGINE_COMMIT`): `python3 make_2611_stages.py --after --status-file STATUS.txt --id-prefix qS --out DIR`. `--after` is required, with no default (a marker name, or `none`), and the first stage waits for `grep -qx " ALL-DONE"`, each later one for the stage before it. The marker must be the LAST item still to run, one that writes its line whether it runs or is skipped: a marker already in the machine's STATUS.txt makes the first stage start at once, beside the running jobs (2026-10-23: `qMA06 ALL-DONE`, written by a skipped study). So `--status-file` is required with a marker (a copy of that STATUS.txt, or `none` when there is no queue) and the generator refuses a marker the file already holds; `--check --drafts DIR --status-file STATUS.txt` makes the same test on drafts that exist. The subset is emitted in ENGINE TIERS in the user's exact pass order (2026-10-24; `--tiers`, on by default, `--no-tiers` for the plain rep-major chain). N defaults to 5, the target is 3 (the paper profile), and the order is `schedule_order`: n=1 T1, n=2 T1, n=3 T1, n=1 T2, n=2 T2, n=3 T2, n=4 T1, n=4 T2, n=5 T1, n=5 T2. Every prefix of it is a complete set of equal replicates for the tiers covered so far (each tier has run passes 1..k, none skipped, and a pass is whole before the next starts); `--reps 3` stops after the first six groups, which is the chain of Tier 1 passes 1-3 and then Tier 2 passes 1-3 as before. Tier 1 is ArcadeDB (the four 26.11.1 arms, when `--arcadedb-release` adds them), SurrealDB, ArangoDB, PostgreSQL with pgvector and AGE, MongoDB with Search, the composed Qdrant plus Neo4j stack, and the embedded references SQLite, LadybugDB and DuckDB; Tier 2 is Memgraph, Neo4j and Qdrant on their own, Redis 8, CrateDB and TimescaleDB (`TIER_ENGINES`, with `ARM_TIER` for TimescaleDB, an arm of the `postgresql` engine that the user lists in Tier 2). Every stage holds the arms of one tier, and stage ids stay one run of consecutive numbers (a stage that mixed both tiers is split in two, so the stage count grows, and each stage repeats its guards and image checks; past 99 stages the ids take three digits, so `qS100` cannot sort before `qS11`). `--check` prints the schedule and proves the order and the prefixes (the plan collapsed into (tier, pass) groups equals `schedule_order`, each group holds the whole pass), and `--check --drafts DIR` lints a draft folder for the same. An arm with no tier runs after Tier 2 in each block. The ArcadeDB arms are Tier 1 but not in this chain: DBBench keeps one ArcadeDB version per host, so they run only once the benchmark pin is 26.11.1 (next paragraph). Inside a tier the order is the one below: a pass is pass k of every stage of the tier (core lanes, then the second list), and each stage runs ONE repetition (`--only-reps $PASS`). With `--no-tiers` the chain is pass 1 of every stage, then pass 2, up to pass N. If time runs out, every engine of a tier has the same n; repetitions days apart show drift. A stage whose environment declares fewer reps (`REPS=k`) is left out of the later passes. Inside a lane the engines run in `ENGINE_PRIORITY` (SurrealDB, ArangoDB, PostgreSQL, MongoDB, Redis 8, composed Qdrant+Neo4j, Neo4j, Qdrant, DuckDB, SQLite, Memgraph, LadybugDB, CrateDB; ArcadeDB's arms lead it when they join; Redis and CrateDB are skipped until their adapters are in `ENGINES`). The rep-1-first rule and the cap rule hold across passes: a pass above 1 runs a cell only when rep 1 of it was clean, and only a timeout of rep 1 censors the larger sizes. WHICH REPS EXIST is `~/rep-state-2026-11-.tsv` on the host (`-jvm` or `-native` is added to the name in an ArcadeDB release chain, because both chains label their cells identically; one `label, rep, ok|fail` line per finished repetition; the rows themselves carry the rep number in the runner's results file): stop after any pass, and the next pass, or a restarted stage, picks up from it with no regeneration; N can go to 3 or stop at 2. Delete that file to start a chain over. The ArcadeDB pass is `--arcadedb-release X.Y.Z` with `--engine-commit`, `--image-digest sha256:...`, `--wheel-name`, `--wheel-sha256` and `--wheel-url`; it refuses unless all five are given, because the release may not exist when the drafts are made. It writes `00`, the pair install (official image by digest, wheel by sha256, `build_matched_pair.sh`, `verify_pair_c25.sh`), and ArcadeDB's lane stages behind it. THE ARCADEDB GATE: the ArcadeDB chain is its own chain (Tier 1 only, passes 1..N in order), generated with `--after` the last comparator stage, so the comparators run the schedule first. `00` waits after its `--after` for the marker file `$HOME/conf/go-arcadedb-2611` on the host (it never creates it; it sits before the ALL-DONE trap, so a killed waiter hands nothing on), and every ArcadeDB lane stage aborts without it. The user or the agent touches that file when `CURRENT_PIN` moves to 26.11.1, and not before; `--check --drafts` lints that the gate is in the pair install and in every ArcadeDB stage and in no comparator stage. The ArcadeDB pair's guards are split by what a stage runs. Every stage exports the pinned wheel (`ARCADEDB_WHEEL`, with `ARCADEDB_SERVER_IMAGE` and `ARCADEDB_ENGINE_COMMIT`) before `build_images.sh`, which bakes it into every generic image (client, duckdb, dense), and so every stage guards that the file exists at the pinned path and has the pinned sha256; `build_images.sh` copies it from that path and nothing in it or the Dockerfiles reads `dist/` itself. Only a stage that runs an ArcadeDB arm (the `--arcadedb-release` pass) also guards the rest: exactly one wheel in `dist/` (it protects `verify_pair_c25.sh`), the server image, the wheel and JPype read back out of `dbbench:arcadedb`, and `verify_pair_c25.sh`. A comparator stage on a host whose `dbbench:arcadedb` is the previous release's therefore runs. `build_images.sh` has no PyPI fallback: it refuses to build the `arcadedb` image (named or by default) without `ARCADEDB_WHEEL`, and refuses a wheel that is not a file, and refuses a wheel whose `arcadedb_embedded/core.py` lacks `_INSERT_MANY_CHUNK` (`wheel_guard.py`: before 26.10.2 `insert_many` parsed the whole input into one Java array, which the kernel killed on l4/ts1000/arcadedb_ts_doc on 2026-10-08; use a 26.10.2 or newer wheel); the images that take no wheel can be built alone by naming them. Every ABORT hands on: the `ALL-DONE` trap is set right after the wait, before the first guard, so a stage that aborts before its START line still writes its marker and the next stage runs (its own guards then run too). The ABORT line is in `STATUS.txt`, where `mini_watch.sh` reports it; to resume, fix the cause and run the aborted stage's script again by hand after the chain, and landing reads only the rows that exist. `make_2611_stages.py --check --drafts DIR` lints both rules. The default order is the core block first, then (under `--profile full`) the second list. Inside the core block the large tiers (deep10m, tpch10, the full SF1 network) run after the small ones, and the cells that dominate the hours (Memgraph's deep10m dense cells, which the paper profile does not stage, SurrealDB's documents at tpch1, the full SF1 network's analytics) run last of all in stages of their own, where they are easiest to drop if time runs short; `BIGGEST` in the file names them, and `--order paper` keeps the table order. Splitting moves stage boundaries only, and `--check` proves the cells are the same. `--project TABLE` prints projected hours per drafted stage from a per-cell hours table (tab separated: lane, workload, scale, backend, class, hours at 3 repetitions, hours at 5 repetitions; a cell with no clean rep carries its one failed attempt). `--check --drafts DIR` lints written scripts: the instrument exported and never 2026-10, the 2611 rows file, only subset arms, each stage waiting for the one before it. Drafts are written to a folder outside the tree and copied to the bench host by hand on the owner's go. Which stage is running is state, not method: it is in `STATUS.txt`'s tail and in HANDOFF, never here. `test_stage_generator_2611.py` holds the written drafts byte for byte against `stage_drafts_2611.sha256`, so a stage cannot move without the list changing in the same commit. diff --git a/benchmarks/experiments/bench_common.py b/benchmarks/experiments/bench_common.py index 8eccb8eb3f8..8a8d0ac18c5 100644 --- a/benchmarks/experiments/bench_common.py +++ b/benchmarks/experiments/bench_common.py @@ -41,6 +41,24 @@ def instrument_at_least(version): return INSTRUMENT >= version +# THE GRAPH ANALYTICAL VIEW MUST LIST THE `Message` SUPERTYPE FROM THIS RELEASE (upstream ArcadeData/arcadedb PR 9320, issue 9301). +# Since that change `GraphTraversalProviderRegistry.findProvider` hands a view to a walk only when the view covers EVERY vertex type of +# the schema (`coversVertexType(null)`, which also counts an abstract supertype such as Message), so a view that leaves Message out is +# silently not used for walks. One place holds the release so the lane that builds the view (l2_graph._gav_types) and the exporter that +# refuses a stale row (export_web._refuse_gav_without_supertype) cannot disagree. +GAV_SUPERTYPE_LISTED_FROM = (26, 11, 1) + + +def engine_release(version): + """(major, minor, patch) of an engine version string the harness records, or None when it carries none. + + Reads the string a row already stamps as `engine_version` ("26.11.1", "server:26.11.1", "26.11.1-SNAPSHOT (build ...)"); never a wheel file + name. A SNAPSHOT of release X counts as X, as the exporter does.""" + import re + m = re.search(r"(\d+)\.(\d+)\.(\d+)", str(version or "")) + return tuple(int(x) for x in m.groups()) if m else None + + # WHAT THE INSTRUMENT MEASURES (CAMPAIGN section 7 rows 72, 81, 82, 84 and the note under that table). These are the method, # not switches: an untimed sparse warm-up on queries disjoint from the timed set, the CPU idle-state hold at the lowest latency # target, an index on the dense lane's vector id, the embedded ArcadeDB JVM's initial heap left unset, large l4 results read on diff --git a/benchmarks/experiments/export_web.py b/benchmarks/experiments/export_web.py index 60b9702a557..1ab998d964f 100644 --- a/benchmarks/experiments/export_web.py +++ b/benchmarks/experiments/export_web.py @@ -1925,7 +1925,7 @@ def _withhold_cells(tables): # stale list. A tier with no message half covers Person and KNOWS only and is not concerned. _GAV_TYPES_FIELD = "gav_types" _GAV_SUPERTYPE = "Message" -_GAV_SUPERTYPE_LISTED_FROM = (26, 11, 1) +_GAV_SUPERTYPE_LISTED_FROM = bench_common.GAV_SUPERTYPE_LISTED_FROM # one release for the lane that builds the view and this refusal def _gav_message_rows(rows): diff --git a/benchmarks/experiments/l2_graph.py b/benchmarks/experiments/l2_graph.py index 27f2be06cc7..65c8c3dafaa 100644 --- a/benchmarks/experiments/l2_graph.py +++ b/benchmarks/experiments/l2_graph.py @@ -279,17 +279,29 @@ def build(self, n_persons): # Person and KNOWS, so no LSQB query could use it. At a tier with no message half the lists are what is loaded # (Person, KNOWS: the view the five projection questions always had); at the full network they are Person plus # ldbc_snb.MSG_VERTEX_LABELS and KNOWS plus MSG_EDGE_TYPES (Message is the abstract supertype of Post and Comment - # and holds no record of its own, so it is not listed, as upstream's runner does not list it). The properties are + # and holds no record of its own, so it is not listed before ArcadeDB 26.11.1, where upstream PR 9320 makes the registry hand a view out only + # when it covers every vertex type, abstract ones included: from there `_gav_types` adds it). The properties are # the five questions' own. An ArcadeDB-only setting, the user's call; the no-view arm (BENCH_GAV=0, backend_arm # nogav) stays on the table as the like-for-like row, and the build time of the larger view is priced on the row. GAV_PROPERTIES = "PROPERTIES (id, name, age, city) EDGE PROPERTIES (since)" def _gav_types(self): - """(vertex types, edge types) the view is created over.""" + """(vertex types, edge types) the view is created over. + + From ArcadeDB 26.11.1 (upstream PR 9320) the registry hands a view to a walk only when it covers every vertex type of the schema, the + abstract `Message` supertype included, so the list carries `Message` there and not on the releases before (bench_common. + GAV_SUPERTYPE_LISTED_FROM). The release is the engine version this adapter already records on its row (`self.version`, set at + connect); one the harness cannot read is refused rather than guessed, since the wrong list is a wrong measurement.""" if not getattr(self, "_load_messages", False): return ["Person"], ["KNOWS"] import ldbc_snb as _ldbc - return ["Person"] + list(_ldbc.MSG_VERTEX_LABELS), ["KNOWS"] + list(self.MSG_EDGE_TYPES) + release = bench_common.engine_release(self.version) + if release is None: + raise RuntimeError(f"cannot choose the view's vertex types: the engine version {self.version!r} carries no release number") + vertices = ["Person"] + list(_ldbc.MSG_VERTEX_LABELS) + if release >= bench_common.GAV_SUPERTYPE_LISTED_FROM: + vertices.insert(1, "Message") + return vertices, ["KNOWS"] + list(self.MSG_EDGE_TYPES) def _gav_ddl(self): v, e = self._gav_types() diff --git a/benchmarks/experiments/make_2611_stages.py b/benchmarks/experiments/make_2611_stages.py index cd35ad01ab1..5981f813799 100644 --- a/benchmarks/experiments/make_2611_stages.py +++ b/benchmarks/experiments/make_2611_stages.py @@ -1507,7 +1507,7 @@ def has_arcadedb_arm(backends): # line per finished repetition (