From 0746485d29c3b77255fb230fc9bfb56a3799066f Mon Sep 17 00:00:00 2001 From: Taewoon Kim Date: Wed, 4 Nov 2026 01:01:48 +0900 Subject: [PATCH 1/3] benchmarks: the PostgreSQL-wire twin of the served SQL cells, with an answers-identical gate DECISIONS #166/#171/#237(3): the served SQL cells (l1, l1tpc, l3s, l3d, l4) run a third time as a twin job whose /query SQL goes over ArcadeDB's postgres plugin through psycopg (never prepared, autocommit); writes, DDL, transactions and bulk loads stay on HTTP. Role stays `sensitivity` so landing and export keep the headline unchanged; `arcadedb_client_wire`=pgwire splits it from the lean twin in the freeze key and the exporter. The twin's server starts the plugin with explicit settings read back from the engine (the plugin list is matched as a tail: the server prepends AutoBackupSchedulerPlugin). pgwire_gate.py compares each res__digest with the headline's (the lane's own coercions) and keeps a mismatching row with `error` set. A resume no longer treats a twin as done because its headline is. Co-Authored-By: Claude Opus 5.5 --- benchmarks/experiments/CAMPAIGN.md | 4 +- benchmarks/experiments/PROTOCOL.md | 2 +- benchmarks/experiments/export_web.py | 13 + benchmarks/experiments/lean_http.py | 230 ++++++++++- benchmarks/experiments/make_paper_tables.py | 2 + benchmarks/experiments/pgwire_gate.py | 122 ++++++ benchmarks/experiments/runner.py | 156 +++++++- .../experiments/test_instrument_method.py | 2 +- benchmarks/experiments/test_pgwire_twin.py | 364 ++++++++++++++++++ 9 files changed, 876 insertions(+), 19 deletions(-) create mode 100644 benchmarks/experiments/pgwire_gate.py create mode 100644 benchmarks/experiments/test_pgwire_twin.py diff --git a/benchmarks/experiments/CAMPAIGN.md b/benchmarks/experiments/CAMPAIGN.md index 423d995d76c..2d042637c44 100644 --- a/benchmarks/experiments/CAMPAIGN.md +++ b/benchmarks/experiments/CAMPAIGN.md @@ -150,7 +150,7 @@ Rules every campaign keeps: | 3 | Dense ingest through `graph_batch(use_wal=True)` with Java float arrays | ArcadeDB only | **built (DECISIONS #116; 1.4x on the laptop)**: `l3d_dense.py` | DECISIONS #116 item 3 | | 4 | The served document and vector arms load through `INSERT ... CONTENT :rows` with the batch bound as one parameter | ArcadeDB only | built (DECISIONS #116, ArcadeData/arcadedb#8337): `l1_tpc.py` (`ArcadeServerTPC.load_batch`, `BENCH_SERVED_LOAD_BATCH`), `l3d_dense.py`, `l3_sparse.py`; parity, since every comparator loads through its vendor's bulk path. **No batch-size sweep is recorded** | `.notes` repros/served-doc-ingest; BUGS F151, F152 | | 5 | The strict durability setting is ArcadeDB `txWalFlush=1`, matched by effect to SQLite `synchronous=FULL`, not `txWalFlush=2` | ArcadeDB only | built (the user, DECISIONS #147; own issue #138): `bench_common.ARCADE_STRICT_TX_WAL_FLUSH`, `test_strict_durability.py`; the October string is kept for rows measured at 2; PROTOCOL section 7 has the row | `concepts/transactions.adoc` in upstream's docs; strace of the three settings on the laptop, 2026-09-24 | -| 6 | Served SQL arms over the Postgres wire (psycopg) instead of HTTP (`requests`) | ArcadeDB only | unblocked: #8288 fixed in ArcadeData/arcadedb#8308, and pgjdbc's default prepare in #8315 (#8244), both merged for 26.10.1 and verified; **decided 2026-09-26 (DECISIONS #116): measure on the bench host at the re-pin and switch if it is the faster read path** (the laptop cannot time below a millisecond; the maintainer already says it is, on #8332). **Still open:** the served arms read through the lean HTTP client (row 72) now, and the comparison on mini is still to run (DECISIONS #162 item 3) | `.notes` repros/protocol-latency | +| 6 | Served SQL arms over the Postgres wire (psycopg) instead of HTTP (`requests`) | ArcadeDB only | unblocked: #8288 fixed in ArcadeData/arcadedb#8308, and pgjdbc's default prepare in #8315 (#8244), both merged for 26.10.1 and verified; **decided 2026-09-26 (DECISIONS #116): measure on the bench host at the re-pin and switch if it is the faster read path** (the laptop cannot time below a millisecond; the maintainer already says it is, on #8332). **Built for the 26.11.1 four-arm pass (DECISIONS #166, #171, #237 (3)):** the served SQL cells run a third time as a twin job over the wire (`runner.PGWIRE_SQL_ARMS`, run_id ending `_clientpgwire`, `lean_http.PgSession` over psycopg, never prepared, autocommit), a labelled sensitivity row and never a headline (it flatters ArcadeDB most, #166 addendum 2). Cells: l1 and l1tpc `arcadedb_server` (oltp, olap), l3s `arcadedb_sparse_server` and `_fp32`, l3d `arcadedb_dense_server` and `_int8`, l4 `arcadedb_ts_doc_server` and `arcadedb_ts_native_server`; not l2 (Cypher stays on HTTP, #162), e2, lifecycle, restart, e4. Only `/query` SQL goes over the wire: every write, DDL, transaction, bulk load and probe stays on HTTP (#171 (c)), so a twin row's write-phase numbers are lean HTTP and are not quoted. The twin's server starts the postgres plugin with explicit settings (`runner.PGWIRE_SERVER_SETTINGS`), read back from the engine and refused if unlisted or different. Answers-identical gate (`pgwire_gate.py`): each `res__digest` of the twin must equal the headline's for the same cell, under the lane's own declared coercions, else the row is kept with `error` set and the reason recorded; the smoke is in the PR body. Still open: the mini run and the numbers | `.notes` repros/protocol-latency | | 7 | What the memory column means: the peak, the working set after a full GC, or the peak with the heap printed beside it | every JVM arm | **decided (the user, DECISIONS #116 item 7): the peak, what an operator has to provision, with the heap setting printed beside it for JVM engines; built** (a `JVM heap` column from each cell's rows in `export_web`; the JVM memory note on every table that shows a JVM arm, BUGS F135). **No generated sentence may phrase the column as efficiency ("uses less memory"): a JVM's peak is the heap it was given; the like-for-like answer is the memory sweep.** **Still open, the memory sweep (the user, 2026-09-27), planned after the chain and the re-pin; it is an instrument change:** the smallest memory each arm completes a cell with. Design, settled before it runs: JVM arms (ArcadeDB both deployments, Neo4j) sweep the heap at a fixed cap, non-JVM arms the container cap; "completes" means the same answer digest within a stated slowdown of the full-memory run; one or two tables first (documents SF10, graph SF10); mini only, one pin; it costs days of bench time | DECISIONS #116 item 7 | | 8 | The sparse corpus generator yields NumPy, and each adapter converts in its own timer | comparators too (same work, moved) | measured 3.7% of ArcadeDB's build; do it with the next sparse measurement | bullet below | | 9 | The re-pin build: the official 26.10.1 with every fix we filed verified by its repro, comparators to latest stable (DECISIONS #129, relaxed by #163 to #165) | routine (steps 3 and 4 of the routine) | **built and verified.** The pin is the release's own jar commit `d36b4ca3a`; each fix we filed was verified by its repro on the laptop (the list is `.notes` issues-trail), and #8626, #8852, #8797, and #8725 landed after and are expected to move published cells. **Read the rows with these in hand, and say so on the page:** some cells get SLOWER (the embedded cross-model strict cells, whose `LSM_VECTOR` build had switched that thread's WAL off, #8129 fixed by #8221, and the sparse table's int8 arms, #8582); the dense and cross-model tables moved with the vector library (#8518), so compare recall at the same settings before reading a latency change; ranks can hold while times drop, because item 2 binds values on every engine. Laptop ratios are not projections of the page; only the re-measure decides. **Still owed:** re-verify #7146, #7183, #7184, and #7190 at 10M on the 26.10.1 pair; #7260 is fixed in 26.11.1 (PR #9289) and not in 26.10.1 | `.notes` issues-trail | @@ -221,7 +221,7 @@ Rules every campaign keeps: | 77 | ArcadeDB's query heap budget (`arcadedb.queryMaxHeapRAM`, default half the JVM heap) raised for the cells the default refuses: the l1tpc tpch10 olap `top_parts` GROUP BY in both ArcadeDB arms (DECISIONS #175, ArcadeData/arcadedb#9402) | ArcadeDB embedded and served, the documents lane, ONLY in the repair stage | **mechanism written, default OFF (humemai/arcadedb-embedded-python#260); the repair stage is generated after qP09 and run before the tier 1 to tier 2 boundary.** `BENCH_ARCADE_QUERY_MAX_HEAP_RAM_MB` adds the JVM option to the embedded arm and to the served arm's JAVA_OPTS and stamps `arcadedb_query_max_heap_ram_mb` and a source on the row; `override: arcadedb_query_ram`. Which rows are the headline (the default rows with the refusal shown, or the raised rows as an escape hatch like row 21's cap and row 75's body limit) is decided at landing. | not an instrument change: unset, nothing differs | | 78 | The comparator client paths are recorded per row (issue #261; DECISIONS #173, #174): `neo4j_rust_ext`, `redis_hiredis`, `es_json_serializer`, and `pg_client_path` (`autocommit`, `copy_binary`, `binary_cursor`); `qdrant_transport` (#172) was already stamped | comparators only: no ArcadeDB arm | built (DECISIONS #173, #174) | issue #261 | | 81 | The sparse lane warms up before its timed pass and the runner holds the CPU idle state for each timed cell (DECISIONS #177). Text on the page too: CPU idle states are not controlled on the host, and single-client times under a millisecond include the cores' wake-up latency (PAGE-SPEC section 0a, `setup.cpu_idle_note`) | the sparse lane (warm-up); every lane's rows (hold) | **built (humemai/arcadedb-embedded-python#278).** `bench_common.SPARSE_WARMUP_DEFAULT` untimed searches per engine, 1,000, one full round as the study behind the warm-up used, drawn from queries DISJOINT from the timed set so a query or result cache cannot flatter an engine: the dataset's extra queries when it holds them, else a fixed-seed sample of documents used as queries; the row stamps `sparse_warmup_queries` as run and `sparse_warmup_source` = `extra_queries` or `document_sample`. The hold is `/dev/cpu_dma_latency` at `BENCH_CPU_DMA_LATENCY_US` for the length of each timed cell: 10 microseconds in the 2026-11 pass, because the tree default `bench_common.CPU_DMA_LATENCY_DEFAULT_US` (0, no sleep state deeper than polling) kept every core in a busy-wait loop and made the host hot and slow (PROTOCOL.md section 7 has the A/B); it needs write access to the device on the host, and a hold that cannot be taken never fails the cell: rows stamp `cpu_dma_latency_us`, `cpu_dma_latency_held`, and `cpu_dma_latency_error` when it failed. `BENCH_SPARSE_WARMUP=N` and `BENCH_CPU_DMA_LATENCY_US=N` set the count and the target (PROTOCOL.md section 7); neither can be turned off. `override: sparse_warmup`, `override: cpu_dma_latency`; `test_idle_and_warmup.py`, `test_instrument_method.py` | part of the instrument | -| 82 | The served ArcadeDB headline is the vendor's own `arcadedb-driver` 0.2.0 (sync, httpx; `lean_http.OfficialSession`, stamp `arcadedb-driver 0.2.0 (sync, httpx)` with `arcadedb_http_client_role` = `headline`), and the lean HTTP client (`lean_http.LeanSession`) is the labelled sensitivity row | every ArcadeDB SERVED arm that talks HTTP through `lean_http.Session()`; no comparator client moves | **built (humemai/arcadedb-embedded-python#282, the instrument's client row).** The sensitivity row is the same cell run again as a twin job (`runner.ARCADEDB_HTTP_ARMS`, run_id ending `_clientlean`, `arcadedb_http_client_role` = `sensitivity`; the runner sets `BENCH_ARCADEDB_HTTP_CLIENT=lean` for it). `lean` is the client the 26.10.1 pass and the published October rows used, so the sensitivity row stays comparable with them; rows measured with the two clients are never mixed in one column. The driver carries `/query` and `/command` calls; the `GET /server` probe, the `/batch` bulk load, transactions, NDJSON streams and bodies with a field the driver has no argument for stay on the lean client. The PostgreSQL wire has no adapter in this harness, so its sensitivity row is not built. The `client` image gains `arcadedb-driver==0.2.0` at its next rebuild (`build_images.sh`, COMPARATORS.md pin table). Measured on the official 26.10.1 image (d36b4ca3a), 7 interleaved rounds: the bound Cypher point read costs the client 55.5 us of CPU per call with `lean` against 11.6 for a bare-socket variant that was tried and retired, identical answer digests; the 100-row read is engine-bound and moves less | part of the instrument | +| 82 | The served ArcadeDB headline is the vendor's own `arcadedb-driver` 0.2.0 (sync, httpx; `lean_http.OfficialSession`, stamp `arcadedb-driver 0.2.0 (sync, httpx)` with `arcadedb_http_client_role` = `headline`), and the lean HTTP client (`lean_http.LeanSession`) is the labelled sensitivity row | every ArcadeDB SERVED arm that talks HTTP through `lean_http.Session()`; no comparator client moves | **built (humemai/arcadedb-embedded-python#282, the instrument's client row).** The sensitivity row is the same cell run again as a twin job (`runner.ARCADEDB_HTTP_ARMS`, run_id ending `_clientlean`, `arcadedb_http_client_role` = `sensitivity`; the runner sets `BENCH_ARCADEDB_HTTP_CLIENT=lean` for it). `lean` is the client the 26.10.1 pass and the published October rows used, so the sensitivity row stays comparable with them; rows measured with the two clients are never mixed in one column. The driver carries `/query` and `/command` calls; the `GET /server` probe, the `/batch` bulk load, transactions, NDJSON streams and bodies with a field the driver has no argument for stay on the lean client. The PostgreSQL wire is the second sensitivity row (row 6): the served SQL cells also run as a `_clientpgwire` twin (`arcadedb_client_wire` = `pgwire`, role `sensitivity`), the read statements only. The `client` image gains `arcadedb-driver==0.2.0` at its next rebuild (`build_images.sh`, COMPARATORS.md pin table). Measured on the official 26.10.1 image (d36b4ca3a), 7 interleaved rounds: the bound Cypher point read costs the client 55.5 us of CPU per call with `lean` against 11.6 for a bare-socket variant that was tried and retired, identical answer digests; the 100-row read is engine-bound and moves less | part of the instrument | | 84 | The dense lane's id index for the delete (humemai/arcadedb-embedded-python#291, DECISIONS #178 index parity): ArcadeDB, pgvector, Neo4j, Memgraph, FalkorDB, DuckDB, and LanceDB get an index on the vector id before or right after the load, the other engines already delete through their own key | the dense lane (l3d), every arm | **built.** Every dense row stamps `dense_id_index` (the statement, or `key`); `override: dense_id_index`; `test_dense_id_index.py`. The index build is inside `ingest_s`. The /next page says ArcadeDB's delete ran without an id index in rows that lack the stamp (`_dense_id_index_note`, list line `dense_id_index`; `test_next_measurement_wording.py` holds the wording) | part of the instrument | | 85 | Streamed results for the served ArcadeDB time-series arms, `BENCH_ARCADEDB_TS_NDJSON=1`, default OFF: both served arms ask the query endpoint for NDJSON and read the records line by line, as the PostgreSQL and MongoDB arms read rows as their server sends them | the time-series lane (l4), `arcadedb_ts_doc_server` and `arcadedb_ts_native_server` only | **mechanism written, default OFF; MEASURED 2026-10-09 and NOT USED at 26.11.1:** at ts100 the switch helped the documents arm (0.853 [0.713, 0.928]) and hurt the native arm (1.087 [1.083, 1.088]), identical rows, so it stays off and the page promises nothing; it needs a reader that parses as data arrives and a ts1000 re-measure first. Unset, nothing differs (`test_ts_ndjson.py`); set, rows stamp `arcadedb_ts_result_format` = `ndjson`; `override: ts_ndjson` | not an instrument change: unset, nothing differs | | 87 | Two failures of the 26.10.1 pass that were not results (humemai/arcadedb-embedded-python#344, the failure triage of 2026-10-20): (1) `chroma_dense` never ran, because the override registry `overrides.py` shadowed chromadb's dependency `overrides` on `/work` (`ImportError: cannot import name 'overrides'`); the registry is now `override_registry.py`. (2) Memgraph's storage access timeout (default 1 s) refused the deep10m vector index build in both dense arms; the dense arms now run `--storage-access-timeout-sec=172800`, longer than any cell cap | Chroma dense (every scale) and Memgraph dense (both precisions); no other arm moves (the rename changes no measurement) | **built on branch issue/344-chroma-memgraph, not on main**: `override_registry.py` and every import, `runner.MEMGRAPH_STORAGE_ACCESS_TIMEOUT_S`, the read-back `memgraph_storage_access_timeout_s`, PROTOCOL.md section 7, COMPARATORS.md. Mini picks it up at the `git pull` of the stage after the merge, which is the user's to time; a repair stage reruns Chroma dense and Memgraph dense deep10m | `.notes/bench/repros/failure-triage-20261020` | diff --git a/benchmarks/experiments/PROTOCOL.md b/benchmarks/experiments/PROTOCOL.md index fc343a5a8d0..1fe961e8fdc 100644 --- a/benchmarks/experiments/PROTOCOL.md +++ b/benchmarks/experiments/PROTOCOL.md @@ -263,7 +263,7 @@ An override nobody can find is a defect, whichever way it moves the number. Ever | all dense arms | `EF_CONSTRUCTION=100`, `EF_SEARCH=100` | operating-point | Shared beam. At the 26.10.1 tag an explicit `efSearch` is a fixed beam of `max(k, efSearch)` while the adaptive default is `max(2k, 20)` on graphs of 10,000 or more vectors, so it is NOT a no-op for ArcadeDB; some comparators move down from their own defaults, unaudited. | lane docstring only | | ArcadeDB server (all arms) | `ARCADEDB_OPTS_GC=` is cleared explicitly, and `JAVA_OPTS` carries `-XX:+UseCompactObjectHeaders` | operating-point | Makes E4 a transport comparison. `ARCADEDB_OPTS_GC` survives a `JAVA_OPTS` override, so clearing it puts both arms on one collector. | DECISIONS #54, enforced by `verify_pair_c25.sh` | | ArcadeDB server vs embedded | both arms run ONE set of upstream jars on ONE JVM (Corretto 25) | fairness | The axis isolates transport, not JVM major, libc, collector, or object-header layout; `-XX:+UseCompactObjectHeaders` is rejected by JDK 21. | DECISIONS #54, enforced by `verify_pair_c25.sh` (jars AND JVM major on both sides) | -| ArcadeDB SERVED arms | the headline HTTP client is the official `arcadedb-driver` 0.2.0 (sync); the same cell runs again on the lean HTTP client as a twin job | the vendor's own client, the same class as the comparators' official clients | Protocol audit round 3: the driver costs about 3x the lean client on a one-row read, so a served latency claim names the client. Bulk loads, the server probe, and transactions stay on the lean HTTP client. | row fields `arcadedb_http_client` and `arcadedb_http_client_role` (`headline` or `sensitivity`) | +| ArcadeDB SERVED arms | the headline HTTP client is the official `arcadedb-driver` 0.2.0 (sync); the same cell runs again on the lean HTTP client as a twin job | the vendor's own client, the same class as the comparators' official clients | Protocol audit round 3: the driver costs about 3x the lean client on a one-row read, so a served latency claim names the client. Bulk loads, the server probe, and transactions stay on the lean HTTP client. The served SQL cells (l1 and l1tpc documents, l3s sparse, l3d dense, l4 time series; not Cypher, not l2 or e2) run a third time as a PostgreSQL-wire twin (psycopg, unnamed statements, `_clientpgwire`): a labelled sensitivity row, never a headline. Only the read and query statements are timed on the wire; writes, DDL, transactions and bulk loads stay on HTTP, so a wire row's write-phase numbers are lean HTTP and are not quoted. The twin's server runs the postgres plugin with explicit, read-back settings, and the twin row passes only if every query digest equals the headline's under the lane's declared coercions (`pgwire_gate.py`); a mismatch keeps the row, marked failed with the reason. | row fields `arcadedb_http_client` and `arcadedb_http_client_role` (`headline` or `sensitivity`); on a wire row also `arcadedb_client_wire`, `arcadedb_pgwire_client`, `arcadedb_pgwire_scope`, `arcadedb_pgwire_calls`, `arcadedb_pgwire_fallback_reasons`, `arcadedb_pgwire_port_open`, `arcadedb_pgwire_gate` and the plugin's settings in `server_settings` | | l4 `q_high` and `q_groupby` | ArcadeDB embedded `to_arrow()`, DuckDB `to_arrow_table()`, SQLite `fetchall()` (no columnar path); `to_list()` / `fetchall()` timed beside it as `_plain_ms` | each engine on its documented best path for a result of thousands of rows | The bindings' docs recommend a bulk read for large results; small results (1 to 100 rows) are unchanged. | row fields `read_path`, `read_path_queries`, `read_path_plain`, `read_path_sensitivity`, `_plain_ms`, `_plain_digest`, `_plain_digest_match` (the cell ends if the typed and plain digests differ; DuckDB's Arrow rows are read by position, like its `fetchall()`) | | JVM initial heap (`-Xms`) per engine | **ArcadeDB, embedded and served: unset** (only `-Xmx` from the cell's memory budget; the served arms through `ARCADEDB_OPTS_MEMORY=-Xmx{heap}`, DECISIONS #200). Elasticsearch: `ES_JAVA_OPTS=-Xms{heap} -Xmx{heap}`, unchanged. Neo4j: `server.memory.heap.initial_size` = `max_size`, unchanged. QuestDB: the harness sets no heap (image defaults), unchanged | ArcadeDB as its maintainers advise for a container with a fixed memory budget (no `-Xms`; ArcadeData/arcadedb#9167) and as the bindings document, so both deployments run one JVM configuration; each other server engine as its own documentation recommends | ArcadeDB's server script (`package/src/main/scripts/server.sh`) leaves the heap to the JVM unless `ARCADEDB_OPTS_MEMORY` is set and shows `-Xms2G -Xmx2G` as its example; Elasticsearch: set `Xms` and `Xmx` to the same value (https://www.elastic.co/guide/en/elasticsearch/reference/current/advanced-configuration.html#set-jvm-heap-size); Neo4j: set initial and maximum heap to the same value (https://neo4j.com/docs/operations-manual/current/performance/memory-configuration/); QuestDB: no setting by the harness (https://questdb.com/docs/configuration/). | `server_heap` / the cell's heap fields, as before | | ArcadeDB server (all arms) | `ARCADEDB_OPTS_MEMORY=-Xmx{heap}` (image ships 2G) | resource fitting | Heap parity per tier. The 2 GB default cost the server 15% p50 and 35% build time at 10M. | T5 caption, FAIRNESS.md F3, `server_heap` read back | diff --git a/benchmarks/experiments/export_web.py b/benchmarks/experiments/export_web.py index 1ab998d964f..9ce7c70a00d 100644 --- a/benchmarks/experiments/export_web.py +++ b/benchmarks/experiments/export_web.py @@ -4986,6 +4986,9 @@ def _twin_client(rs): return named.most_common(1)[0][0] or None if named else None +_PG_TWIN_ROWS = [] + + def _arcadedb_client_note(table): """What the served ArcadeDB rows ran through, from the rows: the official driver is the headline and the lean HTTP client's twin is printed beside it as a labelled sensitivity (`client_twin` on the entry), never pooled @@ -5003,6 +5006,10 @@ def _arcadedb_client_note(table): if twins: text += (f" The same cells also ran through the benchmark's own lean HTTP client ({_join_and(twins)}); " f"each is printed on its row as a labelled sensitivity and is not part of the headline median.") + if any(str(r.get("backend")) in {str(e.get("backend_key")) for e in table.get("entries") or []} for r in _PG_TWIN_ROWS): + text += (" The served SQL cells also ran with their read statements on ArcadeDB's PostgreSQL wire through psycopg, answers checked identical to the " + "headline's; those rows are a labelled sensitivity kept out of every table and every headline median, and only the read statements " + "are on the wire (writes, bulk loads and transactions stay on HTTP).") return _gen(text, *heads, *twins) @@ -7243,6 +7250,12 @@ def main() -> int: if _twin_rows: _twin_ids = {id(r) for r in _twin_rows} rows = [r for r in rows if id(r) not in _twin_ids] + # THE POSTGRESQL-WIRE TWIN IS A SECOND SENSITIVITY, NEVER POOLED WITH THE FIRST (DECISIONS #166, #237). Its rows carry the same role, so the filters above + # already keep them out of every headline; the wire stamp splits them off here so the lean twin's `client_twin` columns never take a wire row. They are + # only mentioned, in the client sentence of the tables whose served SQL cells ran on the wire. + global _PG_TWIN_ROWS + _PG_TWIN_ROWS = [r for r in _twin_rows if str(r.get("arcadedb_client_wire") or "") == "pgwire" and not str(r.get("error") or "").strip()] + _twin_rows = [r for r in _twin_rows if str(r.get("arcadedb_client_wire") or "") != "pgwire"] # THE CAP RULE, AT REPORT TIME (DECISIONS #188): an arm-cell whose pass-1 median is over its rule cap is shown censored # (_censored_cells) and its numbers are not shown, so its rows leave the tables here. Strict-class rows are the # durability table's and stay. diff --git a/benchmarks/experiments/lean_http.py b/benchmarks/experiments/lean_http.py index c672ed647e8..8bd16385c91 100644 --- a/benchmarks/experiments/lean_http.py +++ b/benchmarks/experiments/lean_http.py @@ -47,25 +47,32 @@ import select import socket import ssl +import threading +import time import urllib.parse import zlib CLIENT_ENV = "BENCH_ARCADEDB_HTTP_CLIENT" LEAN = "lean" OFFICIAL = "official" +PGWIRE = "pgwire" # the PostgreSQL-wire twin of the served SQL arms (DECISIONS #166, #171, #237): SQL reads over psycopg, all else over lean HTTP +PG_PORT_ENV = "BENCH_ARCADEDB_PG_PORT" +PG_DEFAULT_PORT = 5432 +PGWIRE_SCOPE = ("read and query statements only: `POST /query` SQL goes over the PostgreSQL wire; every write, DDL, transaction, bulk load, probe and " + "Cypher statement stays on lean HTTP (DECISIONS #171 (c), #162)") OFFICIAL_PIN = "0.2.0" # the arcadedb-driver release the 2026-11 headline measures; recorded per row from the installed package LEAN_NAME = "lean_http (http.client, keep-alive)" def client_choice(): - """'official' (the headline, the default) or 'lean' (the sensitivity twin), from BENCH_ARCADEDB_HTTP_CLIENT, which the runner - sets for the twin cell; anything else is refused rather than guessed.""" + """'official' (the headline, the default), 'lean' or 'pgwire' (the two sensitivity twins), from BENCH_ARCADEDB_HTTP_CLIENT, which the + runner sets for a twin cell; anything else is refused rather than guessed.""" v = (os.environ.get(CLIENT_ENV) or "").strip().lower() if v in ("", OFFICIAL): return OFFICIAL - if v == LEAN: - return LEAN - raise ValueError(f"{CLIENT_ENV} must be 'official' or 'lean', got {v!r}") + if v in (LEAN, PGWIRE): + return v + raise ValueError(f"{CLIENT_ENV} must be 'official', 'lean' or 'pgwire', got {v!r}") # ------------------------------------------------------------------------------------------------------ exceptions @@ -438,6 +445,201 @@ def close(self): self._fallback.close() +# ------------------------------------------------------------------------------------------------------- the PostgreSQL wire +# THE PG-WIRE TWIN (DECISIONS #166, #171, #237; CAMPAIGN section 7 row 6, PROTOCOL.md "ArcadeDB SERVED arms"). The same served SQL cell run again with +# its read statements on ArcadeDB's PostgreSQL-wire plugin through psycopg, as a labelled sensitivity row beside the headline (the official HTTP driver) +# and the lean HTTP twin. It flatters ArcadeDB most (the lightest client of the three, #166 addendum 2), so it is never a headline. +# SCOPE. Only `POST /query/` with language `sql` goes to the wire. Every `/command` (writes, DDL, `sqlscript` transactions), `/batch` (bulk +# loads, #171 (c)), `/server` probe, any other language (Cypher stays on HTTP, #162), a session id, a statement whose params the wire cannot carry +# (a dict) and an NDJSON stream the lane did not ask the wire for, go to a `LeanSession`; each is counted by reason in `fallback_reasons`. +# NEVER PREPARED (`prepare_threshold=None`): psycopg's default prepares a statement at its 5th execution, and the plugin's prepared path sends rows of +# a schemaless type under one row description (upstream ArcadeData/arcadedb#9009), so every execution is an unnamed Parse/Bind/Execute. +_SQL_NAMED = re.compile(r":([A-Za-z_][A-Za-z0-9_]*)") + + +def pg_translate(sql, params): + """(psycopg SQL, params) for an ArcadeDB statement with `:name` placeholders (the HTTP form every lane writes): each becomes `%(name)s`, a bound + parameter on the wire; text inside quotes and backticks is left alone; with params, a literal `%` is doubled for psycopg. Returns None when a + placeholder has no entry in `params` (the HTTP path then reports the error in the server's own words).""" + params = params or {} + out, i, n, used = [], 0, len(sql), [] + while i < n: + c = sql[i] + if c in "'\"`": + j = i + 1 + while j < n: + if sql[j] == "\\": + j += 2 + continue + if sql[j] == c: + break + j += 1 + seg = sql[i:j + 1] + out.append(seg.replace("%", "%%") if params else seg) + i = j + 1 + continue + m = _SQL_NAMED.match(sql, i) if c == ":" and not (i and (sql[i - 1].isalnum() or sql[i - 1] in "_:")) else None + if m: + if m.group(1) not in params: + return None + out.append(f"%({m.group(1)})s") + used.append(m.group(1)) + i = m.end() + continue + out.append("%%" if c == "%" and params else c) + i += 1 + return "".join(out), {k: params[k] for k in used} + + +def pg_param_ok(v): + """Can this value travel as a bound PostgreSQL parameter: None, a bool, int, float, str, or a flat list or tuple of them.""" + if v is None or isinstance(v, (bool, int, float, str)): + return True + return isinstance(v, (list, tuple)) and all(x is None or isinstance(x, (bool, int, float, str)) for x in v) + + +class PgSession: + """The part of `requests.Session` the lanes use, with a served SQL read sent over the PostgreSQL wire (see the block above); the rest goes through + a `LeanSession`. `pg_calls`, `fallback_calls` and `fallback_reasons` count the two; `summary()` is what the cell's log carries.""" + + client_name = None + + def __init__(self): + import psycopg + from psycopg.rows import dict_row + self._psycopg, self._dict_row = psycopg, dict_row + self.version = getattr(psycopg, "__version__", "unknown") + impl = getattr(getattr(psycopg, "pq", None), "__impl__", "unknown") + self.client_name = f"psycopg {self.version} (PostgreSQL wire, {impl}, unnamed statements)" + self.auth = None + self._conns = {} + self._fallback = LeanSession() + self.pg_calls = 0 + self.fallback_calls = 0 + self.fallback_reasons = {} + + def _port(self): + return int(os.environ.get(PG_PORT_ENV) or PG_DEFAULT_PORT) + + def _conn(self, host, db, auth): + key = (host, db, tuple(auth)) + cx = self._conns.get(key) + if cx is None or cx.closed: + cx = self._psycopg.connect(host=host, port=self._port(), dbname=db, user=auth[0], password=auth[1], autocommit=True, + prepare_threshold=None, row_factory=self._dict_row, connect_timeout=30) + self._conns[key] = cx + if not getattr(self, "_beat", None): + # a lane may exit without closing (os._exit), so the counters reach the cell's log on a clock too + self._beat = threading.Thread(target=self._heartbeat, daemon=True) + self._beat.start() + return cx + + def _heartbeat(self, every=10): + last = None + while True: + time.sleep(every) + cur = self.summary() + if cur != last: + print("PGWIRE_SUMMARY " + _json.dumps(cur, sort_keys=True), flush=True) + last = cur + + def _fall(self, reason): + self.fallback_calls += 1 + if reason not in self.fallback_reasons: + print(f"PGWIRE_FALLBACK {reason}", flush=True) # the first sight of each reason, so the cell's log names it even if the lane exits hard + self.fallback_reasons[reason] = self.fallback_reasons.get(reason, 0) + 1 + + def summary(self): + return {"pg_calls": self.pg_calls, "fallback_calls": self.fallback_calls, "fallback_reasons": dict(sorted(self.fallback_reasons.items()))} + + def _route(self, url, json, data, headers, kw): + """(match, reason): the data-plane match when this call goes to the wire, else None and why not.""" + m = _DATA_URL.match(url) + if not m: + return None, "other-endpoint" + if m["kind"] != "query": + return None, "command" + if not (isinstance(json, dict) and data is None and not kw and not headers): + return None, "headers-or-raw-body" + if not (set(json) <= {"language", "command", "params", "limit"} and "language" in json and "command" in json): + return None, "body-fields" + if str(json["language"]).lower() != "sql": + return None, f"language-{str(json['language']).lower()}" + if not all(pg_param_ok(v) for v in (json.get("params") or {}).values()): + return None, "param-type" + return m, None + + def _run(self, m, json, auth): + """Run one routed statement on the wire; returns (rows, truncated). The rows are dicts, the shape the HTTP answers have.""" + t = pg_translate(json["command"], json.get("params")) + if t is None: + raise _Unroutable("param-missing") + sql, args = t + host = m["root"].split("://", 1)[1].rsplit(":", 1)[0] + cur = self._conn(host, m["db"], auth).execute(sql, args or None) + rows = cur.fetchall() if cur.description is not None else [] + limit = json.get("limit") + truncated = False + if limit and limit > 0 and len(rows) > limit: + rows, truncated = rows[:limit], True + return rows, truncated + + def get(self, url, **kw): + kw.setdefault("auth", self.auth) + self._fall("get") + return self._fallback.get(url, **kw) + + def post(self, url, json=None, data=None, auth=None, headers=None, timeout=None, **kw): + auth = auth or self.auth + m, why = self._route(url, json, data, headers, kw) if auth else (None, "no-auth") + if m is not None: + try: + rows, truncated = self._run(m, json, tuple(auth)) + except _Unroutable as e: + why = str(e) + except self._psycopg.Error as e: + self.pg_calls += 1 + return _OfficialResponse(500, {"error": str(e).splitlines()[0] if str(e) else type(e).__name__, "detail": str(e), + "exception": type(e).__name__}, url, type(e).__name__) + else: + self.pg_calls += 1 + if self.pg_calls == 1: + print("PGWIRE_SUMMARY " + _json.dumps(self.summary(), sort_keys=True), flush=True) # the first call proves the wire in the log + return _OfficialResponse(200, {"result": rows, "limit": json.get("limit", -1), "returned": len(rows), "truncated": truncated}, url, "OK") + self._fall(why or "other") + return self._fallback.post(url, json=json, data=data, auth=auth or self.auth, headers=headers, timeout=timeout, **kw) + + def ndjson_rows(self, url, json): + """The rows of a query a lane asked to stream as NDJSON, read over the wire instead (the wire has no NDJSON): a list, or None when this + statement does not go to the wire and the caller streams it over HTTP.""" + auth = self.auth + m, why = self._route(url, json, None, None, {}) if auth else (None, "no-auth") + if m is None: + return None + try: + rows, _ = self._run(m, json, tuple(auth)) + except _Unroutable: + return None + except self._psycopg.Error as e: + raise HTTPError(f"500 Server Error: {type(e).__name__} for url: {url}") from e + self.pg_calls += 1 + return rows + + def close(self): + print("PGWIRE_SUMMARY " + _json.dumps(self.summary(), sort_keys=True), flush=True) + for cx in self._conns.values(): + try: + cx.close() + except Exception: # noqa: BLE001 - closing must not fail a cell + pass + self._conns.clear() + self._fallback.close() + + +class _Unroutable(Exception): + """A statement the wire cannot take after all (a placeholder with no value): it goes over HTTP.""" + + def _driver_version(): try: from importlib.metadata import version @@ -448,8 +650,9 @@ def _driver_version(): # ------------------------------------------------------------------------------------------------------ the factory def Session(): # noqa: N802 - requests' own spelling, so a lane changes by one import - """The ArcadeDB served arms' session: the official driver (the headline), or `LeanSession` when BENCH_ARCADEDB_HTTP_CLIENT=lean.""" - return OfficialSession() if client_choice() == OFFICIAL else LeanSession() + """The ArcadeDB served arms' session: the official driver (the headline), `LeanSession` when BENCH_ARCADEDB_HTTP_CLIENT=lean, `PgSession` when it is pgwire.""" + choice = client_choice() + return OfficialSession() if choice == OFFICIAL else PgSession() if choice == PGWIRE else LeanSession() def row_fields(session): @@ -459,6 +662,13 @@ def row_fields(session): out["arcadedb_http_client_role"] = "headline" elif os.environ.get("BENCH_ARCADEDB_CLIENT_ROLE"): out["arcadedb_http_client_role"] = os.environ["BENCH_ARCADEDB_CLIENT_ROLE"] + if isinstance(session, PgSession): + # the role stays `sensitivity` (every filter that keeps twins out of a headline keys on it); the wire is the second stamp that tells the + # two twins apart (make_paper_tables' key, export_web's split) + out["arcadedb_client_wire"] = PGWIRE + out["arcadedb_pgwire_client"] = {"psycopg": session.version, "prepare_threshold": None, "autocommit": True, "result_format": "text", + "port": session._port()} + out["arcadedb_pgwire_scope"] = PGWIRE_SCOPE return out @@ -501,7 +711,11 @@ def ndjson_records(body): def post_ndjson(session, url, json, timeout=None): - """POST a query asking for NDJSON, raise on an HTTP error, and return its rows (`ndjson_records`).""" + """POST a query asking for NDJSON, raise on an HTTP error, and return its rows (`ndjson_records`). On the PG-wire twin a SQL query is read over the wire.""" + if isinstance(session, PgSession): + rows = session.ndjson_rows(url, json) + if rows is not None: + return rows r = session.post(url, json=json, headers={"Accept": NDJSON_ACCEPT}, timeout=timeout) r.raise_for_status() return ndjson_records(r.content) diff --git a/benchmarks/experiments/make_paper_tables.py b/benchmarks/experiments/make_paper_tables.py index 198e7a38fdc..c5116035f0e 100644 --- a/benchmarks/experiments/make_paper_tables.py +++ b/benchmarks/experiments/make_paper_tables.py @@ -541,6 +541,8 @@ def load_canonical(apply_corpus=True, include_twins=False): k = (r["lane"], r["scale"], r.get("n_docs"), r.get("workload"), r["backend"], r.get("gav") is not False, r["rep"], r.get("durability_class") or "relaxed", r.get("arcadedb_http_client_role") == "sensitivity", + # the wire tells the two twins apart (lean HTTP or the PostgreSQL wire): both are role `sensitivity`, and one cell runs on each + r.get("arcadedb_client_wire") or "", # THE BUILD IS PART OF THE KEY (26.11.1): `arcadedb` (native) and `arcadedb-jvm` of one release stamp the same # engine_version and the same backend name, so without it the later build would replace the other on ts_utc. # Rows before the field are the JVM build; served arms and comparators carry None (arm_identity.build_of). diff --git a/benchmarks/experiments/pgwire_gate.py b/benchmarks/experiments/pgwire_gate.py new file mode 100644 index 00000000000..f8cc334869d --- /dev/null +++ b/benchmarks/experiments/pgwire_gate.py @@ -0,0 +1,122 @@ +#!/usr/bin/env python3 +"""The answers-identical gate of the PostgreSQL-wire twin (DECISIONS #166: adopt nothing the wire cannot prove; #237 (3)). + +A served SQL cell also runs with its read statements on ArcadeDB's PostgreSQL wire (`arcadedb_client_wire` = pgwire, role `sensitivity`). The wire +returns the same rows with other Python types (Decimal for a number, datetime for a timestamp, a list for an array), so a latency beside the headline +means nothing unless the ANSWERS agree. The lanes already record, outside every timed section, one digest per deterministic query +(`res__digest`, `bench_common.record_result`), computed from the rows the timed call returned and under the lane's own declared coercions +(`coerce=` per query: epoch seconds, numbers, months). The twin runs the SAME lane code, so its digests are taken under the same coercions by +construction; this module compares them with the headline's, per query: + + * equal -> the query is checked and passes; + * different -> the twin row FAILS, the reason names the query and both digests; + * a query the headline recorded and the twin did not (silence is not agreement) -> the twin row FAILS; + * a query only the twin recorded -> listed as unchecked, not a failure (nothing to compare with); + * the headline disagrees with itself across its own rows (more than one digest for one query) -> that query is unchecked, named; + * no headline row, or no digest on either side -> the whole row is `unchecked`, recorded, never passed. + +A failed twin row is KEPT with `error` set (the reason is recorded; it is never dropped from the results file), so landing and export see a failed +sensitivity row, not a missing one. The runner applies `stamp()` as the twin cell ends (against the headline rows already in the results file); +`python3 pgwire_gate.py --rows results/runs.jsonl` applies it to a whole file, with every headline row present, and exits 1 on any failure. +""" +import argparse +import json +import re +import sys + +DIGEST_RE = re.compile(r"^res_(.+)_digest$") +WIRE = "pgwire" +CELL_KEY = ("lane", "scale", "workload", "backend", "durability_class", "server_build") + + +def is_twin(row): + return str(row.get("arcadedb_client_wire") or "") == WIRE + + +def is_headline(row): + """A headline row: no sensitivity role (the official driver's rows are `headline`; an older row carries none), and no error.""" + return str(row.get("arcadedb_http_client_role") or "") != "sensitivity" and not row.get("error") + + +def digests(row): + """{query: digest} of a row's `res__digest` fields that hold a value (a declared absence, `unexpressible: ...`, is not a digest).""" + out = {} + for k, v in row.items(): + m = DIGEST_RE.match(k) + if m and isinstance(v, str) and v and not v.startswith("unexpressible"): + out[m.group(1)] = v + return out + + +def cell_key(row): + return tuple(str(row.get(k) if row.get(k) is not None else ("relaxed" if k == "durability_class" else "")) for k in CELL_KEY) + + +def check(twin, headline_rows): + """The gate for one twin row against the headline rows of its cell: {"status": ok|failed|unchecked, "queries_checked", "reasons", "unchecked"}.""" + heads = [h for h in headline_rows if is_headline(h) and cell_key(h) == cell_key(twin)] + seen = {} + for h in heads: + for q, d in digests(h).items(): + seen.setdefault(q, set()).add(d) + mine = digests(twin) + reasons, unchecked, checked = [], [], 0 + if not heads or not seen: + return {"status": "unchecked", "queries_checked": 0, "reasons": [], "unchecked": ["no headline row with digests for this cell"]} + for q in sorted(seen): + if len(seen[q]) > 1: + unchecked.append(f"{q}: the headline has {len(seen[q])} digests across its own rows") + continue + (want,) = seen[q] + if q not in mine: + reasons.append(f"{q}: the headline recorded a digest and the pg-wire row recorded none") + elif mine[q] != want: + reasons.append(f"{q}: the pg-wire digest {mine[q]} is not the headline's {want}") + else: + checked += 1 + unchecked += [f"{q}: only the pg-wire row recorded it" for q in sorted(set(mine) - set(seen))] + return {"status": "failed" if reasons else "ok" if checked else "unchecked", "queries_checked": checked, "reasons": reasons, "unchecked": unchecked} + + +def stamp(row, headline_rows): + """Record the gate on the twin row; a failure also sets `error` (the row stays, marked failed, reason named). Returns the result.""" + res = check(row, headline_rows) + row["arcadedb_pgwire_gate"] = res + if res["status"] == "failed" and not row.get("error"): + row["error"] = "pg-wire answers differ from the headline: " + "; ".join(res["reasons"][:5]) + (" ..." if len(res["reasons"]) > 5 else "") + return res + + +def load(path): + rows = [] + with open(path) as fh: + for line in fh: + if line.strip(): + try: + rows.append(json.loads(line)) + except ValueError: + continue + return rows + + +def check_file(rows): + """[(row, result)] for every twin row of a whole results file.""" + heads = [r for r in rows if not is_twin(r)] + return [(r, check(r, heads)) for r in rows if is_twin(r) and not (r.get("error") and not r.get("arcadedb_pgwire_gate"))] + + +def main(argv=None): + ap = argparse.ArgumentParser(description=__doc__.split("\n")[0]) + ap.add_argument("--rows", required=True, help="a results .jsonl file") + args = ap.parse_args(argv) + got = check_file(load(args.rows)) + bad = 0 + for r, res in got: + print(f"{res['status']:9} {r.get('run_id')}: {res['queries_checked']} checked" + (f"; {res['reasons'][0]}" if res["reasons"] else "")) + bad += res["status"] == "failed" + print(f"{len(got)} pg-wire twin rows, {bad} failed") + return 1 if bad else 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/benchmarks/experiments/runner.py b/benchmarks/experiments/runner.py index 2511ba83154..55b1ba0b3c6 100644 --- a/benchmarks/experiments/runner.py +++ b/benchmarks/experiments/runner.py @@ -2981,13 +2981,21 @@ def requested_settings(text): return got +def setting_matches(key, have, want): + """Is the value the engine reports the one asked for. The plugin list is the one setting the server rewrites at start (ArcadeDBServer.init puts + "AutoBackupSchedulerPlugin," in front of whatever was configured), so there the requested entries must be the tail of what is reported.""" + if str(have) == str(want): + return True + return key == "arcadedb.server.plugins" and str(have).endswith("," + str(want)) + + def settings_problems(got, requested): """Requested settings the engine LISTS and reports with another value (empty: fine). A key the engine does not list is not a problem here: `server_settings_unlisted` names it on the row.""" out = [] for k, want in sorted((requested or {}).items()): have = (got or {}).get(k) - if have is not None and str(have) != str(want): + if have is not None and not setting_matches(k, have, want): out.append(f"setting {k} was requested as {want} and the engine runs {have}") return out @@ -3216,6 +3224,92 @@ def arcadedb_query_ram_readback(cid, envs, from_env): "arcadedb_query_max_heap_ram_source": "requested via the container's JAVA_OPTS (the engine could not be asked from the host)"} +def pgwire_server_patch(cfg): + """A copy of a served ArcadeDB arm's launch configuration that also starts the PostgreSQL-wire plugin with PGWIRE_SERVER_SETTINGS: as -D tokens + in JAVA_OPTS on the JVM image (exactly one entry), on the command line on the native image. Refuses an arm that already sets one of them.""" + out = dict(cfg) + env, cmd, hits = list(cfg.get("server_env", [])), list(cfg.get("server_cmd", [])), 0 + keys = [t.split("=", 1)[0] for t in PGWIRE_SERVER_SETTINGS] + for i, e in enumerate(env): + if isinstance(e, str) and e.startswith("JAVA_OPTS="): + if any(f"-D{k}=" in e for k in keys): + raise SystemExit("pgwire_server_patch: the arm already sets a PostgreSQL-wire setting") + env[i] = e + "".join(f" -D{t}" for t in PGWIRE_SERVER_SETTINGS) + hits += 1 + if cfg.get("server_build") == "native": + if any(isinstance(c, str) and c.startswith(tuple(f"-D{k}=" for k in keys)) for c in cmd): + raise SystemExit("pgwire_server_patch: the arm already sets a PostgreSQL-wire setting") + cmd += [f"-D{t}" for t in PGWIRE_SERVER_SETTINGS] + elif hits != 1: + raise SystemExit(f"pgwire_server_patch: a served JVM ArcadeDB arm must carry exactly one JAVA_OPTS entry (found {hits})") + out["server_env"], out["server_cmd"] = env, cmd + return out + + +def pgwire_settings_problems(server_settings): + """The PostgreSQL-wire settings the engine did not list or reports with another value than PGWIRE_SERVER_SETTINGS wrote ([] when all are in force). + settings_problems alone passes a key the engine does not list; for the twin that is a failure, because a setting not read back is not evidence.""" + out = [] + for tok in PGWIRE_SERVER_SETTINGS: + k, want = tok.split("=", 1) + have = (server_settings or {}).get(k) + if have is None: + out.append(f"the engine did not list {k}, so it is not read back") + elif not setting_matches(k, have, want): + out.append(f"{k} was written as {want} and the engine runs {have}") + return out + + +def pgwire_probe(cid): + """Does the PostgreSQL-wire port of a served ArcadeDB container accept a TCP connection, within 20 s. {arcadedb_pgwire_port_open, arcadedb_pgwire_host}.""" + import socket + ips = (sh(["docker", "inspect", "-f", "{{range .NetworkSettings.Networks}}{{.IPAddress}} {{end}}", cid]) or "").split() + for _ in range(20): + for ip in ips[:1]: + try: + with socket.create_connection((ip, PGWIRE_PORT), timeout=3): + return {"arcadedb_pgwire_port_open": True, "arcadedb_pgwire_host": ip} + except OSError: + pass + time.sleep(1) + return {"arcadedb_pgwire_port_open": False, "arcadedb_pgwire_host": ips[0] if ips else None} + + +_PGWIRE_SUMMARY_RE = re.compile(r"^PGWIRE_SUMMARY (\{.*\})\s*$", re.M) +_PGWIRE_FALLBACK_RE = re.compile(r"^PGWIRE_FALLBACK (\S+)\s*$", re.M) + + +def pgwire_log_fields(log): + """What the twin's client says about the statements it sent, from the LAST PGWIRE_SUMMARY line of its log (printed on the first wire call, every + 10 s while counters move, and at close) and every PGWIRE_FALLBACK line (the first sight of each reason): the calls on the wire, the calls and reasons + that went over HTTP. Empty-handed when the log has none (the row then carries no call count and the cell is failed above).""" + out = {} + hits = _PGWIRE_SUMMARY_RE.findall(log or "") + if hits: + try: + got = json.loads(hits[-1]) + out["arcadedb_pgwire_calls"] = got.get("pg_calls") + out["arcadedb_pgwire_fallback_calls"] = got.get("fallback_calls") + out["arcadedb_pgwire_fallback_reasons"] = got.get("fallback_reasons") + except ValueError: + pass + seen = sorted(set(_PGWIRE_FALLBACK_RE.findall(log or ""))) + if seen and "arcadedb_pgwire_fallback_reasons" not in out: + out["arcadedb_pgwire_fallback_reasons"] = {r: None for r in seen} + return out + + +def pgwire_gate_stamp(row, results_path): + """The answers-identical gate (pgwire_gate.py) against the headline rows already in the results file: stamps `arcadedb_pgwire_gate` and, on a + mismatch, sets `error` (the row stays, marked failed). Never raises; an unreadable file leaves the gate `unchecked`.""" + import pgwire_gate + try: + heads = [r for r in pgwire_gate.load(results_path) if not pgwire_gate.is_twin(r)] if os.path.exists(results_path) else [] + except OSError: + heads = [] + pgwire_gate.stamp(row, heads) + + def arcadedb_body_limit_readback(cid, envs, from_env): """The HTTP request-body limit a served ArcadeDB runs with, asked of the engine. @@ -3419,6 +3513,8 @@ def run_cell(job, rep, scale, cpuset, tier, net_name): # GAV-ablation shadowing this file already documents twice. _dcls = os.environ.get("BENCH_DURABILITY", "relaxed") be, _dnote = durability_server_patch(be, _dcls) + if job.get("http_client") == PGWIRE_CLIENT: + be = pgwire_server_patch(be) # the plugin and its settings, on this twin's server only _dsuffix = "" if _dcls == "relaxed" else f"_d{_dcls}" run_id = f"{job['run_id']}_{scale}_r{rep}{_dsuffix}" stale = os.path.join(RAW, f"{run_id}.json") @@ -3441,6 +3537,7 @@ def run_cell(job, rep, scale, cpuset, tier, net_name): "bench_host": os.environ.get("BENCH_HOST"), "instrument": bench_common.INSTRUMENT, **({"arcadedb_http_client_role": "sensitivity"} if job.get("http_client") else {}), + **({"arcadedb_client_wire": PGWIRE_CLIENT} if job.get("http_client") == PGWIRE_CLIENT else {}), # The class the cell ASKED for. The lane also stamps it, from the # same variable, beside the string the engine itself reports; both # are here so a row missing the lane's stamp (an old artifact, a @@ -3614,6 +3711,12 @@ def run_cell(job, rep, scale, cpuset, tier, net_name): row.update(_rb) _bad = settings_problems(_rb.get("server_settings"), _rb.get("server_settings_requested")) _bad += engine_commit_problems(_rb.get("server_engine_commit"), _LOCAL_ENGINE_COMMIT) + if job.get("http_client") == PGWIRE_CLIENT: + # THE PLUGIN IS UP AND ITS SETTINGS ARE THE ONES WRITTEN: read back from the engine, then the port answered (outside every timer) + _bad += pgwire_settings_problems(_rb.get("server_settings")) + row.update(pgwire_probe(server_cid)) + if not row.get("arcadedb_pgwire_port_open"): + _bad.append(f"the PostgreSQL wire does not answer on port {PGWIRE_PORT}") _lg = arcadedb_server_log_readback(server_cid) row.update(_lg) _bad += server_log_problems(_lg, None if be.get("jvm_defaults") else mem_bytes(heap), _cpuset_size(cpuset), @@ -3823,6 +3926,8 @@ def run_cell(job, rep, scale, cpuset, tier, net_name): if job.get("http_client"): bench_env += ["-e", f"BENCH_ARCADEDB_HTTP_CLIENT={job['http_client']}", "-e", "BENCH_ARCADEDB_CLIENT_ROLE=sensitivity"] + if job["http_client"] == PGWIRE_CLIENT: + bench_env += ["-e", f"BENCH_ARCADEDB_PG_PORT={PGWIRE_PORT}"] # TELL THE CONTAINER WHAT IT IS RUNNING. Only this process knows the # backend's image and pinned server digest; inside the container that @@ -3947,6 +4052,10 @@ def run_cell(job, rep, scale, cpuset, tier, net_name): docker_rm(cli_cid) row["rc"] = rc row["oom_killed"] = (oom == "true") + if job.get("http_client") == PGWIRE_CLIENT: + row.update(pgwire_log_fields((logs.stdout or "") + "\n" + (logs.stderr or ""))) + if rc == 0 and "error" not in row and not row.get("arcadedb_pgwire_calls"): + row["error"] = "no statement went over the PostgreSQL wire (the cell's log has no PGWIRE_SUMMARY with calls); the twin did not measure the wire" if rc != 0 and "error" not in row: detail = (logs.stderr or logs.stdout)[-800:].strip() if not detail: @@ -4410,12 +4519,39 @@ def restricted_arms(lane): # THE SERVED ARCADEDB ARMS' CLIENT SENSITIVITY (CAMPAIGN section 7 row 72). The headline client of a served # ArcadeDB arm is the vendor's `arcadedb-driver` (lean_http.OfficialSession). Every such arm also runs # the same cell on the lean HTTP client, as a twin job whose rows carry `arcadedb_http_client_role` = "sensitivity" and whose run_id ends -# in `_clientlean`. The PostgreSQL wire has no adapter in this harness, so it is not among the twins. +# in `_clientlean`. The served SQL cells also run on the PostgreSQL wire (PGWIRE_SQL_ARMS, run_id ending `_clientpgwire`). ARCADEDB_HTTP_ARMS = ("arcadedb_server", "arcadedb_graph_server", "arcadedb_e2_server", "arcadedb_sparse_server", "arcadedb_sparse_server_fp32", "arcadedb_dense_server", "arcadedb_dense_server_int8", "arcadedb_ts_doc_server", "arcadedb_ts_native_server") SENSITIVITY_CLIENT = "lean" +# THE POSTGRESQL-WIRE TWIN (DECISIONS #166, #171, #237; CAMPAIGN section 7 row 6). The served SQL cells below run a third time, as a twin job whose +# read statements go over ArcadeDB's PostgreSQL-wire plugin through psycopg (lean_http.PgSession) and whose rows carry `arcadedb_http_client_role` = +# "sensitivity" (every filter that keeps a twin out of a headline keys on it) and `arcadedb_client_wire` = "pgwire"; the run_id ends in `_clientpgwire`. +# WHICH CELLS: a served ArcadeDB arm whose lane sends its timed reads as SQL. NOT covered, and why: l2 (Cypher stays on HTTP, #162), e2 (SQL and Cypher +# in one cell, and transactions over a session id), lifecycle and restart (their timed quantity is open, close and restart, not a statement), e4 (it +# measures the client axis itself), arcadedb_imgdefaults_server (a JVM-defaults arm with no twin of its own). Within a covered cell only `/query` SQL +# goes over the wire; writes, DDL, transactions and bulk loads stay on HTTP (#171 (c)). +PGWIRE_CLIENT = "pgwire" +PGWIRE_SQL_ARMS = {"l1": ("arcadedb_server",), "l1tpc": ("arcadedb_server",), + "l3s": ("arcadedb_sparse_server", "arcadedb_sparse_server_fp32"), + "l3d": ("arcadedb_dense_server", "arcadedb_dense_server_int8"), + "l4": ("arcadedb_ts_doc_server", "arcadedb_ts_native_server")} +# THE PLUGIN'S SETTINGS, EXPLICIT (the way EXPLICIT_SERVER_SETTINGS are): the plugin is opt-in (arcadedb.server.plugins is empty by default), so it is +# named; every other value is the engine's own default, written out so the row records them and a changed default is visible. Only the twin's server +# gets them (the headline and lean-twin servers run exactly what they ran), and the runner reads each back from the engine and refuses a cell whose +# engine does not list or report it as written (`pgwire_settings_problems`). +PGWIRE_PLUGIN_SETTING = "arcadedb.server.plugins=Postgres:com.arcadedb.postgres.PostgresProtocolPlugin" +PGWIRE_SERVER_SETTINGS = (PGWIRE_PLUGIN_SETTING, "arcadedb.postgres.host=0.0.0.0", "arcadedb.postgres.port=5432", "arcadedb.postgres.ssl=DISABLED", + "arcadedb.postgres.debug=false", "arcadedb.postgres.quotedIdentifiers=true", "arcadedb.postgres.queryMaxRows=1000000") +PGWIRE_PORT = 5432 + + +def _client_of_run_id(run_id): + """'lean', 'pgwire' or '' (the headline): the client a row's run_id names (`..._clientlean_tiny_r1`). A twin cell is not done because its headline is.""" + m = re.search(r"_client(lean|pgwire)_", str(run_id or "")) + return m.group(1) if m else "" + def build_jobs(lanes, workloads_arg): """Jobs for these lanes, optionally narrowed to some workloads. @@ -4448,8 +4584,12 @@ def build_jobs(lanes, workloads_arg): "script": script, "run_id": f"{lane}_{be}_{wl}"}) if be in ARCADEDB_HTTP_ARMS: - jobs.append({**jobs[-1], "http_client": SENSITIVITY_CLIENT, + head = jobs[-1] + jobs.append({**head, "http_client": SENSITIVITY_CLIENT, "run_id": f"{lane}_{be}_{wl}_client{SENSITIVITY_CLIENT}"}) + if be in PGWIRE_SQL_ARMS.get(lane, ()): + jobs.append({**head, "http_client": PGWIRE_CLIENT, + "run_id": f"{lane}_{be}_{wl}_client{PGWIRE_CLIENT}"}) if want: missing = want - {j["workload"] for j in jobs} if missing: @@ -4673,13 +4813,13 @@ def main(): if _r.get("error") or _r.get("scale") != args.scale: continue if str(_r.get("ts_utc", "")) >= _since: - _done.add((_r.get("backend"), _r.get("rep"))) + _done.add((_r.get("backend"), _r.get("rep"), _client_of_run_id(_r.get("run_id")))) _before = len(cells) - cells = [(j, r) for (j, r) in cells if (j["backend"], r) not in _done] + cells = [(j, r) for (j, r) in cells if (j["backend"], r, j.get("http_client") or "") not in _done] print(f"resume: {_before - len(cells)} of {_before} cells already clean " f"since {args.skip_done_since}; {len(cells)} to run") - for _b, _r in sorted(_done, key=lambda x: (str(x[0]), x[1] or 0)): - print(f" skip {_b} rep{_r}") + for _b, _r, _c in sorted(_done, key=lambda x: (str(x[0]), x[1] or 0, x[2])): + print(f" skip {_b} rep{_r}" + (f" ({_c} twin)" if _c else "")) ts = datetime.now(timezone.utc).strftime("%Y%m%dT%H%M%SZ") manifest = build_manifest(ts, args, workers, shards, jobs) @@ -4738,6 +4878,8 @@ def worker(shard): cv.notify_all() row["manifest"] = ts _AI.stamp_build(row, _ENGINE_BACKEND) # a killed cell's row names the build its wheel is, not the JVM default + if job.get("http_client") == PGWIRE_CLIENT and not row.get("error"): + pgwire_gate_stamp(row, os.path.join(RESULTS, args.results_file)) # `or` rather than a dict default: an EMPTY error string is what # made thirty OOM kills look like blank successes. status = (row.get("error") diff --git a/benchmarks/experiments/test_instrument_method.py b/benchmarks/experiments/test_instrument_method.py index 7d894565408..062132a88ac 100644 --- a/benchmarks/experiments/test_instrument_method.py +++ b/benchmarks/experiments/test_instrument_method.py @@ -114,7 +114,7 @@ def test_served_client_selector_takes_official_or_lean_only(): JOBS = ("import runner as m\n" "j = m.build_jobs(list(m.LANES), [])\n" - "tw = [x for x in j if x.get('http_client')]\n" + "tw = [x for x in j if x.get('http_client') == 'lean']\n" "print(len(tw), sorted({x['backend'] for x in tw}) == sorted({x['backend'] for x in j if x['backend'] in m.ARCADEDB_HTTP_ARMS}),\n" " all(x['run_id'].endswith('_clientlean') for x in tw), len({x['run_id'] for x in j}) == len(j))") diff --git a/benchmarks/experiments/test_pgwire_twin.py b/benchmarks/experiments/test_pgwire_twin.py new file mode 100644 index 00000000000..c2ab9df49d2 --- /dev/null +++ b/benchmarks/experiments/test_pgwire_twin.py @@ -0,0 +1,364 @@ +"""The PostgreSQL-wire twin of ArcadeDB's served SQL cells (DECISIONS #166, #171, #237; CAMPAIGN section 7 row 6). + +Run with `python -m pytest test_pgwire_twin.py -q` from this directory. Nothing starts a server or a container: the wire is a fake psycopg, the server +launch is a dict, the stage generator only writes drafts. +""" +import datetime as dt +import decimal +import json +import sys +import types +from pathlib import Path + +import pytest + +HERE = Path(__file__).resolve().parent +sys.path.insert(0, str(HERE)) + +import bench_common # noqa: E402 +import lean_http # noqa: E402 +import pgwire_gate as gate # noqa: E402 +import runner # noqa: E402 + +# THE SERVED SQL CELLS, written out by hand and not read from runner.PGWIRE_SQL_ARMS, so a change to the table is a visible change to this list. +EXPECTED = {("l1", "arcadedb_server"), ("l1tpc", "arcadedb_server"), + ("l3s", "arcadedb_sparse_server"), ("l3s", "arcadedb_sparse_server_fp32"), + ("l3d", "arcadedb_dense_server"), ("l3d", "arcadedb_dense_server_int8"), + ("l4", "arcadedb_ts_doc_server"), ("l4", "arcadedb_ts_native_server")} + + +# ------------------------------------------------------------------------------------------------------ which cells get the twin +def test_the_twin_is_emitted_for_exactly_the_served_sql_cells(): + jobs = runner.build_jobs(list(runner.LANES), []) + pg = [j for j in jobs if j.get("http_client") == "pgwire"] + assert {(j["lane"], j["backend"]) for j in pg} == EXPECTED + heads = {(j["lane"], j["backend"], j["workload"]) for j in jobs if not j.get("http_client")} + # one twin per workload the headline runs, never more + assert {(j["lane"], j["backend"], j["workload"]) for j in pg} == {h for h in heads if h[:2] in EXPECTED} + assert all(j["run_id"].endswith("_clientpgwire") for j in pg) + assert len({j["run_id"] for j in jobs}) == len(jobs) + + +def test_cypher_mixed_and_lifecycle_cells_have_no_pgwire_twin(): + jobs = runner.build_jobs(["l2", "e2", "lifecycle", "restart", "e4"], []) + assert jobs and not [j for j in jobs if j.get("http_client") == "pgwire"] + + +def test_the_lean_twin_set_is_unchanged_and_the_wire_twin_is_a_third_job(): + jobs = runner.build_jobs(list(runner.LANES), []) + lean = {(j["lane"], j["backend"], j["workload"]) for j in jobs if j.get("http_client") == "lean"} + assert lean == {(j["lane"], j["backend"], j["workload"]) for j in jobs if j["backend"] in runner.ARCADEDB_HTTP_ARMS and not j.get("http_client")} + assert {b for _, b in EXPECTED} <= set(runner.ARCADEDB_HTTP_ARMS) + + +def test_the_stage_generator_schedules_the_twin_for_exactly_the_served_sql_cells(): + import make_2611_stages as G + engines = G.parse_engines(None, "arcadedb") + specs = G.number(G.subset_specs(engines, G.parse_lanes(None, False, "paper-2611"), "biggest-last", G.profile_skip("paper-2611"), True), "qT") + cells = {(c.lane, c.backend, c.workload) for spec in specs for c in G.stage_cells(spec)} + assert cells + served = {c for c in cells if c[1] in runner.ARCADEDB_HTTP_ARMS} + got = set() + for lane, be, wl in served: + for j in runner.build_jobs([lane], [wl]): + if j["backend"] == be and j.get("http_client") == "pgwire": + got.add((lane, be, wl)) + assert {(l, b) for l, b, _ in got} <= EXPECTED and got + assert all(c in got for c in served if c[:2] in EXPECTED) + assert not [c for c in got if c[0] in ("l2", "e2")] + + +def test_a_resume_does_not_treat_a_twin_as_done_because_its_headline_is(): + assert runner._client_of_run_id("l1_arcadedb_server_oltp_clientpgwire_tiny_r1") == "pgwire" + assert runner._client_of_run_id("l1_arcadedb_server_oltp_clientlean_tiny_r1") == "lean" + assert runner._client_of_run_id("l1_arcadedb_server_oltp_tiny_r1") == "" + + +# ------------------------------------------------------------------------------------------------------ the role stamp +def test_the_selector_takes_pgwire_and_still_refuses_the_rest(monkeypatch): + monkeypatch.setenv("BENCH_ARCADEDB_HTTP_CLIENT", "pgwire") + assert lean_http.client_choice() == "pgwire" + monkeypatch.setenv("BENCH_ARCADEDB_HTTP_CLIENT", "grpc") + with pytest.raises(ValueError): + lean_http.client_choice() + + +class _FakePg: + """psycopg, as far as PgSession uses it; `log` records (sql, args) of every statement the wire got.""" + + def __init__(self, monkeypatch, rows=None): + self.log, self.rows, self.connects = [], rows if rows is not None else [{"a": 1}], [] + fake = self + + class Error(Exception): + pass + + class Cur: + description = [("a",)] + + def fetchall(self_): + return list(fake.rows) + + class Conn: + closed = False + + def execute(self_, sql, args=None): + if "BOOM" in sql: + raise Error("syntax error at BOOM\nline 2") + fake.log.append((sql, args)) + return Cur() + + def close(self_): + self_.closed = True + + mod = types.ModuleType("psycopg") + mod.__version__, mod.Error = "3.2.9", Error + mod.pq = types.SimpleNamespace(__impl__="binary") + mod.connect = lambda **kw: fake.connects.append(kw) or Conn() + rowsmod = types.ModuleType("psycopg.rows") + rowsmod.dict_row = "dict_row" + monkeypatch.setitem(sys.modules, "psycopg", mod) + monkeypatch.setitem(sys.modules, "psycopg.rows", rowsmod) + + +BASE = "http://10.0.0.5:2480/api/v1" + + +def _session(monkeypatch, **kw): + fake = _FakePg(monkeypatch, **kw) + s = lean_http.PgSession() + s._beat = True # no heartbeat thread in a unit test + fallback = [] + s._fallback = types.SimpleNamespace(post=lambda url, **k: fallback.append(("post", url)) or "lean-post", + get=lambda url, **k: fallback.append(("get", url)) or "lean-get", close=lambda: None) + s.auth = ("root", "pw") + return s, fake, fallback + + +def test_a_wire_row_is_a_sensitivity_row_with_the_wire_stamp_and_a_headline_is_not(monkeypatch): + monkeypatch.setenv("BENCH_ARCADEDB_CLIENT_ROLE", "sensitivity") + s, _, _ = _session(monkeypatch) + got = lean_http.row_fields(s) + assert got["arcadedb_http_client_role"] == "sensitivity" and got["arcadedb_client_wire"] == "pgwire" + assert got["arcadedb_http_client"].startswith("psycopg 3.2.9 (PostgreSQL wire") + assert got["arcadedb_pgwire_client"] == {"psycopg": "3.2.9", "prepare_threshold": None, "autocommit": True, "result_format": "text", "port": 5432} + assert "read and query statements only" in got["arcadedb_pgwire_scope"] + lean = lean_http.row_fields(lean_http.LeanSession()) + assert "arcadedb_client_wire" not in lean and lean["arcadedb_http_client_role"] == "sensitivity" + + +def test_the_runner_stamps_the_wire_on_a_job_that_never_reached_its_lane(): + src = (HERE / "runner.py").read_text() + assert '**({"arcadedb_client_wire": PGWIRE_CLIENT} if job.get("http_client") == PGWIRE_CLIENT else {})' in src + assert 'f"BENCH_ARCADEDB_PG_PORT={PGWIRE_PORT}"' in src + + +def test_the_factory_hands_a_wire_session_only_to_the_wire_twin(monkeypatch): + _FakePg(monkeypatch) + monkeypatch.setenv("BENCH_ARCADEDB_HTTP_CLIENT", "pgwire") + assert isinstance(lean_http.Session(), lean_http.PgSession) + monkeypatch.setenv("BENCH_ARCADEDB_HTTP_CLIENT", "lean") + assert isinstance(lean_http.Session(), lean_http.LeanSession) + + +# ------------------------------------------------------------------------------------------------------ what goes over the wire +def test_a_sql_query_goes_over_the_wire_bound_and_the_rest_stays_on_http(monkeypatch): + s, fake, fallback = _session(monkeypatch) + r = s.post(f"{BASE}/query/bench", json={"language": "sql", "command": "SELECT FROM T WHERE k = :k AND s LIKE '%:x%'", "params": {"k": 7}}) + assert r.ok and r.json()["result"] == [{"a": 1}] and r.json()["returned"] == 1 + assert fake.log == [("SELECT FROM T WHERE k = %(k)s AND s LIKE '%%:x%%'", {"k": 7})] + kw = fake.connects[0] + assert (kw["host"], kw["port"], kw["dbname"], kw["user"], kw["autocommit"], kw["prepare_threshold"]) == ("10.0.0.5", 5432, "bench", "root", True, None) + base = f"{BASE}/command/bench" + assert s.post(base, json={"language": "sql", "command": "INSERT"}) == "lean-post" # writes stay on HTTP + assert s.post(f"{BASE}/batch/bench", data=b"x") == "lean-post" # bulk loads stay on HTTP + assert s.post(f"{BASE}/query/bench", json={"language": "cypher", "command": "MATCH (n) RETURN n"}) == "lean-post" # Cypher stays on HTTP + assert s.post(f"{BASE}/query/bench", json={"language": "sql", "command": "x", "params": {"m": {"a": 1}}}) == "lean-post" # a dict cannot be bound + assert s.post(f"{BASE}/query/bench", json={"language": "sql", "command": "x"}, headers={"arcadedb-session-id": "1"}) == "lean-post" + assert s.post(f"{BASE}/query/bench", json={"language": "sql", "command": "SELECT :gone", "params": {}}) == "lean-post" + assert s.get(f"{BASE}/server?mode=basic") == "lean-get" + assert s.pg_calls == 1 and s.fallback_calls == 7 + assert set(s.summary()["fallback_reasons"]) == {"command", "other-endpoint", "language-cypher", "param-type", "headers-or-raw-body", "param-missing", "get"} + + +def test_a_list_param_travels_as_a_bound_array_and_a_limit_truncates(monkeypatch): + s, fake, _ = _session(monkeypatch, rows=[{"a": i} for i in range(5)]) + r = s.post(f"{BASE}/query/bench", json={"language": "sql", "command": "SELECT :q", "params": {"q": [0.5, 1.5]}, "limit": 3}) + assert fake.log[0][1] == {"q": [0.5, 1.5]} + assert r.json()["returned"] == 3 and r.json()["truncated"] is True + + +def test_a_server_error_is_a_500_answer_not_an_exception(monkeypatch): + s, _, _ = _session(monkeypatch) + r = s.post(f"{BASE}/query/bench", json={"language": "sql", "command": "BOOM"}) + assert r.status_code == 500 and not r.ok and r.json()["error"] == "syntax error at BOOM" + with pytest.raises(lean_http.HTTPError): + r.raise_for_status() + + +def test_an_ndjson_query_of_the_wire_twin_is_read_over_the_wire(monkeypatch): + s, fake, fallback = _session(monkeypatch, rows=[{"a": 1}, {"a": 2}]) + assert lean_http.post_ndjson(s, f"{BASE}/query/bench", {"language": "sql", "command": "SELECT 1"}) == [{"a": 1}, {"a": 2}] + assert fake.log and not fallback and s.pg_calls == 1 + s.post = lambda *a, **k: types.SimpleNamespace(raise_for_status=lambda: None, content=b'{"record":{"x":1}}\n{"stats":{"returned":1}}\n') + assert lean_http.post_ndjson(s, f"{BASE}/command/bench", {"language": "sql", "command": "SELECT 1"}) == [{"x": 1}] # not a query: HTTP + + +def test_pg_translate_binds_names_and_leaves_quoted_text_alone(): + assert lean_http.pg_translate("SELECT :a, ':b', \":c\", x::y", {"a": 1}) == ("SELECT %(a)s, ':b', \":c\", x::y", {"a": 1}) + assert lean_http.pg_translate("SELECT 1 % 2", None) == ("SELECT 1 % 2", {}) + assert lean_http.pg_translate("SELECT 1 % :a", {"a": 1})[0] == "SELECT 1 %% %(a)s" + assert lean_http.pg_translate("SELECT :a", {}) is None + + +# ------------------------------------------------------------------------------------------------------ the plugin's settings +JVM_ARM = {"server_env": ["-e", "JAVA_OPTS=-Darcadedb.server.mode=development", "-e", "X=1"], "server_cmd": []} +NATIVE_ARM = {"server_env": [], "server_cmd": ["-Xmx4g", "-Darcadedb.server.mode=development"], "server_build": "native"} + + +def test_the_plugin_and_its_settings_are_added_to_the_twin_server_only(): + jvm = runner.pgwire_server_patch(JVM_ARM) + opts = [e for e in jvm["server_env"] if str(e).startswith("JAVA_OPTS=")] + assert len(opts) == 1 + for tok in runner.PGWIRE_SERVER_SETTINGS: + assert f"-D{tok}" in opts[0] + assert JVM_ARM["server_env"][1] == "JAVA_OPTS=-Darcadedb.server.mode=development" # the headline's launch is untouched + nat = runner.pgwire_server_patch(NATIVE_ARM) + assert all(f"-D{t}" in nat["server_cmd"] for t in runner.PGWIRE_SERVER_SETTINGS) and NATIVE_ARM["server_cmd"] == ["-Xmx4g", "-Darcadedb.server.mode=development"] + with pytest.raises(SystemExit): + runner.pgwire_server_patch(jvm) # twice is refused + for name, cfg in runner.BACKENDS.items(): + if name in runner.ARCADEDB_HTTP_ARMS: + assert "arcadedb.postgres.port" not in json.dumps(cfg.get("server_env", [])), name # no headline server carries it + + +def test_a_setting_the_engine_does_not_list_or_reports_otherwise_fails_the_twin(): + listed = {t.split("=", 1)[0]: t.split("=", 1)[1] for t in runner.PGWIRE_SERVER_SETTINGS} + assert runner.pgwire_settings_problems(listed) == [] + listed["arcadedb.server.plugins"] = "AutoBackupSchedulerPlugin," + listed["arcadedb.server.plugins"] # the server puts its own plugin first + assert runner.pgwire_settings_problems(listed) == [] + assert runner.settings_problems(listed, {"arcadedb.server.plugins": runner.PGWIRE_PLUGIN_SETTING.split("=", 1)[1]}) == [] + gone = dict(listed) + gone.pop("arcadedb.postgres.port") + assert "did not list arcadedb.postgres.port" in runner.pgwire_settings_problems(gone)[0] + other = dict(listed, **{"arcadedb.postgres.port": "5433"}) + assert "engine runs 5433" in runner.pgwire_settings_problems(other)[0] + assert runner.pgwire_settings_problems(dict(listed, **{"arcadedb.server.plugins": "Other:x.Y"})) + + +def test_the_cells_log_gives_the_calls_and_the_reasons(): + log = ('noise\nPGWIRE_SUMMARY {"fallback_calls": 9, "fallback_reasons": {"command": 9}, "pg_calls": 1}\nPGWIRE_FALLBACK command\n' + 'PGWIRE_SUMMARY {"fallback_calls": 30, "fallback_reasons": {"command": 20, "get": 10}, "pg_calls": 4000}\n') + assert runner.pgwire_log_fields(log) == {"arcadedb_pgwire_calls": 4000, "arcadedb_pgwire_fallback_calls": 30, + "arcadedb_pgwire_fallback_reasons": {"command": 20, "get": 10}} + assert runner.pgwire_log_fields("PGWIRE_FALLBACK get\n") == {"arcadedb_pgwire_fallback_reasons": {"get": None}} + assert runner.pgwire_log_fields("nothing") == {} + + +# ------------------------------------------------------------------------------------------------------ the answers-identical gate +def _row(wire=False, **dig): + r = {"lane": "l1tpc", "scale": "tpch1", "workload": "olap", "backend": "arcadedb_server", "durability_class": "relaxed", "server_build": "jvm", + "run_id": "x", "arcadedb_http_client_role": "sensitivity" if wire else "headline"} + if wire: + r["arcadedb_client_wire"] = "pgwire" + r.update({f"res_{q}_digest": d for q, d in dig.items()}) + return r + + +def test_equal_digests_pass(): + res = gate.check(_row(True, q1="aaaa", q6="bbbb"), [_row(q1="aaaa", q6="bbbb"), _row(q1="aaaa", q6="bbbb")]) + assert res["status"] == "ok" and res["queries_checked"] == 2 and res["reasons"] == [] + + +def test_an_unequal_digest_fails_the_twin_row_and_names_the_query(): + twin = _row(True, q1="aaaa", q6="cccc") + res = gate.stamp(twin, [_row(q1="aaaa", q6="bbbb")]) + assert res["status"] == "failed" and "q6" in res["reasons"][0] and "cccc" in res["reasons"][0] and "bbbb" in res["reasons"][0] + assert twin["error"].startswith("pg-wire answers differ from the headline: q6") and twin["arcadedb_pgwire_gate"]["status"] == "failed" + + +def test_a_query_the_wire_row_did_not_record_fails_it(): + res = gate.check(_row(True, q1="aaaa"), [_row(q1="aaaa", q6="bbbb")]) + assert res["status"] == "failed" and "recorded none" in res["reasons"][0] + + +def test_no_headline_digests_is_unchecked_and_never_a_pass(): + assert gate.check(_row(True, q1="aaaa"), [])["status"] == "unchecked" + assert gate.check(_row(True, q1="aaaa"), [_row()])["status"] == "unchecked" + other_cell = dict(_row(q1="aaaa"), scale="tpch10") + assert gate.check(_row(True, q1="aaaa"), [other_cell])["status"] == "unchecked" + twin = _row(True, q1="aaaa") + gate.stamp(twin, []) + assert "error" not in twin + + +def test_a_headline_that_disagrees_with_itself_leaves_that_query_unchecked(): + res = gate.check(_row(True, q1="aaaa", q6="bbbb"), [_row(q1="xxxx", q6="bbbb"), _row(q1="yyyy", q6="bbbb")]) + assert res["status"] == "ok" and res["queries_checked"] == 1 and "q1" in res["unchecked"][0] + + +def test_a_declared_absence_is_not_a_digest(): + assert gate.digests({"res_q_digest": "unexpressible: no such operator", "res_r_digest": "abcd"}) == {"r": "abcd"} + + +def test_the_headlines_own_failed_or_twin_rows_are_not_the_reference(): + bad = dict(_row(q1="zzzz"), error="boom") + lean = dict(_row(q1="zzzz"), arcadedb_http_client_role="sensitivity") + assert gate.check(_row(True, q1="aaaa"), [bad, lean, _row(q1="aaaa")])["status"] == "ok" + + +def test_the_gate_compares_digests_taken_under_the_lanes_coercions(): + """What the wire changes is the Python TYPES (Decimal for a number, datetime for a timestamp, tuples for dicts); the lane's declared `coerce` makes + both digests the same, and an uncoerced digest would not be: that is what the gate relies on, and why it compares the lane's own digests.""" + spec = dict(columns=("h", "v"), coerce={"h": "epoch_s", "v": "num"}) + http = [{"h": 1767225600, "v": 12.5}, {"h": 1767229200, "v": 3.0}] + wire = [{"h": dt.datetime(2026, 1, 1, 0, 0, tzinfo=dt.timezone.utc), "v": decimal.Decimal("12.5")}, + {"h": dt.datetime(2026, 1, 1, 1, 0, tzinfo=dt.timezone.utc), "v": decimal.Decimal("3.0")}] + a, b = bench_common.result_digest(http, **spec)["digest"], bench_common.result_digest(wire, **spec)["digest"] + assert a == b + assert gate.check(_row(True, q=b), [_row(q=a)])["status"] == "ok" + wrong = [dict(wire[0], v=decimal.Decimal("12.6")), wire[1]] + assert gate.check(_row(True, q=bench_common.result_digest(wrong, **spec)["digest"]), [_row(q=a)])["status"] == "failed" + + +def test_the_runner_stamps_the_gate_against_the_headline_rows_in_the_results_file(tmp_path): + path = tmp_path / "runs.jsonl" + path.write_text("\n".join(json.dumps(r) for r in (_row(q1="aaaa"), _row(True, q1="aaaa"))) + "\n") + ok, bad = _row(True, q1="aaaa"), _row(True, q1="bbbb") + runner.pgwire_gate_stamp(ok, str(path)) + runner.pgwire_gate_stamp(bad, str(path)) + assert ok["arcadedb_pgwire_gate"]["status"] == "ok" and "error" not in ok + assert bad["arcadedb_pgwire_gate"]["status"] == "failed" and "error" in bad + runner.pgwire_gate_stamp(_row(True, q1="x"), str(tmp_path / "missing.jsonl")) # no file: unchecked, no exception + + +def test_the_file_gate_exits_1_on_a_failure_and_reads_every_headline_row(tmp_path, capsys): + path = tmp_path / "runs.jsonl" + path.write_text("\n".join(json.dumps(r) for r in (_row(True, q1="bbbb"), _row(q1="aaaa"))) + "\n") # the twin comes first in the file + assert gate.main(["--rows", str(path)]) == 1 + assert "1 pg-wire twin rows, 1 failed" in capsys.readouterr().out + + +# ------------------------------------------------------------------------------------------------------ landing and export keep the headline +def test_the_freeze_keeps_one_row_per_twin_and_the_export_leaves_wire_rows_out_of_the_lean_twin(): + tables = (HERE / "make_paper_tables.py").read_text() + assert 'r.get("arcadedb_client_wire") or "",' in tables + export = (HERE / "export_web.py").read_text() + assert '_twin_rows = [r for r in _twin_rows if str(r.get("arcadedb_client_wire") or "") != "pgwire"]' in export + assert "_PG_TWIN_ROWS" in export + + +def test_the_export_sentence_names_the_wire_twin_only_for_clean_rows_of_the_tables_backends(): + import export_web as E + table = {"entries": [{"backend_key": "arcadedb_server"}]} + E._FROZEN_ROWS[:] = [{"backend": "arcadedb_server", "arcadedb_http_client": "arcadedb-driver 0.2.0 (sync, httpx)"}] + try: + E._PG_TWIN_ROWS[:] = [] + assert "PostgreSQL wire" not in (E._arcadedb_client_note(table) or "") + E._PG_TWIN_ROWS[:] = [{"backend": "arcadedb_server"}] + note = E._arcadedb_client_note(table) + assert "PostgreSQL wire" in note and "sensitivity" in note and "stay on HTTP" in note and "—" not in note + E._PG_TWIN_ROWS[:] = [{"backend": "arcadedb_graph_server"}] + assert "PostgreSQL wire" not in E._arcadedb_client_note(table) + finally: + E._PG_TWIN_ROWS[:] = [] From 7e7b710c475dd14674f8ee9e0c440b8d8fe481f2 Mon Sep 17 00:00:00 2001 From: Taewoon Kim Date: Wed, 4 Nov 2026 01:05:06 +0900 Subject: [PATCH 2/3] benchmarks: the wire gate compares recall on the lanes that record no digest; the native-server smoke expects the third row l3s and l3d are checked against ground truth (recall_at_10), not by digest, so the gate compares the wire row's recall with the headline median within 0.01 and says so; l1 records neither and stays unchecked. smoke_native_server.sh now expects l1 to produce a headline, a lean and a wire row and checks the wire row's port, call count, gate and read-back postgres port. Co-Authored-By: Claude Opus 5.5 --- benchmarks/experiments/CAMPAIGN.md | 2 +- benchmarks/experiments/PROTOCOL.md | 2 +- benchmarks/experiments/pgwire_gate.py | 24 +++++++++++++++++++ benchmarks/experiments/smoke_native_server.sh | 16 +++++++++---- benchmarks/experiments/test_pgwire_twin.py | 11 +++++++++ 5 files changed, 49 insertions(+), 6 deletions(-) diff --git a/benchmarks/experiments/CAMPAIGN.md b/benchmarks/experiments/CAMPAIGN.md index 2d042637c44..b72766964b4 100644 --- a/benchmarks/experiments/CAMPAIGN.md +++ b/benchmarks/experiments/CAMPAIGN.md @@ -150,7 +150,7 @@ Rules every campaign keeps: | 3 | Dense ingest through `graph_batch(use_wal=True)` with Java float arrays | ArcadeDB only | **built (DECISIONS #116; 1.4x on the laptop)**: `l3d_dense.py` | DECISIONS #116 item 3 | | 4 | The served document and vector arms load through `INSERT ... CONTENT :rows` with the batch bound as one parameter | ArcadeDB only | built (DECISIONS #116, ArcadeData/arcadedb#8337): `l1_tpc.py` (`ArcadeServerTPC.load_batch`, `BENCH_SERVED_LOAD_BATCH`), `l3d_dense.py`, `l3_sparse.py`; parity, since every comparator loads through its vendor's bulk path. **No batch-size sweep is recorded** | `.notes` repros/served-doc-ingest; BUGS F151, F152 | | 5 | The strict durability setting is ArcadeDB `txWalFlush=1`, matched by effect to SQLite `synchronous=FULL`, not `txWalFlush=2` | ArcadeDB only | built (the user, DECISIONS #147; own issue #138): `bench_common.ARCADE_STRICT_TX_WAL_FLUSH`, `test_strict_durability.py`; the October string is kept for rows measured at 2; PROTOCOL section 7 has the row | `concepts/transactions.adoc` in upstream's docs; strace of the three settings on the laptop, 2026-09-24 | -| 6 | Served SQL arms over the Postgres wire (psycopg) instead of HTTP (`requests`) | ArcadeDB only | unblocked: #8288 fixed in ArcadeData/arcadedb#8308, and pgjdbc's default prepare in #8315 (#8244), both merged for 26.10.1 and verified; **decided 2026-09-26 (DECISIONS #116): measure on the bench host at the re-pin and switch if it is the faster read path** (the laptop cannot time below a millisecond; the maintainer already says it is, on #8332). **Built for the 26.11.1 four-arm pass (DECISIONS #166, #171, #237 (3)):** the served SQL cells run a third time as a twin job over the wire (`runner.PGWIRE_SQL_ARMS`, run_id ending `_clientpgwire`, `lean_http.PgSession` over psycopg, never prepared, autocommit), a labelled sensitivity row and never a headline (it flatters ArcadeDB most, #166 addendum 2). Cells: l1 and l1tpc `arcadedb_server` (oltp, olap), l3s `arcadedb_sparse_server` and `_fp32`, l3d `arcadedb_dense_server` and `_int8`, l4 `arcadedb_ts_doc_server` and `arcadedb_ts_native_server`; not l2 (Cypher stays on HTTP, #162), e2, lifecycle, restart, e4. Only `/query` SQL goes over the wire: every write, DDL, transaction, bulk load and probe stays on HTTP (#171 (c)), so a twin row's write-phase numbers are lean HTTP and are not quoted. The twin's server starts the postgres plugin with explicit settings (`runner.PGWIRE_SERVER_SETTINGS`), read back from the engine and refused if unlisted or different. Answers-identical gate (`pgwire_gate.py`): each `res__digest` of the twin must equal the headline's for the same cell, under the lane's own declared coercions, else the row is kept with `error` set and the reason recorded; the smoke is in the PR body. Still open: the mini run and the numbers | `.notes` repros/protocol-latency | +| 6 | Served SQL arms over the Postgres wire (psycopg) instead of HTTP (`requests`) | ArcadeDB only | unblocked: #8288 fixed in ArcadeData/arcadedb#8308, and pgjdbc's default prepare in #8315 (#8244), both merged for 26.10.1 and verified; **decided 2026-09-26 (DECISIONS #116): measure on the bench host at the re-pin and switch if it is the faster read path** (the laptop cannot time below a millisecond; the maintainer already says it is, on #8332). **Built for the 26.11.1 four-arm pass (DECISIONS #166, #171, #237 (3)):** the served SQL cells run a third time as a twin job over the wire (`runner.PGWIRE_SQL_ARMS`, run_id ending `_clientpgwire`, `lean_http.PgSession` over psycopg, never prepared, autocommit), a labelled sensitivity row and never a headline (it flatters ArcadeDB most, #166 addendum 2). Cells: l1 and l1tpc `arcadedb_server` (oltp, olap), l3s `arcadedb_sparse_server` and `_fp32`, l3d `arcadedb_dense_server` and `_int8`, l4 `arcadedb_ts_doc_server` and `arcadedb_ts_native_server`; not l2 (Cypher stays on HTTP, #162), e2, lifecycle, restart, e4. Only `/query` SQL goes over the wire: every write, DDL, transaction, bulk load and probe stays on HTTP (#171 (c)), so a twin row's write-phase numbers are lean HTTP and are not quoted. The twin's server starts the postgres plugin with explicit settings (`runner.PGWIRE_SERVER_SETTINGS`), read back from the engine and refused if unlisted or different. Answers-identical gate (`pgwire_gate.py`): each `res__digest` of the twin must equal the headline's for the same cell, under the lane's own declared coercions (l1tpc, l4), else the row is kept with `error` set and the reason recorded; l3s and l3d record no digest and are compared by `recall_at_10` within 0.01 of the headline median; l1 records neither, so its wire row is `unchecked` until the lane records digests; the smoke is in the PR body. Still open: the mini run and the numbers | `.notes` repros/protocol-latency | | 7 | What the memory column means: the peak, the working set after a full GC, or the peak with the heap printed beside it | every JVM arm | **decided (the user, DECISIONS #116 item 7): the peak, what an operator has to provision, with the heap setting printed beside it for JVM engines; built** (a `JVM heap` column from each cell's rows in `export_web`; the JVM memory note on every table that shows a JVM arm, BUGS F135). **No generated sentence may phrase the column as efficiency ("uses less memory"): a JVM's peak is the heap it was given; the like-for-like answer is the memory sweep.** **Still open, the memory sweep (the user, 2026-09-27), planned after the chain and the re-pin; it is an instrument change:** the smallest memory each arm completes a cell with. Design, settled before it runs: JVM arms (ArcadeDB both deployments, Neo4j) sweep the heap at a fixed cap, non-JVM arms the container cap; "completes" means the same answer digest within a stated slowdown of the full-memory run; one or two tables first (documents SF10, graph SF10); mini only, one pin; it costs days of bench time | DECISIONS #116 item 7 | | 8 | The sparse corpus generator yields NumPy, and each adapter converts in its own timer | comparators too (same work, moved) | measured 3.7% of ArcadeDB's build; do it with the next sparse measurement | bullet below | | 9 | The re-pin build: the official 26.10.1 with every fix we filed verified by its repro, comparators to latest stable (DECISIONS #129, relaxed by #163 to #165) | routine (steps 3 and 4 of the routine) | **built and verified.** The pin is the release's own jar commit `d36b4ca3a`; each fix we filed was verified by its repro on the laptop (the list is `.notes` issues-trail), and #8626, #8852, #8797, and #8725 landed after and are expected to move published cells. **Read the rows with these in hand, and say so on the page:** some cells get SLOWER (the embedded cross-model strict cells, whose `LSM_VECTOR` build had switched that thread's WAL off, #8129 fixed by #8221, and the sparse table's int8 arms, #8582); the dense and cross-model tables moved with the vector library (#8518), so compare recall at the same settings before reading a latency change; ranks can hold while times drop, because item 2 binds values on every engine. Laptop ratios are not projections of the page; only the re-measure decides. **Still owed:** re-verify #7146, #7183, #7184, and #7190 at 10M on the 26.10.1 pair; #7260 is fixed in 26.11.1 (PR #9289) and not in 26.10.1 | `.notes` issues-trail | diff --git a/benchmarks/experiments/PROTOCOL.md b/benchmarks/experiments/PROTOCOL.md index 1fe961e8fdc..d523292beb5 100644 --- a/benchmarks/experiments/PROTOCOL.md +++ b/benchmarks/experiments/PROTOCOL.md @@ -263,7 +263,7 @@ An override nobody can find is a defect, whichever way it moves the number. Ever | all dense arms | `EF_CONSTRUCTION=100`, `EF_SEARCH=100` | operating-point | Shared beam. At the 26.10.1 tag an explicit `efSearch` is a fixed beam of `max(k, efSearch)` while the adaptive default is `max(2k, 20)` on graphs of 10,000 or more vectors, so it is NOT a no-op for ArcadeDB; some comparators move down from their own defaults, unaudited. | lane docstring only | | ArcadeDB server (all arms) | `ARCADEDB_OPTS_GC=` is cleared explicitly, and `JAVA_OPTS` carries `-XX:+UseCompactObjectHeaders` | operating-point | Makes E4 a transport comparison. `ARCADEDB_OPTS_GC` survives a `JAVA_OPTS` override, so clearing it puts both arms on one collector. | DECISIONS #54, enforced by `verify_pair_c25.sh` | | ArcadeDB server vs embedded | both arms run ONE set of upstream jars on ONE JVM (Corretto 25) | fairness | The axis isolates transport, not JVM major, libc, collector, or object-header layout; `-XX:+UseCompactObjectHeaders` is rejected by JDK 21. | DECISIONS #54, enforced by `verify_pair_c25.sh` (jars AND JVM major on both sides) | -| ArcadeDB SERVED arms | the headline HTTP client is the official `arcadedb-driver` 0.2.0 (sync); the same cell runs again on the lean HTTP client as a twin job | the vendor's own client, the same class as the comparators' official clients | Protocol audit round 3: the driver costs about 3x the lean client on a one-row read, so a served latency claim names the client. Bulk loads, the server probe, and transactions stay on the lean HTTP client. The served SQL cells (l1 and l1tpc documents, l3s sparse, l3d dense, l4 time series; not Cypher, not l2 or e2) run a third time as a PostgreSQL-wire twin (psycopg, unnamed statements, `_clientpgwire`): a labelled sensitivity row, never a headline. Only the read and query statements are timed on the wire; writes, DDL, transactions and bulk loads stay on HTTP, so a wire row's write-phase numbers are lean HTTP and are not quoted. The twin's server runs the postgres plugin with explicit, read-back settings, and the twin row passes only if every query digest equals the headline's under the lane's declared coercions (`pgwire_gate.py`); a mismatch keeps the row, marked failed with the reason. | row fields `arcadedb_http_client` and `arcadedb_http_client_role` (`headline` or `sensitivity`); on a wire row also `arcadedb_client_wire`, `arcadedb_pgwire_client`, `arcadedb_pgwire_scope`, `arcadedb_pgwire_calls`, `arcadedb_pgwire_fallback_reasons`, `arcadedb_pgwire_port_open`, `arcadedb_pgwire_gate` and the plugin's settings in `server_settings` | +| ArcadeDB SERVED arms | the headline HTTP client is the official `arcadedb-driver` 0.2.0 (sync); the same cell runs again on the lean HTTP client as a twin job | the vendor's own client, the same class as the comparators' official clients | Protocol audit round 3: the driver costs about 3x the lean client on a one-row read, so a served latency claim names the client. Bulk loads, the server probe, and transactions stay on the lean HTTP client. The served SQL cells (l1 and l1tpc documents, l3s sparse, l3d dense, l4 time series; not Cypher, not l2 or e2) run a third time as a PostgreSQL-wire twin (psycopg, unnamed statements, `_clientpgwire`): a labelled sensitivity row, never a headline. Only the read and query statements are timed on the wire; writes, DDL, transactions and bulk loads stay on HTTP, so a wire row's write-phase numbers are lean HTTP and are not quoted. The twin's server runs the postgres plugin with explicit, read-back settings, and the twin row passes only if every query digest equals the headline's under the lane's declared coercions (`pgwire_gate.py`: l1tpc and l4 record digests; l3s and l3d are compared by `recall_at_10` within 0.01 of the headline median, not by digest; l1 records neither, so its wire row stays `unchecked`); a mismatch keeps the row, marked failed with the reason. | row fields `arcadedb_http_client` and `arcadedb_http_client_role` (`headline` or `sensitivity`); on a wire row also `arcadedb_client_wire`, `arcadedb_pgwire_client`, `arcadedb_pgwire_scope`, `arcadedb_pgwire_calls`, `arcadedb_pgwire_fallback_reasons`, `arcadedb_pgwire_port_open`, `arcadedb_pgwire_gate` and the plugin's settings in `server_settings` | | l4 `q_high` and `q_groupby` | ArcadeDB embedded `to_arrow()`, DuckDB `to_arrow_table()`, SQLite `fetchall()` (no columnar path); `to_list()` / `fetchall()` timed beside it as `_plain_ms` | each engine on its documented best path for a result of thousands of rows | The bindings' docs recommend a bulk read for large results; small results (1 to 100 rows) are unchanged. | row fields `read_path`, `read_path_queries`, `read_path_plain`, `read_path_sensitivity`, `_plain_ms`, `_plain_digest`, `_plain_digest_match` (the cell ends if the typed and plain digests differ; DuckDB's Arrow rows are read by position, like its `fetchall()`) | | JVM initial heap (`-Xms`) per engine | **ArcadeDB, embedded and served: unset** (only `-Xmx` from the cell's memory budget; the served arms through `ARCADEDB_OPTS_MEMORY=-Xmx{heap}`, DECISIONS #200). Elasticsearch: `ES_JAVA_OPTS=-Xms{heap} -Xmx{heap}`, unchanged. Neo4j: `server.memory.heap.initial_size` = `max_size`, unchanged. QuestDB: the harness sets no heap (image defaults), unchanged | ArcadeDB as its maintainers advise for a container with a fixed memory budget (no `-Xms`; ArcadeData/arcadedb#9167) and as the bindings document, so both deployments run one JVM configuration; each other server engine as its own documentation recommends | ArcadeDB's server script (`package/src/main/scripts/server.sh`) leaves the heap to the JVM unless `ARCADEDB_OPTS_MEMORY` is set and shows `-Xms2G -Xmx2G` as its example; Elasticsearch: set `Xms` and `Xmx` to the same value (https://www.elastic.co/guide/en/elasticsearch/reference/current/advanced-configuration.html#set-jvm-heap-size); Neo4j: set initial and maximum heap to the same value (https://neo4j.com/docs/operations-manual/current/performance/memory-configuration/); QuestDB: no setting by the harness (https://questdb.com/docs/configuration/). | `server_heap` / the cell's heap fields, as before | | ArcadeDB server (all arms) | `ARCADEDB_OPTS_MEMORY=-Xmx{heap}` (image ships 2G) | resource fitting | Heap parity per tier. The 2 GB default cost the server 15% p50 and 35% build time at 10M. | T5 caption, FAIRNESS.md F3, `server_heap` read back | diff --git a/benchmarks/experiments/pgwire_gate.py b/benchmarks/experiments/pgwire_gate.py index f8cc334869d..950f5fcdfa1 100644 --- a/benchmarks/experiments/pgwire_gate.py +++ b/benchmarks/experiments/pgwire_gate.py @@ -26,6 +26,12 @@ DIGEST_RE = re.compile(r"^res_(.+)_digest$") WIRE = "pgwire" +# THE LANES THAT RECORD NO DIGEST. l3s and l3d answer an approximate top-k and are checked against exact ground truth, row by row (`recall_at_10`, +# computed outside the timer from the ids the timed call returned); a wire that returned other ids would show as another recall. Two builds of one +# index need not be identical, so the gate compares the twin's recall with the headline's median within RECALL_TOLERANCE and says so (it is not +# a digest). l1 (the synthetic orders lane) records neither, so its wire row stays `unchecked`. +RECALL_FIELD = "recall_at_10" +RECALL_TOLERANCE = 0.01 CELL_KEY = ("lane", "scale", "workload", "backend", "durability_class", "server_build") @@ -61,6 +67,8 @@ def check(twin, headline_rows): seen.setdefault(q, set()).add(d) mine = digests(twin) reasons, unchecked, checked = [], [], 0 + if heads and not seen: + return _check_recall(twin, heads) if not heads or not seen: return {"status": "unchecked", "queries_checked": 0, "reasons": [], "unchecked": ["no headline row with digests for this cell"]} for q in sorted(seen): @@ -78,6 +86,22 @@ def check(twin, headline_rows): return {"status": "failed" if reasons else "ok" if checked else "unchecked", "queries_checked": checked, "reasons": reasons, "unchecked": unchecked} +def _check_recall(twin, heads): + """The gate for a lane with no digests: the twin's recall against the headline's median (RECALL_FIELD), or `unchecked` when either side has none.""" + ref = sorted(float(h[RECALL_FIELD]) for h in heads if h.get(RECALL_FIELD) not in (None, "")) + mine = twin.get(RECALL_FIELD) + if not ref or mine in (None, ""): + return {"status": "unchecked", "queries_checked": 0, "reasons": [], + "unchecked": ["the lane records no per-query digest and no recall for this cell, so the wire's answers were not compared"]} + med = ref[len(ref) // 2] if len(ref) % 2 else (ref[len(ref) // 2 - 1] + ref[len(ref) // 2]) / 2 + off = abs(float(mine) - med) + note = f"{RECALL_FIELD}: compared within {RECALL_TOLERANCE} of the headline median, not by digest" + if off > RECALL_TOLERANCE: + return {"status": "failed", "queries_checked": 0, "unchecked": [], + "reasons": [f"{RECALL_FIELD}: the pg-wire row has {float(mine):.4f}, the headline median is {med:.4f} (more than {RECALL_TOLERANCE} apart)"]} + return {"status": "ok", "queries_checked": 1, "reasons": [], "unchecked": [note]} + + def stamp(row, headline_rows): """Record the gate on the twin row; a failure also sets `error` (the row stays, marked failed, reason named). Returns the result.""" res = check(row, headline_rows) diff --git a/benchmarks/experiments/smoke_native_server.sh b/benchmarks/experiments/smoke_native_server.sh index 5db221cd449..5903e9f77e5 100755 --- a/benchmarks/experiments/smoke_native_server.sh +++ b/benchmarks/experiments/smoke_native_server.sh @@ -47,21 +47,29 @@ import arm_identity as AI rows = [json.loads(l) for l in open(os.environ["RESULTS_FILE"]) if l.strip()] bad = [] want = {"l2": "arcadedb_graph_server", "l1": "arcadedb_server"} -# every served ArcadeDB arm runs twice: the headline client (the official arcadedb-driver) and its sensitivity twin (lean_http) +# every served ArcadeDB arm runs twice: the headline client (the official arcadedb-driver) and its sensitivity twin (lean_http); a served SQL cell +# (l1 here) a third time, over the PostgreSQL wire (runner.PGWIRE_SQL_ARMS), which is a second sensitivity row marked arcadedb_client_wire=pgwire checks = [] for lane, arm in want.items(): got = [r for r in rows if r.get("lane") == lane and r.get("backend") == arm] roles = sorted(str(r.get("arcadedb_http_client_role")) for r in got) - if roles != ["headline", "sensitivity"]: - bad.append(f"{lane}/{arm}: client roles {roles}, expected one headline and one sensitivity row") + wired = [r for r in got if r.get("arcadedb_client_wire") == "pgwire"] + want_roles = ["headline", "sensitivity"] + (["sensitivity"] if lane == "l1" else []) + if roles != want_roles or len(wired) != (1 if lane == "l1" else 0): + bad.append(f"{lane}/{arm}: client roles {roles} with {len(wired)} wire rows, expected {want_roles} with {1 if lane == 'l1' else 0}") continue checks += [(lane, arm, r) for r in got] for lane, arm, r in checks: - arm = f"{arm}[{r.get('arcadedb_http_client_role')}]" + arm = f"{arm}[{r.get('arcadedb_http_client_role')}{'/pgwire' if r.get('arcadedb_client_wire') == 'pgwire' else ''}]" def need(cond, msg): if not cond: bad.append(f"{lane}/{arm}: {msg}") need(not r.get("error") and r.get("rc") == 0, f"error={r.get('error')!r} rc={r.get('rc')!r}") + if r.get("arcadedb_client_wire") == "pgwire": + g = r.get("arcadedb_pgwire_gate") or {} + need(r.get("arcadedb_pgwire_port_open") is True and (r.get("arcadedb_pgwire_calls") or 0) > 0, f"no statement went over the wire: port_open={r.get('arcadedb_pgwire_port_open')!r} calls={r.get('arcadedb_pgwire_calls')!r}") + need(g.get("status") in ("ok", "unchecked"), f"answers gate: {g!r}") # l1 records no digest, so its wire row is `unchecked`; l1tpc is `ok` + need(str((r.get("server_settings") or {}).get("arcadedb.postgres.port")) == "5432", f"postgres port not read back: {r.get('server_settings')!r}") need(r.get("engine_backend") == "native" and r.get("server_build") == "native", f"engine_backend={r.get('engine_backend')!r} server_build={r.get('server_build')!r}") need(AI.build_of(r) == "native" and AI.served(r), "arm_identity does not read it as a native served row") need(r.get("cpu_baseline", "").startswith("native server image: -march="), f"cpu_baseline={r.get('cpu_baseline')!r}") diff --git a/benchmarks/experiments/test_pgwire_twin.py b/benchmarks/experiments/test_pgwire_twin.py index c2ab9df49d2..e0a6b8e9721 100644 --- a/benchmarks/experiments/test_pgwire_twin.py +++ b/benchmarks/experiments/test_pgwire_twin.py @@ -297,6 +297,17 @@ def test_a_headline_that_disagrees_with_itself_leaves_that_query_unchecked(): assert res["status"] == "ok" and res["queries_checked"] == 1 and "q1" in res["unchecked"][0] +def test_a_recall_lane_is_compared_by_recall_within_a_tolerance_and_says_so(): + head = [dict(_row(), lane="l3d", recall_at_10=r) for r in (0.95, 0.96, 0.97)] + twin = dict(_row(True), lane="l3d") + assert gate.check(dict(twin, recall_at_10=0.955), head)["status"] == "ok" + bad = gate.check(dict(twin, recall_at_10=0.5), head) + assert bad["status"] == "failed" and "0.5000" in bad["reasons"][0] + assert gate.check(twin, head)["status"] == "unchecked" + assert "not by digest" in gate.check(dict(twin, recall_at_10=0.96), head)["unchecked"][0] + assert gate.check(dict(_row(True), recall_at_10=0.9), [_row()])["status"] == "unchecked" # a lane with neither (l1) is never a pass + + def test_a_declared_absence_is_not_a_digest(): assert gate.digests({"res_q_digest": "unexpressible: no such operator", "res_r_digest": "abcd"}) == {"r": "abcd"} From 952ef54c5465e7a27a7a588d893ac564dbd82375 Mon Sep 17 00:00:00 2001 From: Taewoon Kim Date: Wed, 4 Nov 2026 01:17:21 +0900 Subject: [PATCH 3/3] benchmarks: the PostgreSQL-wire twin runs at the small size tier only The twin re-runs the whole cell (load, index build, queries) for a sensitivity row, so at the large tier (deep10m, tpch10, sf1full) it would add hours to a pass. build_jobs takes the invocation's scale and builds no wire twin when it is in LARGE_SCALES, which moves from make_2611_stages.py to runner.py (the generator reads it from there: one definition). The lean twin is unchanged at every scale. DECISIONS #166's "tier 1 only" is the ENGINE tier, which the twin already satisfies (ArcadeDB arms only); the size cut is an added cost decision, stated in PROTOCOL.md and CAMPAIGN.md row 6. Co-Authored-By: Claude Opus 5.5 --- benchmarks/experiments/CAMPAIGN.md | 2 +- benchmarks/experiments/PROTOCOL.md | 2 +- benchmarks/experiments/make_2611_stages.py | 2 +- benchmarks/experiments/runner.py | 13 +++++--- benchmarks/experiments/test_pgwire_twin.py | 35 +++++++++++++++++----- 5 files changed, 40 insertions(+), 14 deletions(-) diff --git a/benchmarks/experiments/CAMPAIGN.md b/benchmarks/experiments/CAMPAIGN.md index b72766964b4..3970109a639 100644 --- a/benchmarks/experiments/CAMPAIGN.md +++ b/benchmarks/experiments/CAMPAIGN.md @@ -150,7 +150,7 @@ Rules every campaign keeps: | 3 | Dense ingest through `graph_batch(use_wal=True)` with Java float arrays | ArcadeDB only | **built (DECISIONS #116; 1.4x on the laptop)**: `l3d_dense.py` | DECISIONS #116 item 3 | | 4 | The served document and vector arms load through `INSERT ... CONTENT :rows` with the batch bound as one parameter | ArcadeDB only | built (DECISIONS #116, ArcadeData/arcadedb#8337): `l1_tpc.py` (`ArcadeServerTPC.load_batch`, `BENCH_SERVED_LOAD_BATCH`), `l3d_dense.py`, `l3_sparse.py`; parity, since every comparator loads through its vendor's bulk path. **No batch-size sweep is recorded** | `.notes` repros/served-doc-ingest; BUGS F151, F152 | | 5 | The strict durability setting is ArcadeDB `txWalFlush=1`, matched by effect to SQLite `synchronous=FULL`, not `txWalFlush=2` | ArcadeDB only | built (the user, DECISIONS #147; own issue #138): `bench_common.ARCADE_STRICT_TX_WAL_FLUSH`, `test_strict_durability.py`; the October string is kept for rows measured at 2; PROTOCOL section 7 has the row | `concepts/transactions.adoc` in upstream's docs; strace of the three settings on the laptop, 2026-09-24 | -| 6 | Served SQL arms over the Postgres wire (psycopg) instead of HTTP (`requests`) | ArcadeDB only | unblocked: #8288 fixed in ArcadeData/arcadedb#8308, and pgjdbc's default prepare in #8315 (#8244), both merged for 26.10.1 and verified; **decided 2026-09-26 (DECISIONS #116): measure on the bench host at the re-pin and switch if it is the faster read path** (the laptop cannot time below a millisecond; the maintainer already says it is, on #8332). **Built for the 26.11.1 four-arm pass (DECISIONS #166, #171, #237 (3)):** the served SQL cells run a third time as a twin job over the wire (`runner.PGWIRE_SQL_ARMS`, run_id ending `_clientpgwire`, `lean_http.PgSession` over psycopg, never prepared, autocommit), a labelled sensitivity row and never a headline (it flatters ArcadeDB most, #166 addendum 2). Cells: l1 and l1tpc `arcadedb_server` (oltp, olap), l3s `arcadedb_sparse_server` and `_fp32`, l3d `arcadedb_dense_server` and `_int8`, l4 `arcadedb_ts_doc_server` and `arcadedb_ts_native_server`; not l2 (Cypher stays on HTTP, #162), e2, lifecycle, restart, e4. Only `/query` SQL goes over the wire: every write, DDL, transaction, bulk load and probe stays on HTTP (#171 (c)), so a twin row's write-phase numbers are lean HTTP and are not quoted. The twin's server starts the postgres plugin with explicit settings (`runner.PGWIRE_SERVER_SETTINGS`), read back from the engine and refused if unlisted or different. Answers-identical gate (`pgwire_gate.py`): each `res__digest` of the twin must equal the headline's for the same cell, under the lane's own declared coercions (l1tpc, l4), else the row is kept with `error` set and the reason recorded; l3s and l3d record no digest and are compared by `recall_at_10` within 0.01 of the headline median; l1 records neither, so its wire row is `unchecked` until the lane records digests; the smoke is in the PR body. Still open: the mini run and the numbers | `.notes` repros/protocol-latency | +| 6 | Served SQL arms over the Postgres wire (psycopg) instead of HTTP (`requests`) | ArcadeDB only | unblocked: #8288 fixed in ArcadeData/arcadedb#8308, and pgjdbc's default prepare in #8315 (#8244), both merged for 26.10.1 and verified; **decided 2026-09-26 (DECISIONS #116): measure on the bench host at the re-pin and switch if it is the faster read path** (the laptop cannot time below a millisecond; the maintainer already says it is, on #8332). **Built for the 26.11.1 four-arm pass (DECISIONS #166, #171, #237 (3)):** the served SQL cells run a third time as a twin job over the wire, ArcadeDB tier 1 only (DECISIONS #166) and at the small size tier only: no twin at `runner.LARGE_SCALES` (deep10m, tpch10, sf1full), because the twin re-runs the whole cell including load and index build (`runner.PGWIRE_SQL_ARMS`, run_id ending `_clientpgwire`, `lean_http.PgSession` over psycopg, never prepared, autocommit), a labelled sensitivity row and never a headline (it flatters ArcadeDB most, #166 addendum 2). Cells: l1 and l1tpc `arcadedb_server` (oltp, olap), l3s `arcadedb_sparse_server` and `_fp32`, l3d `arcadedb_dense_server` and `_int8`, l4 `arcadedb_ts_doc_server` and `arcadedb_ts_native_server`; not l2 (Cypher stays on HTTP, #162), e2, lifecycle, restart, e4. Only `/query` SQL goes over the wire: every write, DDL, transaction, bulk load and probe stays on HTTP (#171 (c)), so a twin row's write-phase numbers are lean HTTP and are not quoted. The twin's server starts the postgres plugin with explicit settings (`runner.PGWIRE_SERVER_SETTINGS`), read back from the engine and refused if unlisted or different. Answers-identical gate (`pgwire_gate.py`): each `res__digest` of the twin must equal the headline's for the same cell, under the lane's own declared coercions (l1tpc, l4), else the row is kept with `error` set and the reason recorded; l3s and l3d record no digest and are compared by `recall_at_10` within 0.01 of the headline median; l1 records neither, so its wire row is `unchecked` until the lane records digests; the smoke is in the PR body. Still open: the mini run and the numbers | `.notes` repros/protocol-latency | | 7 | What the memory column means: the peak, the working set after a full GC, or the peak with the heap printed beside it | every JVM arm | **decided (the user, DECISIONS #116 item 7): the peak, what an operator has to provision, with the heap setting printed beside it for JVM engines; built** (a `JVM heap` column from each cell's rows in `export_web`; the JVM memory note on every table that shows a JVM arm, BUGS F135). **No generated sentence may phrase the column as efficiency ("uses less memory"): a JVM's peak is the heap it was given; the like-for-like answer is the memory sweep.** **Still open, the memory sweep (the user, 2026-09-27), planned after the chain and the re-pin; it is an instrument change:** the smallest memory each arm completes a cell with. Design, settled before it runs: JVM arms (ArcadeDB both deployments, Neo4j) sweep the heap at a fixed cap, non-JVM arms the container cap; "completes" means the same answer digest within a stated slowdown of the full-memory run; one or two tables first (documents SF10, graph SF10); mini only, one pin; it costs days of bench time | DECISIONS #116 item 7 | | 8 | The sparse corpus generator yields NumPy, and each adapter converts in its own timer | comparators too (same work, moved) | measured 3.7% of ArcadeDB's build; do it with the next sparse measurement | bullet below | | 9 | The re-pin build: the official 26.10.1 with every fix we filed verified by its repro, comparators to latest stable (DECISIONS #129, relaxed by #163 to #165) | routine (steps 3 and 4 of the routine) | **built and verified.** The pin is the release's own jar commit `d36b4ca3a`; each fix we filed was verified by its repro on the laptop (the list is `.notes` issues-trail), and #8626, #8852, #8797, and #8725 landed after and are expected to move published cells. **Read the rows with these in hand, and say so on the page:** some cells get SLOWER (the embedded cross-model strict cells, whose `LSM_VECTOR` build had switched that thread's WAL off, #8129 fixed by #8221, and the sparse table's int8 arms, #8582); the dense and cross-model tables moved with the vector library (#8518), so compare recall at the same settings before reading a latency change; ranks can hold while times drop, because item 2 binds values on every engine. Laptop ratios are not projections of the page; only the re-measure decides. **Still owed:** re-verify #7146, #7183, #7184, and #7190 at 10M on the 26.10.1 pair; #7260 is fixed in 26.11.1 (PR #9289) and not in 26.10.1 | `.notes` issues-trail | diff --git a/benchmarks/experiments/PROTOCOL.md b/benchmarks/experiments/PROTOCOL.md index d523292beb5..9f10da1e322 100644 --- a/benchmarks/experiments/PROTOCOL.md +++ b/benchmarks/experiments/PROTOCOL.md @@ -263,7 +263,7 @@ An override nobody can find is a defect, whichever way it moves the number. Ever | all dense arms | `EF_CONSTRUCTION=100`, `EF_SEARCH=100` | operating-point | Shared beam. At the 26.10.1 tag an explicit `efSearch` is a fixed beam of `max(k, efSearch)` while the adaptive default is `max(2k, 20)` on graphs of 10,000 or more vectors, so it is NOT a no-op for ArcadeDB; some comparators move down from their own defaults, unaudited. | lane docstring only | | ArcadeDB server (all arms) | `ARCADEDB_OPTS_GC=` is cleared explicitly, and `JAVA_OPTS` carries `-XX:+UseCompactObjectHeaders` | operating-point | Makes E4 a transport comparison. `ARCADEDB_OPTS_GC` survives a `JAVA_OPTS` override, so clearing it puts both arms on one collector. | DECISIONS #54, enforced by `verify_pair_c25.sh` | | ArcadeDB server vs embedded | both arms run ONE set of upstream jars on ONE JVM (Corretto 25) | fairness | The axis isolates transport, not JVM major, libc, collector, or object-header layout; `-XX:+UseCompactObjectHeaders` is rejected by JDK 21. | DECISIONS #54, enforced by `verify_pair_c25.sh` (jars AND JVM major on both sides) | -| ArcadeDB SERVED arms | the headline HTTP client is the official `arcadedb-driver` 0.2.0 (sync); the same cell runs again on the lean HTTP client as a twin job | the vendor's own client, the same class as the comparators' official clients | Protocol audit round 3: the driver costs about 3x the lean client on a one-row read, so a served latency claim names the client. Bulk loads, the server probe, and transactions stay on the lean HTTP client. The served SQL cells (l1 and l1tpc documents, l3s sparse, l3d dense, l4 time series; not Cypher, not l2 or e2) run a third time as a PostgreSQL-wire twin (psycopg, unnamed statements, `_clientpgwire`): a labelled sensitivity row, never a headline. Only the read and query statements are timed on the wire; writes, DDL, transactions and bulk loads stay on HTTP, so a wire row's write-phase numbers are lean HTTP and are not quoted. The twin's server runs the postgres plugin with explicit, read-back settings, and the twin row passes only if every query digest equals the headline's under the lane's declared coercions (`pgwire_gate.py`: l1tpc and l4 record digests; l3s and l3d are compared by `recall_at_10` within 0.01 of the headline median, not by digest; l1 records neither, so its wire row stays `unchecked`); a mismatch keeps the row, marked failed with the reason. | row fields `arcadedb_http_client` and `arcadedb_http_client_role` (`headline` or `sensitivity`); on a wire row also `arcadedb_client_wire`, `arcadedb_pgwire_client`, `arcadedb_pgwire_scope`, `arcadedb_pgwire_calls`, `arcadedb_pgwire_fallback_reasons`, `arcadedb_pgwire_port_open`, `arcadedb_pgwire_gate` and the plugin's settings in `server_settings` | +| ArcadeDB SERVED arms | the headline HTTP client is the official `arcadedb-driver` 0.2.0 (sync); the same cell runs again on the lean HTTP client as a twin job | the vendor's own client, the same class as the comparators' official clients | Protocol audit round 3: the driver costs about 3x the lean client on a one-row read, so a served latency claim names the client. Bulk loads, the server probe, and transactions stay on the lean HTTP client. The served SQL cells (l1 and l1tpc documents, l3s sparse, l3d dense, l4 time series; not Cypher, not l2 or e2) run a third time as a PostgreSQL-wire twin (psycopg, unnamed statements, `_clientpgwire`) at the small size tier only (not at `runner.LARGE_SCALES`: deep10m, tpch10, sf1full, because the twin re-runs the whole cell including load and index build): a labelled sensitivity row, never a headline. Only the read and query statements are timed on the wire; writes, DDL, transactions and bulk loads stay on HTTP, so a wire row's write-phase numbers are lean HTTP and are not quoted. The twin's server runs the postgres plugin with explicit, read-back settings, and the twin row passes only if every query digest equals the headline's under the lane's declared coercions (`pgwire_gate.py`: l1tpc and l4 record digests; l3s and l3d are compared by `recall_at_10` within 0.01 of the headline median, not by digest; l1 records neither, so its wire row stays `unchecked`); a mismatch keeps the row, marked failed with the reason. | row fields `arcadedb_http_client` and `arcadedb_http_client_role` (`headline` or `sensitivity`); on a wire row also `arcadedb_client_wire`, `arcadedb_pgwire_client`, `arcadedb_pgwire_scope`, `arcadedb_pgwire_calls`, `arcadedb_pgwire_fallback_reasons`, `arcadedb_pgwire_port_open`, `arcadedb_pgwire_gate` and the plugin's settings in `server_settings` | | l4 `q_high` and `q_groupby` | ArcadeDB embedded `to_arrow()`, DuckDB `to_arrow_table()`, SQLite `fetchall()` (no columnar path); `to_list()` / `fetchall()` timed beside it as `_plain_ms` | each engine on its documented best path for a result of thousands of rows | The bindings' docs recommend a bulk read for large results; small results (1 to 100 rows) are unchanged. | row fields `read_path`, `read_path_queries`, `read_path_plain`, `read_path_sensitivity`, `_plain_ms`, `_plain_digest`, `_plain_digest_match` (the cell ends if the typed and plain digests differ; DuckDB's Arrow rows are read by position, like its `fetchall()`) | | JVM initial heap (`-Xms`) per engine | **ArcadeDB, embedded and served: unset** (only `-Xmx` from the cell's memory budget; the served arms through `ARCADEDB_OPTS_MEMORY=-Xmx{heap}`, DECISIONS #200). Elasticsearch: `ES_JAVA_OPTS=-Xms{heap} -Xmx{heap}`, unchanged. Neo4j: `server.memory.heap.initial_size` = `max_size`, unchanged. QuestDB: the harness sets no heap (image defaults), unchanged | ArcadeDB as its maintainers advise for a container with a fixed memory budget (no `-Xms`; ArcadeData/arcadedb#9167) and as the bindings document, so both deployments run one JVM configuration; each other server engine as its own documentation recommends | ArcadeDB's server script (`package/src/main/scripts/server.sh`) leaves the heap to the JVM unless `ARCADEDB_OPTS_MEMORY` is set and shows `-Xms2G -Xmx2G` as its example; Elasticsearch: set `Xms` and `Xmx` to the same value (https://www.elastic.co/guide/en/elasticsearch/reference/current/advanced-configuration.html#set-jvm-heap-size); Neo4j: set initial and maximum heap to the same value (https://neo4j.com/docs/operations-manual/current/performance/memory-configuration/); QuestDB: no setting by the harness (https://questdb.com/docs/configuration/). | `server_heap` / the cell's heap fields, as before | | ArcadeDB server (all arms) | `ARCADEDB_OPTS_MEMORY=-Xmx{heap}` (image ships 2G) | resource fitting | Heap parity per tier. The 2 GB default cost the server 15% p50 and 35% build time at 10M. | T5 caption, FAIRNESS.md F3, `server_heap` read back | diff --git a/benchmarks/experiments/make_2611_stages.py b/benchmarks/experiments/make_2611_stages.py index 5981f813799..4931091a1d0 100644 --- a/benchmarks/experiments/make_2611_stages.py +++ b/benchmarks/experiments/make_2611_stages.py @@ -574,7 +574,7 @@ def profile_report(profile): # (BIGGEST: Memgraph's deep10m dense cells, SurrealDB's documents at tpch1, the full SF1 network's analytics) run last of # all, in stages of their own, where they are easiest to drop if time runs short. Splitting changes the stage boundaries # only: stage_problems proves the cells are the same. -LARGE_SCALES = ("deep10m", "tpch10", "sf1full") +LARGE_SCALES = runner.LARGE_SCALES # one definition: runner.build_jobs reads it too (the PostgreSQL-wire twin runs at the small tier only) BIGGEST = (("l3d", ("memgraph_dense",), ("deep10m",)), ("l1tpc", ("surrealdb_tpc",), ("tpch1",)), ("l2", None, ("sf1full",))) diff --git a/benchmarks/experiments/runner.py b/benchmarks/experiments/runner.py index 55b1ba0b3c6..059b70fa897 100644 --- a/benchmarks/experiments/runner.py +++ b/benchmarks/experiments/runner.py @@ -4533,6 +4533,10 @@ def restricted_arms(lane): # measures the client axis itself), arcadedb_imgdefaults_server (a JVM-defaults arm with no twin of its own). Within a covered cell only `/query` SQL # goes over the wire; writes, DDL, transactions and bulk loads stay on HTTP (#171 (c)). PGWIRE_CLIENT = "pgwire" +# THE WIRE TWIN RUNS AT THE SMALL TIER ONLY. It re-runs the whole cell (load, index build, queries) for a sensitivity row, so at the large tier it would add +# hours to a pass; DECISIONS #166 sized the A/B as "a third of the hours". The large tier is the stage generator's own size definition (make_2611_stages +# titles its stages "(small tier)" and "(large tier)"; an ENGINE tier is another thing); it lives here so the generator and build_jobs read one tuple. +LARGE_SCALES = ("deep10m", "tpch10", "sf1full") PGWIRE_SQL_ARMS = {"l1": ("arcadedb_server",), "l1tpc": ("arcadedb_server",), "l3s": ("arcadedb_sparse_server", "arcadedb_sparse_server_fp32"), "l3d": ("arcadedb_dense_server", "arcadedb_dense_server_int8"), @@ -4553,8 +4557,9 @@ def _client_of_run_id(run_id): return m.group(1) if m else "" -def build_jobs(lanes, workloads_arg): - """Jobs for these lanes, optionally narrowed to some workloads. +def build_jobs(lanes, workloads_arg, scale=None): + """Jobs for these lanes, optionally narrowed to some workloads. `scale` is the scale the invocation runs: a PostgreSQL-wire twin is built only + when it is not a large-tier scale (LARGE_SCALES); None (no scale named) builds it. THE ARGUMENT USED TO BE IGNORED. The parameter was spelled `_workloads_arg` and the loop iterated the LANE's workload list instead, so @@ -4587,7 +4592,7 @@ def build_jobs(lanes, workloads_arg): head = jobs[-1] jobs.append({**head, "http_client": SENSITIVITY_CLIENT, "run_id": f"{lane}_{be}_{wl}_client{SENSITIVITY_CLIENT}"}) - if be in PGWIRE_SQL_ARMS.get(lane, ()): + if be in PGWIRE_SQL_ARMS.get(lane, ()) and scale not in LARGE_SCALES: jobs.append({**head, "http_client": PGWIRE_CLIENT, "run_id": f"{lane}_{be}_{wl}_client{PGWIRE_CLIENT}"}) if want: @@ -4736,7 +4741,7 @@ def main(): + "). Lane paths resolve INSIDE the container, so pointing a" " lane at a host path instead of fixing the mount cannot work.") - jobs = build_jobs(lanes, workloads) + jobs = build_jobs(lanes, workloads, scale=args.scale) if args.backends: keep = set(args.backends.split(",")) jobs = [j for j in jobs if j["backend"] in keep] diff --git a/benchmarks/experiments/test_pgwire_twin.py b/benchmarks/experiments/test_pgwire_twin.py index e0a6b8e9721..040ede64ffc 100644 --- a/benchmarks/experiments/test_pgwire_twin.py +++ b/benchmarks/experiments/test_pgwire_twin.py @@ -28,8 +28,8 @@ # ------------------------------------------------------------------------------------------------------ which cells get the twin -def test_the_twin_is_emitted_for_exactly_the_served_sql_cells(): - jobs = runner.build_jobs(list(runner.LANES), []) +def test_the_twin_is_emitted_for_exactly_the_served_sql_cells_at_the_small_tier(): + jobs = runner.build_jobs(list(runner.LANES), [], scale="tpch1") pg = [j for j in jobs if j.get("http_client") == "pgwire"] assert {(j["lane"], j["backend"]) for j in pg} == EXPECTED heads = {(j["lane"], j["backend"], j["workload"]) for j in jobs if not j.get("http_client")} @@ -39,6 +39,19 @@ def test_the_twin_is_emitted_for_exactly_the_served_sql_cells(): assert len({j["run_id"] for j in jobs}) == len(jobs) +@pytest.mark.parametrize("scale", ["deep10m", "tpch10", "sf1full"]) +def test_the_large_tier_scales_get_no_wire_twin_and_keep_their_lean_twin(scale): + """The twin re-runs the whole cell (load, index build, queries), so the large tier (the generator's LARGE_SCALES) would add hours for a sensitivity row.""" + jobs = runner.build_jobs(list(runner.LANES), [], scale=scale) + assert not [j for j in jobs if j.get("http_client") == "pgwire"] + assert [j for j in jobs if j.get("http_client") == "lean"] + + +def test_the_large_tier_is_the_generators_own_definition(): + import make_2611_stages as G + assert runner.LARGE_SCALES == G.LARGE_SCALES and set(runner.LARGE_SCALES) == {"deep10m", "tpch10", "sf1full"} + + def test_cypher_mixed_and_lifecycle_cells_have_no_pgwire_twin(): jobs = runner.build_jobs(["l2", "e2", "lifecycle", "restart", "e4"], []) assert jobs and not [j for j in jobs if j.get("http_client") == "pgwire"] @@ -57,14 +70,22 @@ def test_the_stage_generator_schedules_the_twin_for_exactly_the_served_sql_cells specs = G.number(G.subset_specs(engines, G.parse_lanes(None, False, "paper-2611"), "biggest-last", G.profile_skip("paper-2611"), True), "qT") cells = {(c.lane, c.backend, c.workload) for spec in specs for c in G.stage_cells(spec)} assert cells + scales = {(c.lane, c.backend, c.workload): set() for spec in specs for c in G.stage_cells(spec)} + for spec in specs: + for c in G.stage_cells(spec): + scales[(c.lane, c.backend, c.workload)].add(c.scale) served = {c for c in cells if c[1] in runner.ARCADEDB_HTTP_ARMS} - got = set() + got, wired_scales = set(), set() for lane, be, wl in served: - for j in runner.build_jobs([lane], [wl]): - if j["backend"] == be and j.get("http_client") == "pgwire": - got.add((lane, be, wl)) + for sc in scales[(lane, be, wl)]: + for j in runner.build_jobs([lane], [wl], scale=sc): + if j["backend"] == be and j.get("http_client") == "pgwire": + got.add((lane, be, wl)) + wired_scales.add(sc) assert {(l, b) for l, b, _ in got} <= EXPECTED and got - assert all(c in got for c in served if c[:2] in EXPECTED) + # small-tier cells of the covered arms get it, large-tier ones (tpch10, deep10m) do not + assert wired_scales == {"tpch1", "small"} + assert all(c in got for c in served if c[:2] in EXPECTED and scales[c] - set(G.LARGE_SCALES)) assert not [c for c in got if c[0] in ("l2", "e2")]