From 76f94b2bb8a66b574f0222fa66efaad082a76cd5 Mon Sep 17 00:00:00 2001 From: Taewoon Kim Date: Mon, 9 Nov 2026 17:44:19 +0900 Subject: [PATCH 01/49] JVM to_json_list: a FLOAT and a float32 float[] arrive as the exact double, as on the native wheel (refs #314) (#320) RowBatcher.keepNonFinite now widens a Float to a Double and a float[] to a double[] before the engine's JSONObject writes it, so the JSON text is the exact float64 widening of the float32 (0.10000000149011612, not the shortest float32 text 0.1). NaN and infinity keep their tag. tests/test_cross_wheel_golden.py is one fixed table with committed golden literals for to_list, to_json_list and to_columns (FLOAT, DOUBLE, float32 float[] of 3 and 9, DECIMAL, LONG beyond 2**53, DATE, DATETIME, NULL, embedded list), run unchanged on both wheels. Co-authored-by: Claude Sonnet 5.5 (cherry picked from commit 551dee97c7a46c3a51f1f84fe40bb2824704dedb) Co-Authored-By: Claude Sonnet 5.5 --- src/java/com/arcadedb/python/RowBatcher.java | 37 ++-- tests/test_cross_wheel_golden.py | 215 +++++++++++++++++++ 2 files changed, 240 insertions(+), 12 deletions(-) create mode 100644 tests/test_cross_wheel_golden.py diff --git a/src/java/com/arcadedb/python/RowBatcher.java b/src/java/com/arcadedb/python/RowBatcher.java index b1548348..150299ec 100644 --- a/src/java/com/arcadedb/python/RowBatcher.java +++ b/src/java/com/arcadedb/python/RowBatcher.java @@ -53,23 +53,36 @@ private RowBatcher() { * JSON has no NaN or infinity, and the engine's writers turn them into null. This returns {@code value} with every * NaN, Infinity and -Infinity (a Double or Float, or an element of a float[], a double[], a Collection or a Map, at any * depth) replaced by the tag {@code {"\u0001": ["f", "NaN"]}} (the text is {@code Double.toString}), which Python turns - * back into a float. A value with nothing non-finite is returned as it is, with no copy, so the common row costs one - * type test per value. + * back into a float. A Float, and each element of a float[], is also widened to a double (a float[] becomes a double[]), + * so the JSON text is the exact double and Python reads the same float that to_list() returns (#314). A value that is + * neither a Float nor a float[] and has nothing non-finite is returned as it is, with no copy, so the common row costs + * one type test per value. */ static Object keepNonFinite(final Object value) { - if (value instanceof Double || value instanceof Float) { - final double d = ((Number) value).doubleValue(); + if (value instanceof Double) { + final double d = (Double) value; return Double.isFinite(d) ? value : tag(d); } + if (value instanceof Float) { + // widened to a Double: the JSON writer prints a Float as the shortest float32 text ("0.1"), which Python reads as a + // different double than the exact widening of the float32 that to_list() and the native build return + final double d = ((Float) value).doubleValue(); + return Double.isFinite(d) ? (Object) d : tag(d); + } if (value instanceof float[] floats) { - for (final float f : floats) - if (!Float.isFinite(f)) { - final List out = new ArrayList<>(floats.length); - for (final float x : floats) - out.add(Float.isFinite(x) ? (Object) x : tag(x)); - return out; - } - return value; + final double[] wide = new double[floats.length]; + boolean finite = true; + for (int i = 0; i < floats.length; i++) { + wide[i] = floats[i]; + if (!Double.isFinite(wide[i])) + finite = false; + } + if (finite) + return wide; + final List out = new ArrayList<>(wide.length); + for (final double x : wide) + out.add(Double.isFinite(x) ? (Object) x : tag(x)); + return out; } if (value instanceof double[] doubles) { for (final double d : doubles) diff --git a/tests/test_cross_wheel_golden.py b/tests/test_cross_wheel_golden.py new file mode 100644 index 00000000..a5e62209 --- /dev/null +++ b/tests/test_cross_wheel_golden.py @@ -0,0 +1,215 @@ +"""One fixed table, committed golden literals, run on both wheels: the two builds return the same values (#314). + +``tests/`` runs against each installed wheel, but until now a value was checked against one literal in a test +that only one wheel could run, so a difference between the wheels stayed invisible. This file has no branch on +``arcadedb.__backend__``: the golden literals below are asserted on ``arcadedb`` (native) and on ``arcadedb-jvm`` +alike, so a literal that holds on both IS the parity check. + +The table holds a FLOAT, a DOUBLE, two float32 ``float[]`` (3 values, which the native build writes element by +element, and 9, which it sends as a float32 tail), a DECIMAL, a LONG beyond 2**53, a DATE, a DATETIME, a NULL and an +embedded list. Every float32 input is written as the exact float64 widening of that float32, so the stored value is +the literal. ``to_json_list()`` of a FLOAT or a float32 vector used to differ: the JVM build printed the shortest +float32 text (``0.1``), which Python reads as a different double than the exact widening +(``0.10000000149011612``) that the native build and ``to_list()`` return. +""" + +from datetime import date, datetime +from decimal import Decimal + +import numpy as np +import pytest + +SELECT = "SELECT n, f, d, v3, v9, dec, big, day, ts, maybe, lst FROM Golden ORDER BY n" + +# float32 values written as the float64 they widen to, so each is exact in a FLOAT and in a float32 vector +F32_TENTH = 0.10000000149011612 # float32(0.1) +F32_THIRD = 0.3333333432674408 # float32(1/3) +F32_MILLI = 0.0010000000474974513 # float32(0.001) +F32_SEVEN_TENTHS = 0.699999988079071 # float32(0.7) +F32_NINE_TENTHS = 0.8999999761581421 # float32(0.9) + +V9_ROW0 = [ + 0.10000000149011612, + 0.20000000298023224, + 0.30000001192092896, + 0.4000000059604645, + 0.5, + 0.6000000238418579, + 0.699999988079071, + 0.800000011920929, + 0.8999999761581421, +] +V9_ROW1 = [ + F32_MILLI, + F32_SEVEN_TENTHS, + F32_NINE_TENTHS, + 0.5, + F32_THIRD, + F32_TENTH, + 1.0, + 2.0, + -2.75, +] + +ROWS = [ + { + "n": 0, + "f": F32_TENTH, + "d": 0.1, + "v3": [F32_TENTH, F32_THIRD, 0.5], + "v9": V9_ROW0, + "dec": Decimal("1.50"), + "big": 2**53 + 1, + "day": date(2026, 1, 2), + "ts": datetime(2026, 1, 2, 3, 4, 5, 678000), + "maybe": 0.25, + "lst": [1, "a", 2.5], + }, + { + "n": 1, + "f": 0.5, + "d": 1 / 3, + "v3": [0.5, 1.0, -2.75], + "v9": V9_ROW1, + "dec": Decimal("-0.25"), + "big": -(2**53) - 1, + "day": date(1969, 12, 31), + "ts": datetime(1969, 12, 31, 23, 59, 59, 999000), + "maybe": None, + "lst": ["x"], + }, +] + +# to_list: the Python types of the values as stored +GOLDEN_LIST = ROWS + +# to_json_list: JSON-native values; DATE and DATETIME as epoch milliseconds, DECIMAL as a float +GOLDEN_JSON = [ + { + "n": 0, + "f": F32_TENTH, + "d": 0.1, + "v3": [F32_TENTH, F32_THIRD, 0.5], + "v9": V9_ROW0, + "dec": 1.5, + "big": 9007199254740993, + "day": 1767312000000, + "ts": 1767323045678, + "maybe": 0.25, + "lst": [1, "a", 2.5], + }, + { + "n": 1, + "f": 0.5, + "d": 1 / 3, + "v3": [0.5, 1.0, -2.75], + "v9": V9_ROW1, + "dec": -0.25, + "big": -9007199254740993, + "day": -86400000, + "ts": -1, + "maybe": None, + "lst": ["x"], + }, +] + +# to_columns: one array per column, the numpy dtype is part of the golden +GOLDEN_COLUMNS = { + "n": np.array([0, 1], dtype=np.int64), + "f": np.array([F32_TENTH, 0.5], dtype=np.float64), + "d": np.array([0.1, 1 / 3], dtype=np.float64), + "v3": np.array([[F32_TENTH, F32_THIRD, 0.5], [0.5, 1.0, -2.75]], dtype=np.float32), + "v9": np.array([V9_ROW0, V9_ROW1], dtype=np.float32), + "big": np.array([9007199254740993, -9007199254740993], dtype=np.int64), + "day": np.array(["2026-01-02", "1969-12-31"], dtype="datetime64[ms]"), + "ts": np.array( + ["2026-01-02T03:04:05.678", "1969-12-31T23:59:59.999"], dtype="datetime64[ms]" + ), + "maybe": np.array([0.25, np.nan], dtype=np.float64), +} + + +@pytest.fixture +def golden(temp_db): + db = temp_db + db.command("sql", "CREATE DOCUMENT TYPE Golden") + for name, type_name in [ + ("n", "INTEGER"), + ("f", "FLOAT"), + ("d", "DOUBLE"), + ("v3", "ARRAY_OF_FLOATS"), + ("v9", "ARRAY_OF_FLOATS"), + ("dec", "DECIMAL"), + ("big", "LONG"), + ("day", "DATE"), + ("ts", "DATETIME"), + ("maybe", "DOUBLE"), + ("lst", "LIST"), + ]: + db.command("sql", f"CREATE PROPERTY Golden.{name} {type_name}") + db.insert_many("Golden", [dict(row) for row in ROWS]) + return db + + +def _json_rows(db): + return db.query("sql", SELECT).to_json_list() + + +def test_to_json_list_floats_are_the_exact_widening_of_the_float32(golden): + """The float cases of #314: FLOAT, float32 float[] of 3 and of 9 values, next to a DOUBLE and the shortest-text trap.""" + rows = _json_rows(golden) + for have, want in zip(rows, GOLDEN_JSON): + for name in ("f", "d", "v3", "v9", "maybe"): + assert have[name] == want[name], (have["n"], name) + # the shortest text of float32(0.1) is "0.1", which Python reads as a different double + assert rows[0]["f"] != 0.1 + assert rows[0]["f"] == 0.10000000149011612 + + +def test_to_json_list_floats_equal_to_list_floats(golden): + """The two readers of the same rows agree: row["v9"] == to_list()[i]["v9"] (false on the JVM build before #314).""" + as_json = _json_rows(golden) + as_list = golden.query("sql", SELECT).to_list() + for from_json, from_list in zip(as_json, as_list): + for name in ("f", "d", "v3", "v9", "maybe"): + assert from_json[name] == from_list[name], (from_json["n"], name) + + +def test_to_json_list_golden(golden): + assert _json_rows(golden) == GOLDEN_JSON + rows = _json_rows(golden) + assert type(rows[0]["dec"]) is float + assert type(rows[0]["big"]) is int and type(rows[0]["day"]) is int + assert type(rows[0]["ts"]) is int + assert all(type(x) is float for row in rows for x in row["v3"] + row["v9"]) + + +def test_to_list_golden(golden): + rows = golden.query("sql", SELECT).to_list() + assert rows == GOLDEN_LIST + assert type(rows[0]["dec"]) is Decimal + assert type(rows[0]["big"]) is int + assert type(rows[0]["day"]) is date + assert type(rows[0]["ts"]) is datetime + assert all(type(x) is float for row in rows for x in row["v3"] + row["v9"]) + + +def _as_list(column): + return [list(x) if isinstance(x, (list, np.ndarray)) else x for x in column] + + +@pytest.mark.parametrize("name", sorted(GOLDEN_COLUMNS)) +def test_to_columns_golden(golden, name): + column = golden.query("sql", SELECT).to_columns()[name] + want = GOLDEN_COLUMNS[name] + assert isinstance(column, np.ndarray), type(column) + assert column.dtype == want.dtype, (column.dtype, want.dtype) + assert column.shape == want.shape + np.testing.assert_array_equal(column, want) + + +def test_to_columns_decimal_and_list_columns(golden): + columns = golden.query("sql", SELECT).to_columns() + assert list(columns["dec"]) == [Decimal("1.50"), Decimal("-0.25")] + assert all(type(v) is Decimal for v in columns["dec"]) + assert _as_list(columns["lst"]) == [[1, "a", 2.5], ["x"]] From 946dfda18fa713bb92d1171d6a5084cd98f5d4a8 Mon Sep 17 00:00:00 2001 From: Taewoon Kim Date: Mon, 9 Nov 2026 18:05:10 +0900 Subject: [PATCH 02/49] Native columnar readers: infer Arrow types first, skip per-key and per-row steps, one _kind per column (refs #312) (#321) * Native columnar readers: infer Arrow types first, skip per-key and per-row steps, one _kind per column (refs #312) Bindings-only, Python in arcadedb._native, no image rebuild. Output is meant to be identical to before: - columns.to_arrow tries pa.array(values) first and keeps the old per-value rules for a column that inference cannot type (ArrowInvalid, ArrowTypeError) or types differently (temporal types, unsigned ints). - columns._names tests each row's keys against the known ones in C instead of a Python step per key. - columns.to_columns calls _kind once per column instead of twice. - NativeResultSet._fill can hand a reply's rows straight to _drain_dicts, which no longer builds a tuple and a generator step per row. The float32-tail view in codec.loads_tagged is not done: a vector must stay a list of floats for the row API. tests/test_native_columns_reference.py keeps the old rules as a private reference and compares tables, columns, names and the drain on mixed, null-heavy, ragged and vector inputs. No speed is claimed here. Co-Authored-By: Claude Sonnet 5.5 * Native columns: a failed inference of any kind falls to the old rules; tests compare NaN tables by value (refs #312) pa.array([2**63]) raises OverflowError, not ArrowInvalid, so an int above int64 next to floats would have raised where the old explicit-type call fell back to exact text. _inferred now leaves every failed inference to the rules, so the outcome of such a column is decided by the old code. Four reference cases cover it. Table.equals is False for a NaN next to the same NaN, so the comparison is the schema plus the repr of the values. Co-Authored-By: Claude Sonnet 5.5 --------- Co-authored-by: Claude Sonnet 5.5 (cherry picked from commit 6c6a5a12cbca3ad2ae6ab7993023a8c4de39d2e7) Co-Authored-By: Claude Sonnet 5.5 --- src/arcadedb/_native/columns.py | 40 ++- src/arcadedb/_native/database.py | 32 +- tests/conftest.py | 2 + tests/test_native_columns_reference.py | 428 +++++++++++++++++++++++++ 4 files changed, 489 insertions(+), 13 deletions(-) create mode 100644 tests/test_native_columns_reference.py diff --git a/src/arcadedb/_native/columns.py b/src/arcadedb/_native/columns.py index 90f59df6..1d789922 100644 --- a/src/arcadedb/_native/columns.py +++ b/src/arcadedb/_native/columns.py @@ -5,12 +5,16 @@ def _names(rows, columns): + """The column names: ``columns``, or the keys of the rows in the order they are first seen.""" if columns is not None: return list(columns) seen = {} + known = seen.keys() for row in rows: - for k in row: - seen.setdefault(k, None) + # a row whose keys are all known (nearly every row) costs one C-level subset test, not one Python step per key + if not row.keys() <= known: + for k in row: + seen.setdefault(k, None) return list(seen) @@ -84,13 +88,13 @@ def to_columns(rows, columns=None): out = {} for name in _names(rows, columns): values = [row.get(name) for row in rows] - if _kind(values) == "obj": + kind = _kind(values) + if kind == "obj": vectors = _float_vectors(values) if vectors is not None: out[name] = vectors continue has_null = any(v is None for v in values) - kind = _kind(values) if kind == "i": out[name] = ( np.array([np.nan if v is None else v for v in values], dtype=np.float64) @@ -124,12 +128,40 @@ def to_columns(rows, columns=None): return out +def _inferred(values): + """``pa.array(values)`` when pyarrow's own inference gives the array the rules of :func:`to_arrow` would give, else None. + + Inference replaces the per-value Python pass of :func:`_kind` for a column of ints, floats, bools, strings, lists and + other values it types on its own. Two inferred types differ from the rules: a temporal type (the rules write + ``timestamp[ms]`` for a datetime and a date, inference gives ``timestamp[us]`` or ``date32``) and an unsigned integer + (an int above int64 is an error under the rules). A column that inference cannot type is also left to the rules, + whatever the error (``ArrowInvalid``, ``ArrowTypeError``, and ``OverflowError`` for an int above int64, which the + explicit-type call turns into ``ArrowInvalid``), so a failed inference never decides the outcome: the rules have the + fallbacks (exact text for an int beyond 2**53 next to floats, for a Decimal wider than decimal256, for a value of mixed + types). + """ + import pyarrow as pa + + try: + array = pa.array(values) + except Exception: # the rules decide the outcome of a column inference cannot type + return None + kind = array.type + if pa.types.is_temporal(kind) or pa.types.is_unsigned_integer(kind): + return None + return array + + def to_arrow(rows, columns=None): import pyarrow as pa arrays = {} for name in _names(rows, columns): values = [row.get(name) for row in rows] + inferred = _inferred(values) + if inferred is not None: + arrays[name] = inferred + continue kind = _kind(values) if kind == "i": arrays[name] = pa.array(values, type=pa.int64()) diff --git a/src/arcadedb/_native/database.py b/src/arcadedb/_native/database.py index f30dc526..924d5a17 100644 --- a/src/arcadedb/_native/database.py +++ b/src/arcadedb/_native/database.py @@ -248,8 +248,13 @@ def _readable(self) -> bool: "with block), so the rows it had not returned are gone. Run the query again to read them." ) - def _fill(self) -> bool: - """Fetch the next batch into the buffer; False once the cursor is drained.""" + def _fill(self, into: Optional[list] = None) -> bool: + """Fetch the next batch into the buffer; False once the cursor is drained. + + ``into``: a list that takes the rows (without their meta) of the batch when the buffer is empty, so a drain that + wants only the rows does not build one tuple per row to take them apart again. The buffer then stays empty, and the + answer is the same: whether the batch had a row (or an error is pending). + """ if self._pending_error is not None: err, self._pending_error = self._pending_error, None self._more = False @@ -279,10 +284,16 @@ def _fill(self) -> bool: raise ArcadeDBError( f"Reading the result set failed: {reply.get('cls')}: {reply.get('error')}" ) - self._buf.extend(zip(reply["rows"], reply["meta"])) + if into is not None and not self._buf: + batch = reply["rows"] + into.extend(batch) + has_rows = bool(batch) + else: + self._buf.extend(zip(reply["rows"], reply["meta"])) + has_rows = bool(self._buf) self._more = not reply["drained"] self._pending_error = reply.get("rerr") - return bool(self._buf) or self._pending_error is not None + return has_rows or self._pending_error is not None def _finish(self) -> None: self._exhausted = True @@ -334,11 +345,14 @@ def _drain_dicts(self) -> List[Dict[str, Any]]: out: List[Dict[str, Any]] = [] if not self._readable(): return out - while True: - out.extend(p for p, _ in self._buf) - self._buf.clear() - if not self._fill(): - break + buf = self._buf + if ( + buf + ): # rows read ahead (the first reply, or rows a caller has not taken): the only rows that carry a tuple + out.extend([p for p, _ in buf]) + buf.clear() + while self._fill(out): + pass self._finish() return out diff --git a/tests/conftest.py b/tests/conftest.py index ec3147c7..eaa3c695 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -83,6 +83,8 @@ collect_ignore.append("test_native_point_op_glue.py") # the address table of append_samples (arcadedb_n_async_bin), observed on the native library collect_ignore.append("test_native_timeseries_binary.py") + # arcadedb._native.columns builds the columnar readers' tables from the rows; the file compares it with the rules it replaced (#312) + collect_ignore.append("test_native_columns_reference.py") if not os.path.isfile( os.path.join(os.path.dirname(os.path.dirname(__file__)), "docs", "index.md") ): diff --git a/tests/test_native_columns_reference.py b/tests/test_native_columns_reference.py new file mode 100644 index 00000000..f1f68bad --- /dev/null +++ b/tests/test_native_columns_reference.py @@ -0,0 +1,428 @@ +"""The native columnar readers give the same table as the rules they replaced (refs #312). + +``arcadedb._native.columns`` infers an Arrow column with ``pa.array(values)`` first and falls back to the per-value +rules (``_kind``) when inference cannot type the column or types it differently; ``to_columns`` calls ``_kind`` once +per column; ``_names`` skips the per-key step for a row whose keys are known; ``NativeResultSet._drain_dicts`` takes +the rows of a reply without one tuple per row. None of that may change a value, a dtype or a column order. + +The reference functions below are the code as it was before: the rules of ``_kind``, ``_names``, ``to_columns`` and +``to_arrow``, copied. ``_ms`` and ``_float_vectors`` were not changed and are shared with the module. Each case builds +rows, runs both and compares the outcome, an exception included (a column that raised before raises the same class now). +Pure Python on the rows: no database is needed except for the drain test at the end. + +conftest.py collects this file on the native backend only (the module lives in the native package). +""" + +import random +from datetime import date, datetime, timedelta, timezone +from decimal import Decimal + +import numpy as np +import pyarrow as pa +import pytest + +from arcadedb._native import columns as new +from arcadedb._native.columns import _float_vectors, _ms + +# --- the reference: the rules before this change -------------------------------------------------------------------------- + + +def _names_ref(rows, columns): + if columns is not None: + return list(columns) + seen = {} + for row in rows: + for k in row: + seen.setdefault(k, None) + return list(seen) + + +def _kind_ref(values): + kinds = set() + for v in values: + if v is None: + continue + t = type(v) + if t is bool: + kinds.add("b") + elif t is int: + kinds.add("i") + elif t is float: + kinds.add("f") + elif t is datetime or t is date: + kinds.add("dt") + elif t is Decimal: + kinds.add("dec") + else: + kinds.add("obj") + if not kinds: + return "obj" + if len(kinds) == 1: + return kinds.pop() + if kinds == {"i", "f"}: + return "f" + return "obj" + + +def _to_columns_ref(rows, columns=None): + out = {} + for name in _names_ref(rows, columns): + values = [row.get(name) for row in rows] + if _kind_ref(values) == "obj": + vectors = _float_vectors(values) + if vectors is not None: + out[name] = vectors + continue + has_null = any(v is None for v in values) + kind = _kind_ref(values) + if kind == "i": + out[name] = ( + np.array([np.nan if v is None else v for v in values], dtype=np.float64) + if has_null + else np.array(values, dtype=np.int64) + ) + elif kind == "f": + out[name] = np.array( + [np.nan if v is None else v for v in values], dtype=np.float64 + ) + elif kind == "b": + out[name] = values if has_null else np.array(values, dtype=bool) + elif kind == "dt": + out[name] = np.array( + [ + ( + np.datetime64("NaT", "ms") + if v is None + else np.datetime64(_ms(v), "ms") + ) + for v in values + ], + dtype="datetime64[ms]", + ) + elif kind == "dec": + arr = np.empty(len(values), dtype=object) + arr[:] = values + out[name] = arr + else: + out[name] = values + return out + + +def _to_arrow_ref(rows, columns=None): + arrays = {} + for name in _names_ref(rows, columns): + values = [row.get(name) for row in rows] + kind = _kind_ref(values) + if kind == "i": + arrays[name] = pa.array(values, type=pa.int64()) + elif kind == "f": + try: + arrays[name] = pa.array(values, type=pa.float64()) + except pa.ArrowInvalid: + arrays[name] = pa.array( + [None if v is None else str(v) for v in values], type=pa.string() + ) + elif kind == "b": + arrays[name] = pa.array(values, type=pa.bool_()) + elif kind == "dt": + arrays[name] = pa.array( + [None if v is None else _ms(v) for v in values], type=pa.timestamp("ms") + ) + elif kind == "dec": + try: + arrays[name] = pa.array(values) + except pa.ArrowInvalid: + arrays[name] = pa.array( + [None if v is None else str(v) for v in values], type=pa.string() + ) + else: + try: + arrays[name] = pa.array([None if v is None else v for v in values]) + except (pa.ArrowInvalid, pa.ArrowTypeError): + arrays[name] = pa.array( + [None if v is None else str(v) for v in values], type=pa.string() + ) + return pa.table(arrays) + + +# --- the inputs ----------------------------------------------------------------------------------------------------------- + +BIG = 2**53 + 1 +STAMP = datetime(2026, 1, 2, 3, 4, 5, 678901) +AWARE = datetime(2026, 1, 2, 12, 0, 0, tzinfo=timezone(timedelta(hours=9))) +WIDE40 = Decimal("1234567890123456789012345678901234567890.5") # decimal256 +WIDE80 = Decimal("1" * 80) # wider than decimal256: exact text + +# one column per case, each a list of values that go in rows as {"c": value} +COLUMNS = { + "ints": [1, 2, 3], + "ints-null": [1, None, 3], + "ints-int64-edges": [-(2**63), 2**63 - 1, 0], + "int-above-int64": [2**63, 1], + "int-above-uint64": [2**64, 1], + "int-below-int64": [-(2**63) - 1, 1], + "floats": [0.5, 1.5, float("nan")], + "floats-null": [0.5, None, float("inf")], + "ints-and-floats": [1, 2.5, None, 3], + "int-beyond-2-53-and-floats": [BIG, 0.5], + "big-int-beyond-2-53-and-floats": [2**60, 1.5, None], + "int-above-int64-and-floats": [2**63, 0.5], + "int-above-uint64-and-floats": [2**64, 1.5, None], + "int-below-int64-and-floats": [-(2**63) - 1, 0.5], + "int-above-int64-and-string": [2**63, "x"], + "bools": [True, False, True], + "bools-null": [True, None, False], + "bool-and-int": [True, 1, 0], + "bool-and-float": [True, 0.5], + "strings": ["a", "bb", ""], + "strings-null": ["a", None, "c"], + "string-and-int": ["seven", 1], + "all-null": [None, None, None], + "single-null": [None], + "datetimes": [STAMP, datetime(1969, 12, 31, 23, 59, 59, 999000)], + "datetimes-null": [STAMP, None], + "dates": [date(2026, 1, 2), date(1969, 12, 31)], + "date-and-datetime": [date(2026, 1, 2), STAMP, None], + "aware-datetime": [AWARE, STAMP], + "datetime-and-string": [STAMP, "x"], + "decimals": [Decimal("1.50"), Decimal("-0.25")], + "decimals-null": [Decimal("2"), None, Decimal("30")], + "decimal-40-digits": [WIDE40, Decimal("1")], + "decimal-80-digits": [WIDE80, Decimal("1")], + "decimal-and-int": [Decimal("1.5"), 2], + "vectors": [[0.5, 1.5], [0.25, 2.0]], + "vectors-null": [[0.5, 1.5], None, [0.25, 2.0]], + "vector-not-float32": [[0.1, 0.2], [0.3, 0.4]], + "vector-ragged": [[0.5], [0.5, 1.5]], + "lists-of-ints": [[1, 2], [3]], + "lists-mixed": [[1, "a", 2.5], ["x"]], + "empty-lists": [[], []], + "dicts": [{"a": 1}, {"b": [1.5, None]}], + "nested-float-lists": [[[0.5, 1.5]], [[2.5]]], + "list-of-datetimes": [[STAMP], [None]], + "bytes-like": [b"ab", b"c"], +} + + +def _rows(values, name="c"): + return [{name: v} for v in values] + + +def _wide_null_heavy(seed=7, count=1000): + """Many rows, ragged keys, about nine values in ten missing or None, three vectors, strings, ints and floats.""" + rng = random.Random(seed) # nosec B311 - fixed test data, not a secret + rows = [] + for i in range(count): + row = {} + if rng.random() < 0.9: + row["i"] = rng.randrange(-1000, 1000) if rng.random() < 0.9 else None + if rng.random() < 0.1: + row["f"] = rng.random() * 10 + if rng.random() < 0.1: + row["s"] = f"s{i}" if rng.random() < 0.8 else None + if rng.random() < 0.1: + row["late-" + str(i % 3)] = i + if rng.random() < 0.1: + row["v"] = [float(np.float32(rng.random())) for _ in range(4)] + if rng.random() < 0.05: + row["m"] = rng.choice([1, 2.5, "x", None, True]) + rows.append(row) + return rows + + +# --- the comparison ------------------------------------------------------------------------------------------------------- + + +def _outcome(fn, *args): + try: + return ("ok", fn(*args)) + except ( + Exception + ) as exc: # noqa: BLE001 - the class of the failure is part of the result + return ("raises", type(exc)) + + +def _same_table(a, b): + assert a.schema == b.schema, (a.schema, b.schema) + assert a.column_names == b.column_names + # not Table.equals: it is False for a NaN next to the same NaN. repr counts NaN, -0.0 and the type of every value + assert repr(a.to_pydict()) == repr(b.to_pydict()) + + +def _same_column(a, b, name): + assert type(a) is type(b), (name, type(a), type(b)) + if isinstance(a, np.ndarray): + assert a.dtype == b.dtype and a.shape == b.shape, (name, a.dtype, b.dtype) + assert repr(a.tolist()) == repr(b.tolist()), name + else: + assert repr(a) == repr(b), name + + +def _check_arrow(rows, columns=None): + want = _outcome(_to_arrow_ref, rows, columns) + have = _outcome(new.to_arrow, rows, columns) + assert have[0] == want[0], (have, want) + if want[0] == "ok": + _same_table(have[1], want[1]) + else: + assert have[1] is want[1] + + +def _check_columns(rows, columns=None): + want = _outcome(_to_columns_ref, rows, columns) + have = _outcome(new.to_columns, rows, columns) + assert have[0] == want[0], (have, want) + if want[0] == "ok": + assert list(have[1]) == list(want[1]) + for name in want[1]: + _same_column(have[1][name], want[1][name], name) + else: + assert have[1] is want[1] + + +@pytest.mark.parametrize("case", sorted(COLUMNS)) +def test_a_single_column_gives_the_same_arrow_table(case): + _check_arrow(_rows(COLUMNS[case])) + + +@pytest.mark.parametrize("case", sorted(COLUMNS)) +def test_a_single_column_gives_the_same_columns(case): + _check_columns(_rows(COLUMNS[case])) + + +def test_a_table_of_all_the_cases_gives_the_same_arrow_table_and_columns(): + names = sorted(COLUMNS) + count = max(len(COLUMNS[n]) for n in names) + rows = [ + {n: COLUMNS[n][i] for n in names if i < len(COLUMNS[n])} for i in range(count) + ] + # a case that raises on its own raises for the table: compare the table without the columns that raise + ok = [n for n in names if _outcome(_to_arrow_ref, _rows(COLUMNS[n]))[0] == "ok"] + rows = [{n: r[n] for n in ok if n in r} for r in rows] + _check_arrow(rows) + ok_columns = [ + n for n in ok if _outcome(_to_columns_ref, _rows(COLUMNS[n]))[0] == "ok" + ] + _check_columns([{n: r[n] for n in ok_columns if n in r} for r in rows]) + + +@pytest.mark.parametrize("seed", [1, 7, 42]) +def test_a_null_heavy_ragged_table_gives_the_same_tables(seed): + rows = _wide_null_heavy(seed) + _check_arrow(rows) + _check_columns(rows) + + +def test_a_chosen_column_list_gives_the_same_tables(): + rows = _wide_null_heavy() + for columns in (["i", "s"], ["v", "nope", "i"], []): + _check_arrow(rows, columns) + _check_columns(rows, columns) + + +@pytest.mark.parametrize("columns", [None, ["a"]]) +def test_no_rows_gives_the_same_tables(columns): + _check_arrow([], columns) + _check_columns([], columns) + + +def test_rows_of_empty_dicts_give_the_same_tables(): + _check_arrow([{}, {}]) + _check_columns([{}, {}]) + + +def test_names_are_the_keys_in_the_order_they_are_first_seen(): + rows = [{"b": 1}, {"a": 1, "b": 2}, {"b": 3}, {"c": 1, "a": 2}, {"d": 1, "b": 1}] + assert new._names(rows, None) == _names_ref(rows, None) == ["b", "a", "c", "d"] + rows = _wide_null_heavy() + assert new._names(rows, None) == _names_ref(rows, None) + assert new._names(rows, ["z", "y"]) == ["z", "y"] + + +def test_the_inference_only_takes_a_column_the_rules_type_the_same(): + """A temporal column and an int above int64 are left to the rules; the rest is the inferred array.""" + assert new._inferred([STAMP, None]) is None + assert new._inferred([date(2026, 1, 2)]) is None + assert new._inferred([2**63]) is None + assert new._inferred([1, None, 3]).type == pa.int64() + assert new._inferred([0.5, 1]).type == pa.float64() + assert new._inferred([True, None]).type == pa.bool_() + assert new._inferred(["a", None]).type == pa.string() + assert new._inferred([None, None]).type == pa.null() + + +def test_to_columns_asks_for_the_kind_of_a_column_once(monkeypatch): + calls = [] + real = new._kind + + def counting(values): + calls.append(len(values)) + return real(values) + + monkeypatch.setattr(new, "_kind", counting) + new.to_columns(_rows([1, None, 3]), None) + assert calls == [3] + calls.clear() + new.to_columns(_rows([[0.5, 1.5], None]), None) + assert calls == [2] + + +# --- the drain ------------------------------------------------------------------------------------------------------------ + +DRAIN_ROWS = 5000 # more than the first reply (256) and two further replies (2048 each) + + +@pytest.fixture +def drain_db(temp_db): + temp_db.command("sql", "CREATE DOCUMENT TYPE Drain") + temp_db.command("sql", "CREATE PROPERTY Drain.id INTEGER") + temp_db.insert_many( + "Drain", + [{"id": i, "txt": f"t{i}", "val": i / 8.0} for i in range(DRAIN_ROWS)], + ) + return temp_db + + +SELECT_DRAIN = "SELECT id, txt, val FROM Drain ORDER BY id" + + +def test_the_drain_returns_the_rows_row_by_row_iteration_returns(drain_db): + by_rows = [r.to_dict() for r in drain_db.query("sql", SELECT_DRAIN)] + drained = drain_db.query("sql", SELECT_DRAIN)._drain_dicts() + assert len(drained) == DRAIN_ROWS + assert drained == by_rows + assert drained[0] == {"id": 0, "txt": "t0", "val": 0.0} + assert drained[-1] == { + "id": DRAIN_ROWS - 1, + "txt": f"t{DRAIN_ROWS - 1}", + "val": (DRAIN_ROWS - 1) / 8.0, + } + + +@pytest.mark.parametrize("taken", [1, 255, 256, 257, 2304, 4999]) +def test_the_drain_after_some_rows_were_taken_returns_the_rest(drain_db, taken): + rs = drain_db.query("sql", SELECT_DRAIN) + head = [next(rs).to_dict() for _ in range(taken)] + rest = rs._drain_dicts() + want = [r.to_dict() for r in drain_db.query("sql", SELECT_DRAIN)] + assert head + rest == want + + +def test_the_drain_of_an_empty_result_and_a_second_drain_are_empty(drain_db): + rs = drain_db.query("sql", "SELECT id FROM Drain WHERE id < 0") + assert rs._drain_dicts() == [] + rs = drain_db.query("sql", SELECT_DRAIN) + assert len(rs._drain_dicts()) == DRAIN_ROWS + assert rs._drain_dicts() == [] + + +def test_to_arrow_and_to_columns_of_a_result_equal_the_reference_over_its_rows( + drain_db, +): + rows = drain_db.query("sql", SELECT_DRAIN)._drain_dicts() + table = drain_db.query("sql", SELECT_DRAIN).to_arrow() + _same_table(table, _to_arrow_ref(rows)) + columns = drain_db.query("sql", SELECT_DRAIN).to_columns() + want = _to_columns_ref(rows) + for name in want: + _same_column(columns[name], want[name], name) From b0f89f9f6843561b81e2de9ab9a927afa9c1e2d6 Mon Sep 17 00:00:00 2001 From: Taewoon Kim Date: Mon, 9 Nov 2026 18:50:57 +0900 Subject: [PATCH 03/49] Native: read each library reply from its address, decode it in one copy, free it once (#323) The reply was copied as bytes (string_at), cut for the float tail, decoded to str, then parsed. It is now decoded straight from the library's memory with PyUnicode_FromStringAndSize, only the last 40 bytes and the float tail are copied as bytes, and the free is in the finally of the same call. Co-Authored-By: Claude Sonnet 5.5 --- src/arcadedb/_native/lib.py | 97 +++++++++------ tests/conftest.py | 2 + tests/test_native_reply_reader.py | 195 ++++++++++++++++++++++++++++++ 3 files changed, 254 insertions(+), 40 deletions(-) create mode 100644 tests/test_native_reply_reader.py diff --git a/src/arcadedb/_native/lib.py b/src/arcadedb/_native/lib.py index 58b261ba..4659df1a 100644 --- a/src/arcadedb/_native/lib.py +++ b/src/arcadedb/_native/lib.py @@ -327,16 +327,60 @@ def _thread_handle(): return t.handle +# One reply is read straight from its address: the text becomes a ``str`` in one copy (no ``bytes`` in between), and the library's +# memory is freed in the ``finally`` of the same call, once, whatever the decode does (a reply never outlives the call that took it, +# so no view of it can). ``PyUnicode_FromStringAndSize`` decodes UTF-8 as ``bytes.decode("utf-8")`` does, and raises the same +# UnicodeDecodeError; the function pointers are the ones of this module, ``ctypes.pythonapi`` itself is not changed. +_str_from_utf8 = ctypes.PYFUNCTYPE(ctypes.py_object, ctypes.c_void_p, ctypes.c_ssize_t)( + ("PyUnicode_FromStringAndSize", ctypes.pythonapi) +) +_strlen = ctypes.CFUNCTYPE(ctypes.c_size_t, ctypes.c_void_p)( + ( + "strlen", + ctypes.cdll.msvcrt if sys.platform == "win32" else ctypes.CDLL(None), + ) +) +_TAIL_KEY = b',"bin":' +_TAIL_WINDOW = 40 + + +def _take(lib, th, ptr, tail: bool): + """Read the reply at ``ptr`` as ``(text, floats)`` and free it: ``floats`` is None unless ``tail`` and the reply ends with + ``,"bin":N}`` (then N float32 follow the text's NUL, 4-byte aligned, and the text is returned without the ``,"bin":N`` part). + """ + if not ptr: + raise ArcadeDBError("The native library returned no reply") + try: + end = _strlen(ptr) + floats = None + if tail: + # only the last bytes can hold the marker: peek at them (at most 40 bytes), never at the whole reply + start = end - _TAIL_WINDOW if end > _TAIL_WINDOW else 0 + peek = ctypes.string_at(ptr + start, end - start) + if peek.endswith(b"}"): + i = peek.rfind(_TAIL_KEY) + if i >= 0: + n = int(peek[i + 7 : -1]) + import numpy as np + + off = (end + 1 + 3) & ~3 + floats = np.frombuffer( + ctypes.string_at(ptr + off, 4 * n), dtype=np.float32 + ) + # the text ends at the marker, closed by the "}" that the JSON object needs: written over the marker's comma + # in the reply's own memory (it is freed below), so the decode reads the JSON as it stands + ctypes.c_char.from_address(ptr + start + i).value = b"}" + end = start + i + 1 + return _str_from_utf8(ptr, end), floats + finally: + lib.arcadedb_free(th, ptr) + + def call(name: str, *args) -> dict: """Run one entry point; returns the decoded reply, raises ArcadeDBError-free (callers map errors).""" lib = _lib or _load() th = _thread_handle() - ptr = getattr(lib, name)(th, *args) - try: - text = ctypes.string_at(ptr).decode("utf-8") - finally: - lib.arcadedb_free(th, ptr) - return text + return _take(lib, th, getattr(lib, name)(th, *args), False)[0] def call_bin(name: str, *args): @@ -345,24 +389,7 @@ def call_bin(name: str, *args): """ lib = _lib or _load() th = _thread_handle() - ptr = getattr(lib, name)(th, *args) - try: - raw = ctypes.string_at(ptr) - floats = None - if raw.endswith(b"}"): - i = raw.rfind(b',"bin":', max(0, len(raw) - 40)) - if i >= 0: - n = int(raw[i + 7 : -1]) - import numpy as np - - off = (len(raw) + 1 + 3) & ~3 - floats = np.frombuffer( - ctypes.string_at(ptr + off, 4 * n), dtype=np.float32 - ) - raw = raw[:i] + b"}" - finally: - lib.arcadedb_free(th, ptr) - return raw.decode("utf-8"), floats + return _take(lib, th, getattr(lib, name)(th, *args), True) def alloc(size: int): @@ -383,22 +410,12 @@ def exec_query(handle, flags, language, command, params, count): lib = _lib or _load() t = _local.__dict__.get("thread") th = t.handle if t is not None else _thread_handle() - ptr = lib.arcadedb_n_exec(th, handle, flags, language, command, params, count) - try: - raw = ctypes.string_at(ptr) - floats = None - # a negative start counts from the end and stops at 0 for a short reply - i = raw.rfind(b',"bin":', -40) - if i >= 0 and raw.endswith(b"}"): - n = int(raw[i + 7 : -1]) - import numpy as np - - off = (len(raw) + 1 + 3) & ~3 - floats = np.frombuffer(ctypes.string_at(ptr + off, 4 * n), dtype=np.float32) - raw = raw[:i] + b"}" - finally: - lib.arcadedb_free(th, ptr) - return raw.decode("utf-8"), floats + return _take( + lib, + th, + lib.arcadedb_n_exec(th, handle, flags, language, command, params, count), + True, + ) def call_void(name: str, *args) -> None: diff --git a/tests/conftest.py b/tests/conftest.py index ec3147c7..2f0bfcce 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -65,6 +65,8 @@ collect_ignore.append("test_native_async_callbacks.py") # the short call path of a point read (codec.encode_params, lib.exec_query, first()), observed on the native library collect_ignore.append("test_native_point_read.py") + # how a reply is read out of the library (lib._take): the copies and the frees, observed with spies on ctypes.string_at and arcadedb_free + collect_ignore.append("test_native_reply_reader.py") # an RID built from an RID keeps the text, so the fast named-parameter path of codec.encode_params can encode it (#265) collect_ignore.append("test_native_rid_params.py") # to_json_list walks the rows with codec.json_native only when a reply restored a tagged value (#219), observed on the native library diff --git a/tests/test_native_reply_reader.py b/tests/test_native_reply_reader.py new file mode 100644 index 00000000..0b789097 --- /dev/null +++ b/tests/test_native_reply_reader.py @@ -0,0 +1,195 @@ +"""How the native build reads a reply out of the library (``lib._take``; ``call``, ``call_bin`` and ``exec_query`` all end there). + +A reply is a malloc'd UTF-8 text, optionally followed by a float32 tail, and it must be freed exactly once. It used to be copied as +``bytes`` (``ctypes.string_at``), cut for the tail marker, decoded to ``str``, and only then parsed. It is now decoded straight from its +address in one copy, and only the last 40 bytes (the marker) and the float tail are ever copied as ``bytes``. + +The oracle is a private copy of the former reader (``_former``): every case is read by both, from two identical replies, and the text, +the floats and the number of frees must agree. The copy counts are read from a spy on ``ctypes.string_at``: the former reader called it +once with the whole reply, the new one never does for a text and asks only for the marker window and the tail. The free counts are read +from a spy on ``arcadedb_free``: once, after the use, also when the decode or the tail fails. conftest.py collects this file on the +native backend only: it needs the native library. +""" + +import ctypes +import json + +import numpy as np +import pytest + +from arcadedb._native import lib +from arcadedb.exceptions import ArcadeDBError + +_string_at = ctypes.string_at + + +def _reply(raw: bytes, floats=None): + """A reply in the library's own memory, as the library hands one out: the text, its NUL, then (4-byte aligned) the floats.""" + tail = b"" + if floats is not None: + raw = raw[:-1] + b',"bin":%d}' % len(floats) + pad = -(len(raw) + 1) % 4 + tail = b"\0" * pad + np.asarray(floats, dtype=np.float32).tobytes() + data = raw + b"\0" + tail + ptr = lib.alloc(len(data)) + ctypes.memmove(ptr, data, len(data)) + return ptr, len(raw) + + +def _former(th, ptr): + """The reader this file replaces (``call_bin`` before the change), for the oracle.""" + try: + raw = _string_at(ptr) + floats = None + if raw.endswith(b"}"): + i = raw.rfind(b',"bin":', max(0, len(raw) - 40)) + if i >= 0: + n = int(raw[i + 7 : -1]) + off = (len(raw) + 1 + 3) & ~3 + floats = np.frombuffer(_string_at(ptr + off, 4 * n), dtype=np.float32) + raw = raw[:i] + b"}" + finally: + lib._lib.arcadedb_free(th, ptr) + return raw.decode("utf-8"), floats + + +@pytest.fixture +def frees(temp_db, monkeypatch): + """Every pointer given to ``arcadedb_free``, in order.""" + seen = [] + real = lib._lib.arcadedb_free + + def spy(th, ptr): + seen.append(ptr) + return real(th, ptr) + + monkeypatch.setattr(lib._lib, "arcadedb_free", spy) + return seen + + +@pytest.fixture +def string_at_sizes(monkeypatch): + """The size asked of each ``ctypes.string_at`` call (None: the whole NUL-terminated string).""" + seen = [] + + def spy(ptr, size=-1): + seen.append(None if size == -1 else size) + return _string_at(ptr, size) + + monkeypatch.setattr(ctypes, "string_at", spy) + return seen + + +LONG = json.dumps({"ok": True, "rows": [{"t": "x" * 5000}], "meta": []}).encode() + +CASES = { + "empty object": (b"{}", None), + "ascii": (b'{"ok":true,"rows":[{"a":1}],"drained":true}', None), + "non-ascii": ('{"ok":true,"t":"café 日本語 \U0001f600"}'.encode(), None), + "long text": (LONG, None), + "tail": ( + b'{"ok":true,"rows":[{"v":["B",0,4]}],"meta":[],"drained":true}', + [1, 2, 3, 4], + ), + "tail, short text": (b'{"v":1}', [0.5, 1.5]), + "tail, non-ascii text": ( + '{"ok":true,"rows":[{"t":"日本語 café \U0001f600","v":["B",0,3]}],"drained":true}'.encode(), + [1.5, -2.5, 3.25], + ), + "tail, long text": (LONG, np.arange(16, dtype=np.float32)), + "tail of no floats": (b'{"ok":true}', []), + **{ + f"tail, text of {k} digits": (b'{"a":' + b"1" * k + b"}", [7.0, 8.0]) + for k in range(1, 5) # the four paddings that align the floats to 4 bytes + }, + "marker deep in the text": ( + b'{"a":"' + b"x" * 100 + b'","b":1,"bin":2,"c":' + b"y" * 60 + b"}", + None, + ), +} + + +@pytest.mark.parametrize("name", list(CASES)) +@pytest.mark.parametrize("tail", [True, False]) +def test_every_case_reads_as_the_former_reader_does(temp_db, frees, name, tail): + raw, floats = CASES[name] + th = lib._thread_handle() + a, _ = _reply(raw, floats) + b, _ = _reply(raw, floats) + want_text, want_floats = _former(th, a) + if not tail: # call(): the text as it stands, tail marker included + want_text = _string_at(b).decode("utf-8") + del frees[:] + got_text, got_floats = lib._take(lib._lib, th, b, tail) + assert got_text == want_text + assert frees == [b] # once, and the reply that was read + if tail and floats is not None: + assert got_floats.dtype == np.float32 + assert np.array_equal(got_floats, want_floats) + assert ( + json.loads(got_text) is not None + ) # the text is a whole JSON value without the marker + else: + assert got_floats is None + + +def test_a_text_reply_is_not_copied_as_bytes(frees, string_at_sizes): + text = lib.call("arcadedb_n_diag") + assert json.loads(text) + assert ( + string_at_sizes == [] + ) # the former reader called string_at once, for the whole reply + assert len(frees) == 1 + + +def test_a_reply_with_a_tail_copies_only_the_marker_window_and_the_floats( + temp_db, frees, string_at_sizes +): + ptr, _ = _reply(LONG, np.arange(16, dtype=np.float32)) + th = lib._thread_handle() + text, floats = lib._take(lib._lib, th, ptr, True) + assert len(text) > 5000 and floats.size == 16 + assert sorted(string_at_sizes) == [ + 40, + 64, + ] # the last 40 bytes, 16 floats: never the 5 kB of text + assert frees == [ptr] + + +def test_a_reply_that_fails_to_decode_is_still_freed_once(temp_db, frees): + th = lib._thread_handle() + for raw in (b'{"a":"\xff"}', b'{"a":"\xc3"}', b'{"a":"\xed\xa0\x80"}'): + ptr, _ = _reply(raw) + del frees[:] + with pytest.raises(UnicodeDecodeError): + lib._take(lib._lib, th, ptr, False) + assert frees == [ptr] + ptr, _ = _reply(raw) + del frees[:] + with pytest.raises(UnicodeDecodeError): + lib._take(lib._lib, th, ptr, True) + assert frees == [ptr] + + +def test_a_reply_whose_tail_fails_is_still_freed_once(temp_db, frees): + th = lib._thread_handle() + ptr, _ = _reply(b'{"ok":true,"bin":x}') + with pytest.raises(ValueError): + lib._take(lib._lib, th, ptr, True) + assert frees == [ptr] + + +def test_no_reply_raises_and_frees_nothing(temp_db, frees): + with pytest.raises(ArcadeDBError): + lib._take(lib._lib, lib._thread_handle(), None, False) + assert frees == [] + + +def test_the_public_readers_free_each_reply_once(temp_db, frees): + temp_db.command("sql", "CREATE DOCUMENT TYPE R") + del frees[:] + text, floats = lib.call_bin("arcadedb_n_diag") + assert json.loads(text) and floats is None + assert len(frees) == 1 + lib.call("arcadedb_n_diag") + assert len(frees) == 2 From cb678b553f749c467cd82e2e8c3c35aa01fff499 Mon Sep 17 00:00:00 2001 From: Taewoon Kim Date: Mon, 9 Nov 2026 18:53:19 +0900 Subject: [PATCH 04/49] Native insert_columns(parallel=True): take the columnar entry (closes #325) A parallel call skipped the columnar fast path and went through one dict per row and one JSON text per chunk of 10,000 rows. The columnar entry arcadedb_n_insert_colp now takes "parallel": true in its header: the image builds the records from the column buffers and hands them to the async writers (db.async().createRecord with a failure token), as DocumentBatcher.insertColumnsParallel does on the JVM wheel. The Python side waits for the writers and reads the failures exactly as insert_many(parallel=True) does (_finish_parallel, shared). Rows the buffers cannot carry still take the row path. The native library is rebuilt from the unchanged engine commit. Tests (native only): a spy proves one colp crossing and no insert_parallel chunk; the stored records equal those of insert_many(parallel=True); nulls, vertex types, rejected records, an edge type, a missing type, the fall back and the off switch. Co-Authored-By: Claude Sonnet 5.5 --- .../org/humemai/nativelib/ArcadeNative.java | 5 +- .../java/org/humemai/nativelib/BulkOps.java | 117 ++++++---- .../org/humemai/nativelib/NativeExtras.java | 37 +++- src/arcadedb/_native/bulk.py | 27 ++- src/arcadedb/_native/columnar.py | 21 +- src/arcadedb/_native/database.py | 24 +- tests/conftest.py | 2 + tests/test_native_columnar_pack.py | 5 + tests/test_native_columnar_parallel.py | 208 ++++++++++++++++++ 9 files changed, 378 insertions(+), 68 deletions(-) create mode 100644 tests/test_native_columnar_parallel.py diff --git a/native/src/main/java/org/humemai/nativelib/ArcadeNative.java b/native/src/main/java/org/humemai/nativelib/ArcadeNative.java index 9b5acb91..1703b965 100644 --- a/native/src/main/java/org/humemai/nativelib/ArcadeNative.java +++ b/native/src/main/java/org/humemai/nativelib/ArcadeNative.java @@ -561,7 +561,8 @@ public static CCharPointer insertCols(final IsolateThread thread, final long h, * Columnar insert by pointer: no copy of the columns into one buffer. spec points at 6 longs per column (data address and bytes, aux * address and bytes, null-bitmap address and bytes, see BulkOps.insertColumns) and nspec is their count. Kinds q d b f as * arcadedb_n_insert_cols, plus i (int32), e (float32) and s (Arrow UTF-8 string: int32 offsets and the bytes they index), and null bitmaps. - * The caller keeps every buffer alive and unchanged until the call returns. Returns {"ok":true,"n":rows}. + * The caller keeps every buffer alive and unchanged until the call returns. Returns {"ok":true,"n":rows}; with "parallel":true in the + * header the records go to the async executor and the reply also carries "fid" (see BulkOps.insertColumnsReply). */ @CEntryPoint(name = "arcadedb_n_insert_colp") public static CCharPointer insertColp(final IsolateThread thread, final long h, final CCharPointer type, final CCharPointer header, @@ -572,7 +573,7 @@ public static CCharPointer insertColp(final IsolateThread thread, final long h, final long[] table = new long[nspec]; for (int i = 0; i < nspec; i++) table[i] = ((Pointer) spec).readLong(WordFactory.signed(8L * i)); - return out("{\"ok\":true,\"n\":" + BulkOps.insertColumns(db(h), str(type), str(header), table) + "}"); + return out(BulkOps.insertColumnsReply(db(h), str(type), str(header), table)); } catch (final Throwable t) { return err(t); } diff --git a/native/src/main/java/org/humemai/nativelib/BulkOps.java b/native/src/main/java/org/humemai/nativelib/BulkOps.java index 37cedf90..40f789e0 100644 --- a/native/src/main/java/org/humemai/nativelib/BulkOps.java +++ b/native/src/main/java/org/humemai/nativelib/BulkOps.java @@ -10,6 +10,9 @@ import com.arcadedb.database.Database; import com.arcadedb.database.MutableDocument; +import com.arcadedb.database.async.DatabaseAsyncExecutor; +import com.arcadedb.database.async.ErrorCallback; +import com.arcadedb.schema.EdgeType; import com.arcadedb.schema.VertexType; import org.graalvm.nativeimage.c.type.CTypeConversion; @@ -55,13 +58,27 @@ static long insertColumns(final Database db, final String type, final String hea spec[SLOTS * c + 1] = bytes; pos += bytes; } - return insertColumns(db, type, h, spec); + return insertColumns(db, type, h, spec, null); } - /** The pointer form (arcadedb_n_insert_colp): the table holds SLOTS longs per column, see {@link #insertColumns(Database, String, Map, long[])}. */ + /** + * The pointer form (arcadedb_n_insert_colp): the table holds SLOTS longs per column, see {@link #insertColumns(Database, String, Map, long[], ErrorCallback)}. + * Returns the reply text: {"ok":true,"n":rows}, and for a header with "parallel":true also "fid", the token that the "failures" operation of + * arcadedb_n_async reads (see there). In that mode the records are handed to the database's async executor, the call does not wait for + * the writers, and it neither opens, commits nor rolls back a transaction ("commit" does not apply), exactly as insert_parallel does for rows. + */ @SuppressWarnings("unchecked") - static long insertColumns(final Database db, final String type, final String header, final long[] spec) { - return insertColumns(db, type, (Map) Json.parse(header), spec); + static String insertColumnsReply(final Database db, final String type, final String header, final long[] spec) { + final Map h = (Map) Json.parse(header); + if (!Boolean.TRUE.equals(h.get("parallel"))) + return "{\"ok\":true,\"n\":" + insertColumns(db, type, h, spec, null) + "}"; + final NativeExtras.FailureToken token = new NativeExtras.FailureToken(); + try { + return "{\"ok\":true,\"n\":" + insertColumns(db, type, h, spec, token.onError) + ",\"fid\":" + token.fid + "}"; + } catch (final Throwable t) { + token.drop(); // the caller gets the error, not a token to read failures from + throw t; + } } /** @@ -72,9 +89,13 @@ static long insertColumns(final Database db, final String type, final String hea * means no nulls. The memory belongs to the caller for the duration of the call; nothing is kept. Every length is checked here before * the first document is built, so a wrong table is an exception and nothing is written. Same transaction contract as * DocumentBatcher.insertColumns: batch commits and the rollback on failure apply only to a transaction opened here. + * onError null: the synchronous mode above. Not null (header "parallel":true): every record goes to db.async().createRecord with that + * callback and no transaction is touched, as DocumentBatcher.insertColumnsParallel does; an edge type is refused there, as it is by + * insert_parallel. The columns are validated before the first record is handed over in both modes. */ @SuppressWarnings("unchecked") - private static long insertColumns(final Database db, final String type, final Map h, final long[] spec) { + private static long insertColumns(final Database db, final String type, final Map h, final long[] spec, + final ErrorCallback onError) { final long nRows = ((Number) h.get("n")).longValue(); if (nRows < 0 || nRows > Integer.MAX_VALUE) throw new IllegalArgumentException("row count out of range: " + nRows); @@ -137,45 +158,27 @@ private static long insertColumns(final Database db, final String type, final Ma } } final byte[] scratch = new byte[longest]; - final boolean vertex = db.getSchema().getType(type) instanceof VertexType; + final Object schemaType = db.getSchema().getType(type); + final boolean vertex = schemaType instanceof VertexType; + if (onError != null) { + if (schemaType instanceof EdgeType) + throw new IllegalArgumentException("Cannot bulk-insert into '" + type + + "' because it is an edge type; load the vertices, then create the edges (graph_batch().new_edges() or CREATE EDGE)"); + final DatabaseAsyncExecutor async = db.async(); + for (int i = 0; i < n; i++) { + final MutableDocument doc = vertex ? db.newVertex(type) : db.newDocument(type); + setRow(doc, i, names, kinds, dims, data, nulls, blobs, scratch); + async.createRecord(doc, null, onError); + } + return n; + } final boolean wasActive = db.isTransactionActive(); if (!wasActive) db.begin(); try { for (int i = 0; i < n; i++) { final MutableDocument doc = vertex ? db.newVertex(type) : db.newDocument(type); - for (int c = 0; c < k; c++) { - final Pointer base = WordFactory.pointer(data[c]); - if (nulls[c] != 0 && ((WordFactory.pointer(nulls[c]).readByte(WordFactory.signed(i >> 3)) >> (i & 7)) & 1) == 0) { - doc.set(names[c], null); - continue; - } - switch (kinds[c]) { - case 'q' -> doc.set(names[c], base.readLong(WordFactory.signed(8L * i))); - case 'd' -> doc.set(names[c], base.readDouble(WordFactory.signed(8L * i))); - case 'b' -> doc.set(names[c], base.readByte(WordFactory.signed(i)) != 0); - case 'i' -> doc.set(names[c], base.readInt(WordFactory.signed(4L * i))); - case 'e' -> doc.set(names[c], base.readFloat(WordFactory.signed(4L * i))); - case 's' -> { - final int from = base.readInt(WordFactory.signed(4L * i)); - final int len = base.readInt(WordFactory.signed(4L * (i + 1L))) - from; - if (len == 0) - doc.set(names[c], ""); - else { - blobs[c].get(from, scratch, 0, len); - doc.set(names[c], new String(scratch, 0, len, StandardCharsets.UTF_8)); - } - } - default -> { - final int dim = dims[c]; - final float[] v = new float[dim]; - final long at = 4L * i * dim; - for (int j = 0; j < dim; j++) - v[j] = base.readFloat(WordFactory.signed(at + 4L * j)); - doc.set(names[c], v); - } - } - } + setRow(doc, i, names, kinds, dims, data, nulls, blobs, scratch); doc.save(); if (!wasActive && commitEvery > 0 && (i + 1) % commitEvery == 0) { db.commit(); @@ -197,6 +200,44 @@ private static long insertColumns(final Database db, final String type, final Ma return n; } + /** Sets the properties of row i on doc from the columns: the one place that reads a value out of a buffer. */ + private static void setRow(final MutableDocument doc, final int i, final String[] names, final char[] kinds, final int[] dims, + final long[] data, final long[] nulls, final ByteBuffer[] blobs, final byte[] scratch) { + final int k = names.length; + for (int c = 0; c < k; c++) { + final Pointer base = WordFactory.pointer(data[c]); + if (nulls[c] != 0 && ((WordFactory.pointer(nulls[c]).readByte(WordFactory.signed(i >> 3)) >> (i & 7)) & 1) == 0) { + doc.set(names[c], null); + continue; + } + switch (kinds[c]) { + case 'q' -> doc.set(names[c], base.readLong(WordFactory.signed(8L * i))); + case 'd' -> doc.set(names[c], base.readDouble(WordFactory.signed(8L * i))); + case 'b' -> doc.set(names[c], base.readByte(WordFactory.signed(i)) != 0); + case 'i' -> doc.set(names[c], base.readInt(WordFactory.signed(4L * i))); + case 'e' -> doc.set(names[c], base.readFloat(WordFactory.signed(4L * i))); + case 's' -> { + final int from = base.readInt(WordFactory.signed(4L * i)); + final int len = base.readInt(WordFactory.signed(4L * (i + 1L))) - from; + if (len == 0) + doc.set(names[c], ""); + else { + blobs[c].get(from, scratch, 0, len); + doc.set(names[c], new String(scratch, 0, len, StandardCharsets.UTF_8)); + } + } + default -> { + final int dim = dims[c]; + final float[] v = new float[dim]; + final long at = 4L * i * dim; + for (int j = 0; j < dim; j++) + v[j] = base.readFloat(WordFactory.signed(at + 4L * j)); + doc.set(names[c], v); + } + } + } + } + /** * Documents (or vertices of a vertex type) from rows [{property: value},..] in the UTF-8 JSON text b[0, n): one record per row through * newDocument/newVertex, set per property and save, as DocumentBatcher.insertManyJson does on the JPype path; no transaction is opened diff --git a/native/src/main/java/org/humemai/nativelib/NativeExtras.java b/native/src/main/java/org/humemai/nativelib/NativeExtras.java index a62574ad..c0e87e87 100644 --- a/native/src/main/java/org/humemai/nativelib/NativeExtras.java +++ b/native/src/main/java/org/humemai/nativelib/NativeExtras.java @@ -527,6 +527,28 @@ else if (opName.equals("create_vertices")) private static final Map ASYNC_FAILS = new ConcurrentHashMap<>(); private static final Map> ASYNC_FIRST = new ConcurrentHashMap<>(); + /** A failure token for one parallel load: the id the caller reads with the "failures" operation, and the callback that counts. */ + static final class FailureToken { + final long fid = ArcadeNative.IDS.getAndIncrement(); + final AtomicLong count = new AtomicLong(); + final AtomicReference first = new AtomicReference<>(); + final ErrorCallback onError = t -> { + count.incrementAndGet(); + first.compareAndSet(null, t); + }; + + FailureToken() { + ASYNC_FAILS.put(fid, count); + ASYNC_FIRST.put(fid, first); + } + + /** The load failed before it returned: the caller gets the error, not a token to read failures from. */ + void drop() { + ASYNC_FAILS.remove(fid); + ASYNC_FIRST.remove(fid); + } + } + private static String replyRows(final ResultSet rs) { final StringBuilder sb = new StringBuilder("{\"ok\":true,\"rs\":0,"); NativeCodec.appendBatch(sb, rs, Integer.MAX_VALUE); @@ -824,15 +846,9 @@ else if (a.get("slot") instanceof Number slot) ex.appendSamples((String) a.get("type"), stamps, columns); } case "insert_parallel" -> { - final long fid = ArcadeNative.IDS.getAndIncrement(); - final AtomicLong count = new AtomicLong(); - final AtomicReference first = new AtomicReference<>(); - ASYNC_FAILS.put(fid, count); - ASYNC_FIRST.put(fid, first); - final ErrorCallback onError = t -> { - count.incrementAndGet(); - first.compareAndSet(null, t); - }; + final FailureToken token = new FailureToken(); + final long fid = token.fid; + final ErrorCallback onError = token.onError; final String type = (String) a.get("type"); try { final List rows = (List) a.get("rows"); @@ -846,8 +862,7 @@ else if (a.get("slot") instanceof Number slot) ex.createRecord(doc, null, onError); } } catch (final Throwable t) { - ASYNC_FAILS.remove(fid); // the caller gets the error, not a token to read failures from - ASYNC_FIRST.remove(fid); + token.drop(); // the caller gets the error, not a token to read failures from throw t; } return ArcadeNative.out("{\"ok\":true,\"fid\":" + fid + "}"); diff --git a/src/arcadedb/_native/bulk.py b/src/arcadedb/_native/bulk.py index d244c48c..dcaca257 100644 --- a/src/arcadedb/_native/bulk.py +++ b/src/arcadedb/_native/bulk.py @@ -2,7 +2,8 @@ ints, floats and bools, lists of one value type (None is a null), strings (UTF-8 bytes plus offsets, as Arrow lays them out), float32 vector columns and Arrow arrays are laid out by :mod:`.columnar`; anything else (nested values, mixed-type columns, date-times, decimals, integers past 64 bits) returns None and the caller uses the per-row JSON path. The documents are built image-side as -``DocumentBatcher.insertColumns`` builds them on JPype. ``ARCADEDB_NATIVE_BULK=0`` turns the columnar path off. +``DocumentBatcher.insertColumns`` builds them on JPype; with ``parallel`` they are handed to the async writers instead of a transaction +(``DocumentBatcher.insertColumnsParallel``). ``ARCADEDB_NATIVE_BULK=0`` turns the columnar path off. """ import os @@ -21,20 +22,19 @@ def _usable(): ) -def _send(db, type_name, packed, rows, commit_every, prefix): +def _send(db, type_name, packed, rows, commit_every, prefix, parallel=False): from .database import _call spec = columnar.spec_of(packed) - reply = _call( + return _call( prefix, "arcadedb_n_insert_colp", db._h, lib.b(type_name), - lib.b(columnar.header_of(rows, commit_every, packed)), + lib.b(columnar.header_of(rows, commit_every, packed, parallel)), spec, len(spec), ) # `packed` holds the arrays the addresses point into until the call returns - return int(reply["n"]) def insert_columns_fast( @@ -51,7 +51,22 @@ def insert_columns_fast( if packed is None: return None rows, cols = packed - return _send(db, type_name, cols, rows, commit_every, prefix) + return int(_send(db, type_name, cols, rows, commit_every, prefix)["n"]) + + +def insert_columns_parallel(db, type_name, columns, declared=None): + """Hand ``columns`` (name -> column) to the async writers in one crossing: ``(rows, failure token)``, or None when a column cannot be + sent this way. The call does not wait for the writers: the caller waits for them and reads the failures for the token + (``Database._finish_parallel``), as for ``insert_many(parallel=True)``. + """ + if not _usable(): + return None + packed = columnar.pack(columns, columnar.Declared(db, type_name, declared)) + if packed is None: + return None + rows, cols = packed + reply = _send(db, type_name, cols, rows, 0, "Failed to bulk-insert", parallel=True) + return int(reply["n"]), reply["fid"] def insert_rows_columnar(db, type_name, chunk): diff --git a/src/arcadedb/_native/columnar.py b/src/arcadedb/_native/columnar.py index f288043c..fe62837a 100644 --- a/src/arcadedb/_native/columnar.py +++ b/src/arcadedb/_native/columnar.py @@ -343,17 +343,20 @@ def spec_of(packed): return (ctypes.c_int64 * len(flat))(*flat) -def header_of(rows, commit_every, packed): - """The JSON header text: row count, commit interval and ``[name, kind, dim]`` per column.""" +def header_of(rows, commit_every, packed, parallel=False): + """The JSON header text: row count, commit interval and ``[name, kind, dim]`` per column; ``"parallel": true`` hands the records to the + async executor (the image then ignores the commit interval and the reply carries the failure token). + """ import json - return json.dumps( - { - "n": rows, - "commit": min(int(commit_every), 2**31 - 1), - "cols": [[c.name, c.kind, c.dim] for c in packed], - } - ) + header = { + "n": rows, + "commit": min(int(commit_every), 2**31 - 1), + "cols": [[c.name, c.kind, c.dim] for c in packed], + } + if parallel: + header["parallel"] = True + return json.dumps(header) def rows_to_columns(rows): diff --git a/src/arcadedb/_native/database.py b/src/arcadedb/_native/database.py index f30dc526..b0c07932 100644 --- a/src/arcadedb/_native/database.py +++ b/src/arcadedb/_native/database.py @@ -1040,6 +1040,12 @@ def _insert_many_parallel(self, type_name, rows) -> int: except Exception: # nosec B110 - the original error is the one to report pass raise + return self._finish_parallel(type_name, tokens, n) + + def _finish_parallel(self, type_name, tokens, n) -> int: + """The end of a parallel load: wait for the async writers, then read the failure count of every token and raise when a record failed.""" + from . import extras + try: self._async().wait_completion() except Exception as e: @@ -1314,7 +1320,11 @@ def export_to_csv( def insert_columns( self, type_name, columns, commit_every: int = 10_000, parallel: bool = False ) -> int: - """Bulk insert from whole columns (dict of name -> numpy array or list), as rows through :meth:`insert_many`.""" + """Bulk insert from whole columns (dict of name -> numpy array or list): columns the buffers can carry cross in one call + (``arcadedb_n_insert_colp``), any other column set goes as rows through :meth:`insert_many`. With ``parallel=True`` the + same one call hands the records to the async writers (no transaction, ``commit_every`` does not apply, the call waits for the + writers and raises when a record failed), as ``insert_many(parallel=True)`` does for rows. + """ self._check_not_closed() items = list(columns.items()) if hasattr(columns, "items") else None if not items: @@ -1330,7 +1340,17 @@ def insert_columns( known = {n: len(v) for n, v in items if hasattr(v, "__len__")} if len(set(known.values())) > 1: raise ValueError(f"columns differ in length: {known}") - if not parallel: + if parallel: + from .bulk import insert_columns_parallel + + try: + sent = insert_columns_parallel(self, type_name, columns) + except ArcadeDBError as e: + # as insert_many(parallel=True): rows handed to the writers before the failure may be stored + raise self._parallel_load_failed(type_name, e) from e + if sent is not None: + return self._finish_parallel(type_name, [sent[1]], sent[0]) + else: from .bulk import insert_columns_fast done = insert_columns_fast(self, type_name, columns, commit_every) diff --git a/tests/conftest.py b/tests/conftest.py index ec3147c7..be5fa583 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -47,6 +47,8 @@ collect_ignore.append("test_native_vector_hit_lazy.py") # the columnar insert's stored records are compared between the JSON and the columnar entry points of the native library collect_ignore.append("test_native_columnar_insert.py") + # insert_columns(parallel=True) takes the columnar entry and stores what the row path stores (arcadedb_n_insert_colp with "parallel") + collect_ignore.append("test_native_columnar_parallel.py") # the rows of an insert_many chunk with date-times, Decimals, bytes and numpy scalars are told apart by the entry points they cross collect_ignore.append("test_native_mixed_chunk.py") # the native build's ingest defaults (insert_many and graph_batch commit size), observed on the native library diff --git a/tests/test_native_columnar_pack.py b/tests/test_native_columnar_pack.py index cda0a1f7..56b283e1 100644 --- a/tests/test_native_columnar_pack.py +++ b/tests/test_native_columnar_pack.py @@ -233,6 +233,11 @@ def test_the_address_table_and_header_describe_the_columns(): "cols": [["id", "q", 0], ["s", "s", 0]], } assert json.loads(columnar.header_of(rows, 2**40, packed))["commit"] == 2**31 - 1 + # "parallel" is in the header only when asked for (the image hands the records to the async writers then) + assert json.loads(columnar.header_of(rows, 10_000, packed, parallel=True))[ + "parallel" + ] + assert "parallel" not in json.loads(columnar.header_of(rows, 10_000, packed)) def test_rows_become_columns_only_when_every_row_has_the_same_keys(): diff --git a/tests/test_native_columnar_parallel.py b/tests/test_native_columnar_parallel.py new file mode 100644 index 00000000..f51fd7c7 --- /dev/null +++ b/tests/test_native_columnar_parallel.py @@ -0,0 +1,208 @@ +"""Native ``insert_columns(parallel=True)`` takes the columnar entry and stores what the row path of ``insert_many(parallel=True)`` stores. + +Before, a parallel call skipped the columnar fast path: the columns became one dict per row, then one JSON text per chunk of 10,000 +rows for ``arcadedb_n_async insert_parallel``. Now the same ``arcadedb_n_insert_colp`` crossing that the synchronous call uses carries +``"parallel": true`` and the image hands the records to the async writers, as ``DocumentBatcher.insertColumnsParallel`` does on the +JVM wheel. Needs the native library, so conftest.py collects this file on the native backend only. Each test loads the same data +through the row path (the private reference: ``insert_many(parallel=True)`` with dict rows) and through the columns, compares the +stored records value by value and type by type, and spies on the entry points so a silent fall back cannot pass. +""" + +import numpy as np +import pytest + +from arcadedb._native import bulk, extras, lib # noqa: E402 +from arcadedb.exceptions import ArcadeDBError # noqa: E402 + +COLP, ROWS = "arcadedb_n_insert_colp", "arcadedb_n_insert_rows" + + +class Spy: + """Counts the crossings by name and the calls that hand a chunk of dict rows to the parallel writers.""" + + def __init__(self, monkeypatch): + self.calls, self.row_chunks = [], 0 + real_call, real_rows = lib.call, extras.insert_parallel + + def call(name, *args): + self.calls.append(name) + return real_call(name, *args) + + def insert_parallel(database, type_name, rows): + self.row_chunks += 1 + return real_rows(database, type_name, rows) + + monkeypatch.setattr(lib, "call", call) + monkeypatch.setattr(extras, "insert_parallel", insert_parallel) + + def count(self, name): + return self.calls.count(name) + + +@pytest.fixture +def spy(monkeypatch): + return Spy(monkeypatch) + + +def _count(db, name): + return db.query("sql", f"SELECT count(*) AS n FROM {name}").to_list()[0][ + "n" + ] # nosec B608 - test-owned type name + + +def _types(db, *names, kind="DOCUMENT", buckets=3, unique=False): + for name in names: + db.command("sql", f"CREATE {kind} TYPE {name} BUCKETS {buckets}") + db.command("sql", f"CREATE PROPERTY {name}.id INTEGER") + db.command("sql", f"CREATE PROPERTY {name}.val DOUBLE") + db.command("sql", f"CREATE PROPERTY {name}.flag BOOLEAN") + db.command("sql", f"CREATE PROPERTY {name}.txt STRING") + db.command("sql", f"CREATE PROPERTY {name}.vec ARRAY_OF_FLOATS") + if unique: + db.command("sql", f"CREATE INDEX ON {name} (id) UNIQUE") + + +def _stored(db, name): + """Every record of a type as (value, python type) pairs per property, ordered by id (the writers store in no fixed order).""" + sql = f"SELECT FROM {name} ORDER BY id" # nosec B608 - test-owned type name + return [ + {k: (v, type(v).__name__) for k, v in row.items() if not k.startswith("@")} + for row in db.query("sql", sql).to_list() + ] + + +def _columns(n, dim=4, seed=11): + rng = np.random.default_rng(seed) + ids = np.arange(n, dtype=np.int64) + return { + "id": ids, + "val": (ids * 37 % 10007) / 7.0, + "flag": ids % 3 == 0, + "txt": ["row-%d-%s" % (k, "é" * (k % 5)) for k in range(n)], + "vec": rng.standard_normal((n, dim), dtype=np.float32), + } + + +def _rows(columns): + names = list(columns) + lists = [c.tolist() if hasattr(c, "tolist") else c for c in columns.values()] + return [dict(zip(names, vals)) for vals in zip(*lists)] + + +def test_a_parallel_call_uses_the_columnar_entry_and_stores_the_row_paths_records( + temp_db, spy +): + _types(temp_db, "Ref", "Col") + columns = _columns(3_100) + assert temp_db.insert_many("Ref", _rows(columns), parallel=True) == 3_100 + assert spy.row_chunks == 1 and spy.count(COLP) == 0 # the reference is the row path + + spy.calls.clear() + spy.row_chunks = 0 + assert temp_db.insert_columns("Col", columns, parallel=True) == 3_100 + assert spy.count(COLP) == 1, spy.calls # one crossing for the whole call + assert spy.row_chunks == 0 and spy.count(ROWS) == 0 # no dict rows, no JSON text + + expected = _stored(temp_db, "Ref") + assert len(expected) == 3_100 and _count(temp_db, "Col") == 3_100 + assert expected[7]["id"][1] == "int" and expected[7]["vec"][1] == "list" + assert _stored(temp_db, "Col") == expected + + +def test_nulls_and_awkward_strings_are_stored_as_the_row_path_stores_them(temp_db, spy): + _types(temp_db, "Ref", "Col") + columns = { + "id": list(range(8)), + "txt": ["", "a", None, "snow☃man", "x" * 300, None, "\n\t", "z"], + "val": [1.5, None, 2.5, 1e300, -1e-300, 0.0, 3.0, None], + } + temp_db.insert_many("Ref", _rows(columns), parallel=True) + spy.calls.clear() + temp_db.insert_columns("Col", columns, parallel=True) + assert spy.count(COLP) == 1 + got, want = _stored(temp_db, "Col"), _stored(temp_db, "Ref") + assert got == want and got[2]["txt"][0] is None + + +def test_a_vertex_type_gets_vertices_in_parallel(temp_db, spy): + _types(temp_db, "Ref", "Col", kind="VERTEX") + columns = _columns(1_500) + temp_db.insert_many("Ref", _rows(columns), parallel=True) + spy.calls.clear() + temp_db.insert_columns("Col", columns, parallel=True) + assert spy.count(COLP) == 1 + assert _stored(temp_db, "Col") == _stored(temp_db, "Ref") + kinds = temp_db.query( + "sql", "SELECT FROM schema:types WHERE name = 'Col'" + ).to_list() + assert kinds[0]["type"] == "vertex" + + +def test_a_record_the_writers_reject_is_reported_after_the_wait(temp_db, spy): + _types(temp_db, "Col", unique=True) + ids = np.arange(400, dtype=np.int64) + ids[300] = ids[7] # a duplicate key + with pytest.raises(ArcadeDBError, match="failed record"): + temp_db.insert_columns("Col", {"id": ids, "txt": ["t"] * 400}, parallel=True) + assert spy.count(COLP) == 1 and spy.row_chunks == 0 + stored = _count(temp_db, "Col") + assert 0 < stored <= 400 # the call had waited for the writers: nothing lands later + + +def test_a_value_the_declared_type_refuses_raises_and_the_call_is_final(temp_db, spy): + _types(temp_db, "Col") + values = [str(i) for i in range(1_000)] + values[700] = "not a number" + with pytest.raises( + ArcadeDBError, match="may have been stored|failed record" + ) as err: + temp_db.insert_columns("Col", {"id": values}, parallel=True) + assert "rolled back" not in str(err.value) + assert spy.count(COLP) == 1 + stored = _count(temp_db, "Col") + temp_db.async_executor().wait_completion() + assert _count(temp_db, "Col") == stored <= 1_000 + + +def test_an_edge_type_is_refused_as_before(temp_db): + temp_db.command("sql", "CREATE EDGE TYPE Link") + with pytest.raises(ArcadeDBError, match="edge type"): + temp_db.insert_columns("Link", {"id": [1, 2, 3]}, parallel=True) + + +def test_a_missing_type_is_the_schemas_error(temp_db): + with pytest.raises(ArcadeDBError, match="NoSuchType"): + temp_db.insert_columns("NoSuchType", {"id": [1, 2, 3]}, parallel=True) + + +def test_a_column_the_buffers_cannot_carry_still_goes_through_the_row_path( + temp_db, spy +): + _types(temp_db, "Col") + columns = {"id": [1, 2, 3, 4], "txt": ["a", 2, "c", 4.5]} # a mixed-type column + assert temp_db.insert_columns("Col", columns, parallel=True) == 4 + assert spy.count(COLP) == 0 and spy.row_chunks == 1 + assert _count(temp_db, "Col") == 4 + + +def test_the_switch_that_turns_the_columnar_path_off_applies_to_parallel( + temp_db, spy, monkeypatch +): + _types(temp_db, "Col") + monkeypatch.setattr(bulk, "_ENABLED", False) + assert temp_db.insert_columns("Col", _columns(50), parallel=True) == 50 + assert spy.count(COLP) == 0 and spy.row_chunks == 1 + assert _count(temp_db, "Col") == 50 + + +def test_no_rows_is_zero_and_writes_nothing(temp_db, spy): + _types(temp_db, "Col") + assert temp_db.insert_columns("Col", {"id": []}, parallel=True) == 0 + assert spy.count(COLP) == 0 and _count(temp_db, "Col") == 0 + + +def test_the_synchronous_call_is_unchanged(temp_db, spy): + _types(temp_db, "Col", unique=True) + assert temp_db.insert_columns("Col", _columns(500), commit_every=100) == 500 + assert spy.count(COLP) == 1 and spy.row_chunks == 0 + assert _count(temp_db, "Col") == 500 From 9d6d56c8168b0bb1cf89a5d874b2565d68670747 Mon Sep 17 00:00:00 2001 From: Taewoon Kim Date: Mon, 9 Nov 2026 18:54:50 +0900 Subject: [PATCH 05/49] Native: save() of a new record and new_edge read the row in the crossing of the statement (#317) The create path read the row it always reads with a second crossing (arcadedb_n_next_bin after arcadedb_n_cmd). It now uses the form of arcadedb_n_exec whose reply carries the first rows, and so do the two UPDATE forms of save(). The stale-record check stays: an UPDATE's own reply comes after the write. Co-Authored-By: Claude Sonnet 5.5 --- src/arcadedb/_native/database.py | 15 +- src/arcadedb/_native/records.py | 8 +- tests/conftest.py | 2 + tests/test_native_one_row_crossings.py | 214 +++++++++++++++++++++++++ 4 files changed, 233 insertions(+), 6 deletions(-) create mode 100644 tests/test_native_one_row_crossings.py diff --git a/src/arcadedb/_native/database.py b/src/arcadedb/_native/database.py index f30dc526..8016245e 100644 --- a/src/arcadedb/_native/database.py +++ b/src/arcadedb/_native/database.py @@ -40,6 +40,7 @@ _FIRST_BATCH = 256 _EPOCH_DAY_ORDINAL = date(1970, 1, 1).toordinal() _NEXT_BATCH = 2048 +_EAGER = 8 # a flag of _exec that never reaches the library: the statement's first rows come back in its own reply _wrap_record = None _KIND = {"v": "v", "e": "e", "d": "d"} @@ -608,12 +609,14 @@ def _exec( self, flags: int, prefix: str, language: str, command: str, args ) -> NativeResultSet: self._check_not_closed() + eager = flags & _EAGER + flags &= ~_EAGER if args: refuse_surplus_positionals( "command" if flags & 1 else "query", language, command, args ) interrupts.clear() - if flags & 1 and _LAZY_COMMAND_ROWS and lib.CMD_PATH: + if flags & 1 and _LAZY_COMMAND_ROWS and lib.CMD_PATH and not eager: # phase 8: the cursor id comes back as a long (no reply text to free and decode); the rows are read late or never text, buf, count = encode_params(args, binary=True) rid = lib.cmd( @@ -630,7 +633,7 @@ def _exec( raise _error(prefix, json.loads(lib.last_error())) return NativeResultSet._lazy(self, rid) # phase 7 (arcadedb_n_exec flags): 2 = a command's rows are pulled but encoded only when read, 4 = float arrays come back as a float32 tail - if flags & 1 and _LAZY_COMMAND_ROWS: + if flags & 1 and _LAZY_COMMAND_ROWS and not eager: flags |= 2 if _BINARY_REPLY: flags |= 4 @@ -680,6 +683,14 @@ def command(self, language: str, command: str, *args) -> NativeResultSet: """Run a statement that may write.""" return self._exec(1, "Command failed", language, command, args) + def _command_first_row(self, language: str, command: str, *args) -> NativeResultSet: + """:meth:`command` for a caller that reads the first row right away (a create or an update that returns its row): the statement runs + and its first rows come back in the same crossing (``arcadedb_n_exec`` without the lazy bit) instead of a cursor id that a second + crossing (``arcadedb_n_next_bin``) reads. The rows, the values and the errors are those of :meth:`command`; an error met while + the first rows are computed is raised when they are read, as on the lazy path. + """ + return self._exec(1 | _EAGER, "Command failed", language, command, args) + # -- transactions def _tx_text(self, op: int, prefix: str) -> str: diff --git a/src/arcadedb/_native/records.py b/src/arcadedb/_native/records.py index 4711c193..c42d9735 100644 --- a/src/arcadedb/_native/records.py +++ b/src/arcadedb/_native/records.py @@ -338,7 +338,7 @@ def save(self): from .extras import veto_count vetoes = veto_count() - row = db.command( + row = db._command_first_row( "sql", self._CREATE.format(t=_q(self._type)), {"row": self._props} ).first() if row is None and veto_count() != vetoes: @@ -388,11 +388,11 @@ def _save_update(self, db) -> None: vetoes = veto_count() params = {"rid": RID(self._rid), "row": self._props} if exact: - row = db.command("sql", self._UPDATE_SENT, params).first() + row = db._command_first_row("sql", self._UPDATE_SENT, params).first() if row is not None and row.get("count") and veto_count() == vetoes: self._base = sent else: - after = db.command("sql", self._UPDATE, params).first() + after = db._command_first_row("sql", self._UPDATE, params).first() if ( after is not None ): # a veto or a record that is gone gives no row: nothing was written, the copy stays @@ -450,7 +450,7 @@ def new_edge(self, label: str, target: "Vertex", **kwargs) -> "Edge": if kwargs: sql += " CONTENT :row" params["row"] = kwargs - row = self._db.command("sql", sql, params).first() + row = self._db._command_first_row("sql", sql, params).first() edge = Edge(self._db, label, mutable=False) edge._adopt(row) return edge diff --git a/tests/conftest.py b/tests/conftest.py index ec3147c7..ee4981ff 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -65,6 +65,8 @@ collect_ignore.append("test_native_async_callbacks.py") # the short call path of a point read (codec.encode_params, lib.exec_query, first()), observed on the native library collect_ignore.append("test_native_point_read.py") + # the crossings of save() and new_edge() (lib.cmd, lib.call, lib.call_bin, lib.exec_query are the native library's ctypes doors) + collect_ignore.append("test_native_one_row_crossings.py") # an RID built from an RID keeps the text, so the fast named-parameter path of codec.encode_params can encode it (#265) collect_ignore.append("test_native_rid_params.py") # to_json_list walks the rows with codec.json_native only when a reply restored a tagged value (#219), observed on the native library diff --git a/tests/test_native_one_row_crossings.py b/tests/test_native_one_row_crossings.py new file mode 100644 index 00000000..227404b2 --- /dev/null +++ b/tests/test_native_one_row_crossings.py @@ -0,0 +1,214 @@ +"""A created or updated record costs the fewest native crossings (conftest.py collects this file on the native backend only). + +``save()`` of a new record and ``Vertex.new_edge`` read the row the statement returns, so they used the lazy form of the command +(``arcadedb_n_cmd``: a cursor id, then a second crossing, ``arcadedb_n_next_bin``, for the row they always read). They now use the +form whose reply carries the first rows (``arcadedb_n_exec``): one crossing. ``save()`` of a saved record spent four crossings (the +transaction check, the SELECT of the stale-record check, and two for the UPDATE) and now spends three. The stale-record check still +reads the stored record first: the engine's reply to an UPDATE comes after the write, and the check must refuse before writing +anything inside the caller's transaction, so it cannot be read from the update's own result. + +The counts are crossings (the ctypes doors ``cmd``, ``call``, ``call_bin``, ``exec_query`` and the batched cursor close), not seconds: +deterministic. The oracle for the values and the errors is the lazy form itself (``db.command(...).first()``), kept for every other caller. +""" + +import datetime as dt +from decimal import Decimal + +import numpy as np +import pytest + +from arcadedb import ArcadeDBError +from arcadedb._native import lib + + +class _Crossings: + """Every call into the native library made while installed, by entry point.""" + + def __init__(self, monkeypatch): + self.names = [] + for attr in ("cmd", "call", "call_bin", "exec_query"): + monkeypatch.setattr(lib, attr, self._spy(attr, getattr(lib, attr))) + real_close = lib._lib.arcadedb_n_rs_close_many + + def close_many(*args): + self.names.append("arcadedb_n_rs_close_many") + return real_close(*args) + + monkeypatch.setattr(lib._lib, "arcadedb_n_rs_close_many", close_many) + + def _spy(self, attr, real): + def spy(*args, **kwargs): + self.names.append(args[0] if attr in ("call", "call_bin") else attr) + return real(*args, **kwargs) + + return spy + + def reset(self): + lib.flush_closes() # a cursor closed earlier is not part of the call measured + del self.names[:] + + +@pytest.fixture +def db(temp_db): + temp_db.command("sql", "CREATE DOCUMENT TYPE D") + temp_db.command("sql", "CREATE VERTEX TYPE V") + temp_db.command("sql", "CREATE EDGE TYPE E") + return temp_db + + +@pytest.fixture +def crossings(db, monkeypatch): + return _Crossings(monkeypatch) + + +def _plain(values): + return {k: v for k, v in values.items() if not k.startswith("@")} + + +# -- create + + +def test_saving_a_new_document_is_one_crossing(db, crossings): + with db.transaction(): + doc = db.new_document("D") + doc.set("a", 1) + crossings.reset() + doc.save() + assert crossings.names == [ + "exec_query" + ] # was: arcadedb_n_cmd, then arcadedb_n_next_bin + assert doc.get_identity() is not None + + +def test_saving_a_new_vertex_with_a_float_array_is_one_crossing(db, crossings): + with db.transaction(): + vertex = db.new_vertex("V") + vertex.set("v", np.arange(16, dtype=np.float32)) + crossings.reset() + vertex.save() + assert crossings.names == [ + "arcadedb_n_exec_bin" + ] # the float side buffer: the long path, still one crossing + + +def test_new_edge_is_one_crossing_with_and_without_properties(db, crossings): + with db.transaction(): + a, b = db.new_vertex("V"), db.new_vertex("V") + a.save() + b.save() + crossings.reset() + plain = a.new_edge("E", b) + assert crossings.names == ["exec_query"] + crossings.reset() + weighted = a.new_edge("E", b, w=1.5, tag="x") + assert crossings.names == ["exec_query"] + assert plain.get_identity() != weighted.get_identity() + assert weighted.get("w") == 1.5 and weighted.get("tag") == "x" + + +CREATE_CASES = { + "ints and text": {"a": 1, "b": "text", "c": None, "d": True, "e": 2.5}, + "non-ascii": {"s": "café 日本語 \U0001f600"}, + "date and time": { + "d": dt.date(2026, 11, 9), + "t": dt.datetime(2026, 11, 9, 12, 30, 15, 123000), + }, + "decimal": {"x": Decimal("1.50")}, + "list and map": {"l": [1, 2, 3], "m": {"k": [1.5, "v"]}}, + "float array": {"v": np.arange(16, dtype=np.float32)}, + "nothing": {}, +} + + +@pytest.mark.parametrize("case", list(CREATE_CASES)) +def test_a_created_record_has_the_values_the_lazy_form_gives(db, case): + values = CREATE_CASES[case] + with db.transaction(): + doc = db.new_document("D") + for name, value in values.items(): + doc.set(name, value) + doc.save() + # the oracle: the same statement through the lazy command form every other caller still uses + want = db.command("sql", "INSERT INTO D CONTENT :row", {"row": values}).first() + got = db.lookup_by_rid(str(doc.get_identity())) + assert doc.get_type_name() == want.get_element().get_type_name() == "D" + assert _plain(doc.to_dict()) == _plain(want.to_dict()) == _plain(got.to_dict()) + + +def test_a_create_that_the_engine_refuses_fails_with_the_lazy_forms_message(db): + db.command("sql", "CREATE PROPERTY D.k STRING") + db.command("sql", "CREATE INDEX ON D (k) UNIQUE") + with db.transaction(): + db.command("sql", "INSERT INTO D SET k = 'dup'").close() + with pytest.raises(ArcadeDBError) as lazy: + with db.transaction(): + db.command( + "sql", "INSERT INTO D CONTENT :row", {"row": {"k": "dup"}} + ).first() + with pytest.raises(ArcadeDBError) as eager: + with db.transaction(): + doc = db.new_document("D") + doc.set("k", "dup") + doc.save() + assert str(eager.value) == str(lazy.value) + assert type(eager.value) is type(lazy.value) + + +# -- update + + +def _saved(db, **values): + with db.transaction(): + doc = db.new_document("D") + for name, value in values.items(): + doc.set(name, value) + doc.save() + return str(doc.get_identity()) + + +def test_updating_a_saved_record_is_three_crossings(db, crossings): + rid = _saved(db, a=1, name="x") + with db.transaction(): + record = db.lookup_by_rid(rid).modify() + record.set("a", 2) + crossings.reset() + record.save() + # the transaction check, the SELECT of the stale-record check, the UPDATE (was four: the UPDATE took two) + assert crossings.names == ["arcadedb_n_tx", "exec_query", "exec_query"] + assert db.lookup_by_rid(rid).get("a") == 2 + + +def test_an_update_that_reads_its_row_back_is_three_crossings(db, crossings): + rid = _saved(db, a=1) + with db.transaction(): + record = db.lookup_by_rid(rid).modify() + record.set("brand_new", "value") # a new property is read back (RETURN AFTER) + crossings.reset() + record.save() + assert crossings.names == ["arcadedb_n_tx", "exec_query", "exec_query"] + assert db.lookup_by_rid(rid).get("brand_new") == "value" + + +def test_a_stale_record_is_still_refused_before_anything_is_written(db): + rid = _saved(db, a=1, b=1) + mine = db.lookup_by_rid(rid).modify() + with db.transaction(): + db.command( + "sql", f"UPDATE {rid} SET b = 9" + ).close() # nosec B608 - a record id this test read + mine.set("a", 2) + with pytest.raises(ArcadeDBError, match="ConcurrentModificationException"): + with db.transaction(): + mine.save() + stored = db.lookup_by_rid(rid) + assert (stored.get("a"), stored.get("b")) == (1, 9) # nothing of mine was written + + +def test_an_update_outside_a_transaction_still_raises_before_reading(db, crossings): + rid = _saved(db, a=1) + record = db.lookup_by_rid(rid).modify() + record.set("a", 2) + crossings.reset() + with pytest.raises(ArcadeDBError, match="Transaction not begun"): + record.save() + assert crossings.names == ["arcadedb_n_tx"] From 25bceaa8d7dce93cf57d438a010031b0344df502 Mon Sep 17 00:00:00 2001 From: Taewoon Kim Date: Mon, 9 Nov 2026 18:56:41 +0900 Subject: [PATCH 06/49] Native insert_arrow and insert_dataframe with parallel=True: the same columnar entry (refs #325) A document load with parallel=True left the Arrow columns as Python values and went to insert_many(parallel=True). It now keeps the columns as they are for the columnar entry, which hands the records to the async writers; a batch the buffers cannot carry still takes the row path. A test spies on the entry points and compares the stored records with the row path of the old code. Co-Authored-By: Claude Sonnet 5.5 --- src/arcadedb/dataframe.py | 19 +++++++++++--- tests/test_native_columnar_parallel.py | 35 ++++++++++++++++++++++++++ 2 files changed, 51 insertions(+), 3 deletions(-) diff --git a/src/arcadedb/dataframe.py b/src/arcadedb/dataframe.py index 9d3ee0b7..d65e51b8 100644 --- a/src/arcadedb/dataframe.py +++ b/src/arcadedb/dataframe.py @@ -434,7 +434,7 @@ def insert_table( batch.column(i), pa, native, - arrow=native and not is_vertex and not parallel, + arrow=native and not is_vertex, ) for i, name in enumerate(names) } @@ -497,9 +497,22 @@ def _insert_documents_native( """One batch of documents on the native backend. When every column can cross as memory (numbers, bools, strings, nulls, float vectors for an ``ARRAY_OF_FLOATS`` property) the batch goes through the columnar entry, no Python values and no JSON text; otherwise the columns become one dict per row and go to ``insert_many`` (one crossing per 10,000 rows). Either way - the batch is one transaction, or joins the caller's. + the batch is one transaction, or joins the caller's. With ``parallel`` the same columnar entry hands the records to the async + writers (no transaction), and the call waits for them and raises when a record failed, as ``insert_many(parallel=True)`` does. """ - if not parallel: + if parallel: + from ._native.bulk import insert_columns_parallel + + try: + sent = insert_columns_parallel( + db, type_name, columns, {} if declared is None else declared + ) + except ArcadeDBError as e: + raise db._parallel_load_failed(type_name, e) from e + if sent is not None: + db._finish_parallel(type_name, [sent[1]], sent[0]) + return + else: from ._native.bulk import insert_columns_fast if ( diff --git a/tests/test_native_columnar_parallel.py b/tests/test_native_columnar_parallel.py index f51fd7c7..0f5d80d9 100644 --- a/tests/test_native_columnar_parallel.py +++ b/tests/test_native_columnar_parallel.py @@ -206,3 +206,38 @@ def test_the_synchronous_call_is_unchanged(temp_db, spy): assert temp_db.insert_columns("Col", _columns(500), commit_every=100) == 500 assert spy.count(COLP) == 1 and spy.row_chunks == 0 assert _count(temp_db, "Col") == 500 + + +def test_insert_arrow_parallel_takes_the_columnar_entry(temp_db, spy, monkeypatch): + import pyarrow as pa + + _types(temp_db, "Ref", "ByTable") + columns = _columns(2_000) + table = pa.table( + { + "id": columns["id"], + "val": pa.array( + [None if i % 7 == 0 else v for i, v in enumerate(columns["val"])] + ), + "txt": pa.array(columns["txt"]), + } + ) + with ( + monkeypatch.context() as m + ): # the private reference: the row path of the old code + m.setattr(bulk, "_ENABLED", False) + assert ( + temp_db.insert_arrow("Ref", table, batch_size=700, parallel=True) == 2_000 + ) + assert spy.count(COLP) == 0 and spy.row_chunks >= 1 + + spy.calls.clear() + spy.row_chunks = 0 + assert ( + temp_db.insert_arrow("ByTable", table, batch_size=700, parallel=True) == 2_000 + ) + assert spy.count(COLP) == 3 and spy.row_chunks == 0 # one crossing per batch + + expected = _stored(temp_db, "Ref") + assert len(expected) == 2_000 and expected[0]["val"][0] is None + assert _stored(temp_db, "ByTable") == expected From 8bf380cc350b3e0d3413794772365547c8afe3d9 Mon Sep 17 00:00:00 2001 From: Taewoon Kim Date: Mon, 9 Nov 2026 18:58:18 +0900 Subject: [PATCH 07/49] insert_many and create_vertices: check the rows in the one json.dumps pass, not in a loop before it (refs #318) json_bulk_dumps and json_bulk_dumps_tagged walked every value of every row in Python before json.dumps walked the rows again. The rows are now written once and the checks come from what that pass shows: the text (a surrogate escape, an integer token of 2**63 and above) and a census of the types of the keys and values in C. The exact per-row walk (_check_rows) is the reference and runs only when a value may be bad, and when json.dumps itself refuses the rows, so the exception type and message for the first bad row in row order are the old loop's. Fuzzed against the old code: 60000 cases, no difference. JVM GraphBatch.create_vertices: classify_vertex_rows decides the shapes by type census instead of a json_bulk_scalar_ok call per value; a scalar the text would change (NaN, integer beyond 64 bits, lone surrogate, subnormal) now sends its own chunk through the property-matrix path, as a tagged chunk already did. The untagged path now also refuses a subnormal, as the tagged one did. NativeGraphBatch: the dead _create_vertices_json_bulk is deleted, and the row type check is a C census. Co-Authored-By: Claude Sonnet 5.5 --- src/arcadedb/_native/extras.py | 29 +- src/arcadedb/graph_batch.py | 83 ++--- src/arcadedb/type_conversion.py | 228 +++++++++--- tests/conftest.py | 1 + tests/test_bulk_json_paths.py | 11 +- tests/test_bulk_single_pass_validation.py | 418 ++++++++++++++++++++++ tests/test_graph_batch_single_pass.py | 84 +++++ tests/test_graph_batch_tagged_shapes.py | 2 + 8 files changed, 723 insertions(+), 133 deletions(-) create mode 100644 tests/test_bulk_single_pass_validation.py create mode 100644 tests/test_graph_batch_single_pass.py diff --git a/src/arcadedb/_native/extras.py b/src/arcadedb/_native/extras.py index 5f8a4702..82440e9a 100644 --- a/src/arcadedb/_native/extras.py +++ b/src/arcadedb/_native/extras.py @@ -28,6 +28,7 @@ _UPCALL_TYPE = ctypes.CFUNCTYPE( None, ctypes.c_longlong, ctypes.c_char_p, ctypes.c_void_p ) +_DICT_OR_NONE = frozenset((dict, type(None))) _callbacks: dict = {} _callback_ids = itertools.count(1) _callbacks_lock = threading.Lock() @@ -229,21 +230,6 @@ def _to_java_varargs(self, properties: dict) -> dict: raise TypeError(f"property names must be str, got {key!r}") return properties - def _create_vertices_json_bulk(self, type_name: str, rows): - """The bulk path (JSON rows in, RIDs out): the RIDs, or None for rows JSON would change (the caller falls back).""" - from ..type_conversion import json_bulk_scalar_ok - - for row in rows: - for key, value in (row or {}).items(): - if type(key) is not str or not json_bulk_scalar_ok(value): - return None - return self._op( - f"Failed to create batch vertices for type '{type_name}'", - "create_vertices", - type=type_name, - rows=[r or {} for r in rows], - )["rids"] - def create_vertices(self, type_name: str, count_or_properties) -> list: """The RIDs of the new vertices. Rows go in chunks of the batch's ``commit_every`` (1000 unless graph_batch() got another value; 0: 10,000), one crossing and one transaction per chunk: chunks that already went in stay stored if a later chunk fails. @@ -274,11 +260,14 @@ def create_vertices(self, type_name: str, count_or_properties) -> list: ] return rids rows = list(count_or_properties) - for row in rows: - if row is not None and not isinstance(row, dict): - raise TypeError( - "create_vertices() property rows must be dicts or None" - ) + if ( + not set(map(type, rows)) <= _DICT_OR_NONE + ): # exact types in C; only an odd row pays the isinstance loop + for row in rows: + if row is not None and not isinstance(row, dict): + raise TypeError( + "create_vertices() property rows must be dicts or None" + ) # chunked, as the JPype wrapper does: one crossing, one JSON text and one reply per chunk, so a 1M-vertex load # never holds the text, the parsed rows and the reply of all of it at once (the Serial GC copies what lives) rids = [] diff --git a/src/arcadedb/graph_batch.py b/src/arcadedb/graph_batch.py index 03495844..efc1aff2 100644 --- a/src/arcadedb/graph_batch.py +++ b/src/arcadedb/graph_batch.py @@ -12,9 +12,11 @@ from .exceptions import ArcadeDBError from .graph import Document, Vertex from .type_conversion import ( + classify_vertex_rows, convert_python_to_java, declared_float_vectors, - is_vector_candidate, + declared_scalars_ok, + json_bulk_dumps, json_bulk_dumps_tagged, json_bulk_scalar_ok, json_bulk_shape_ok, @@ -256,58 +258,40 @@ def _create_vertices_json_bulk(self, type_name, rows): or bytes value crosses inside the JSON as the tagged value json_bulk_dumps_tagged writes and VertexBatcher decodes (the stored value is what the property-matrix path stores). An input whose values - are all scalars takes the untagged json.dumps path as before. A chunk - holding a value the text cannot carry (a NaN, an integer beyond 64 - bits or a date-time inside a list, a subnormal) goes through the - property-matrix path on its own. Returns None when - unavailable/unsuitable so the caller falls back to the property-matrix - path for the whole input.""" + are all scalars is written by json_bulk_dumps. The values are checked + in the one json.dumps pass of each chunk (see type_conversion._dumps_once), + not in a loop over the rows before it: a chunk holding a value the text + cannot carry (a NaN, an integer beyond 64 bits, a lone surrogate, a + subnormal, or a date-time inside a list) goes through the property-matrix + path on its own. Returns None when unavailable/unsuitable (a property + name that is not a str, a type the text does not carry) so the caller + falls back to the property-matrix path for the whole input.""" try: vertex_batcher = jpype.JClass("com.arcadedb.python.VertexBatcher") declared = set(declared_float_vectors(self._java_db, type_name)) except Exception: return None - tagged = False # a value that needs the tagged text; False keeps the untagged code of before - for row in rows: - if row: - for key, value in row.items(): - if type(key) is not str: - # a property name that is not a str, which json.dumps would - # write as text ({1: "a"} as "1") where the matrix path - # refuses it - return None - if key in declared and is_vector_candidate(value): - continue # the vector split below takes it, or its chunk goes the matrix way - if not json_bulk_scalar_ok(value): - if not json_bulk_shape_ok(value): - # a type the tagged JSON does not carry (a set, a numpy - # or pandas value, any subclass, a list of only numbers: - # the matrix path converts those), or a scalar JSON - # would change: an integer beyond 64 bits, NaN or - # Infinity, a lone surrogate - return None - tagged = True - - import json + tagged = classify_vertex_rows(rows, declared) + if tagged is None: + # a property name that is not a str (json.dumps would write 1 as "1" where the matrix path refuses it), or a type the + # tagged JSON does not carry (a set, a numpy or pandas value, any subclass, a list of only numbers: the matrix path + # converts those) + return None + dumps = json_bulk_dumps_tagged if tagged else json_bulk_dumps # chunked so huge ingests never materialize one giant JSON string rids = [] for start in range(0, len(rows), self._BULK_CHUNK): chunk = rows[start : start + self._BULK_CHUNK] split = split_float_vectors(self._java_db, type_name, chunk, declared) rest = split[0] if split else chunk - if not tagged: - text = None - if not declared or self._json_bulk_ok(rest): - text = json.dumps([row or {} for row in rest]) - else: - text = None - if self._json_bulk_declared_ok(rest, declared): - try: - text = json_bulk_dumps_tagged([row or {} for row in rest]) - except (TypeError, ValueError): - text = None # a value the text cannot carry, see the docstring + text = None + if declared_scalars_ok(rest, declared): + try: + text = dumps([row or {} for row in rest]) + except (TypeError, ValueError): + text = None # a value the text cannot carry, see the docstring if text is None: # a list the matrix cannot carry (ragged, a null, not numbers) # is left in this chunk, or a value the text would change: the @@ -330,25 +314,6 @@ def _create_vertices_json_bulk(self, type_name, rows): rids.extend(joined.split(";")) return rids - @staticmethod - def _json_bulk_ok(rows): - for row in rows: - if row: - for value in row.values(): - if not json_bulk_scalar_ok(value): - return False - return True - - @staticmethod - def _json_bulk_declared_ok(rows, declared): - """False when a declared ARRAY_OF_FLOATS property still holds a list (or any non-scalar) after the vector split.""" - for row in rows: - if row: - for key in declared: - if key in row and not json_bulk_scalar_ok(row[key]): - return False - return True - def new_edge( self, source_vertex_or_rid, diff --git a/src/arcadedb/type_conversion.py b/src/arcadedb/type_conversion.py index 1eca63ed..8f90133c 100644 --- a/src/arcadedb/type_conversion.py +++ b/src/arcadedb/type_conversion.py @@ -5,6 +5,7 @@ developer experience and integration with Python ecosystem (pandas, numpy, etc.). """ +import itertools import json import math import re @@ -804,16 +805,38 @@ def _dumps_checked(rows: Any, **kwargs: Any) -> str: return text -def json_bulk_dumps(rows: Any) -> str: - """json.dumps for the bulk paths: raises ValueError (or TypeError) when - any value in `rows` would change on the way to the engine. +def _int64_overflow_pattern() -> str: + """The digit runs of 2**63 and above, which a JSON integer token beyond 64 bits has. + + A run of 20 digits or more, or of 19 digits that is not below 9223372036854775808. A run that follows a digit or a "." is not an + integer token (the digits after the point of a float such as 0.00012345678901234568 are no token), so only a run start counts. + """ + limit = str(2**63) + parts = [r"[0-9]{20,}"] + for i, digit in enumerate(limit): + low = int(digit) if i == len(limit) - 1 else int(digit) + 1 + if low <= 9: + parts.append(f"{limit[:i]}[{low}-9][0-9]{{{len(limit) - 1 - i}}}") + return r"(? None: + """The per-row walk: raise ValueError (or TypeError) for the first row, in row order, that holds a value the JSON text would change. + + This is the exact reference. The bulk dumps do not run it on every call: `_dumps_once` runs it only when the text or the types of + the rows show that a value may be bad, and when json.dumps itself refused the rows, so the error is always the walk's. + `tagged` is insert_many's format (see json_bulk_dumps_tagged), where the kinds bulk_tag_default writes are allowed in a row. """ int_min, int_max = _INT64_MIN, _INT64_MAX + kinds = _TAGGED_KINDS for row in rows: for key in row: if type(key) is not str: @@ -832,11 +855,87 @@ def json_bulk_dumps(rows: Any) -> str: ) elif kind is float or kind is bool or value is None: continue + elif tagged and kind in kinds: + continue # bulk_tag_default checks it as json.dumps writes it + elif tagged and kind is dict and _TAG in value: + raise ValueError( + "a map keyed by the tag marker cannot go through the JSON bulk path" + ) + elif tagged and ( + isinstance(value, (datetime, date, Decimal, bytes, bytearray)) + or (_np is not None and isinstance(value, _np.generic)) + or _is_pandas_null(kind) + ): + continue # a subclass (pandas.Timestamp), a numpy scalar, pandas NA or NaT else: _json_bulk_check( value ) # nested lists and dicts, and subclasses of the types above - return _dumps_checked(rows) + + +def _kind_is_tagged(kind: type) -> bool: + """Whether bulk_tag_default (not the walk) checks a value of this type: the kinds `_check_rows` lets through in tagged mode.""" + return ( + kind in _TAGGED_KINDS + or issubclass(kind, (datetime, date, Decimal, bytes, bytearray)) + or (_np is not None and issubclass(kind, _np.generic)) + or _is_pandas_null(kind) + ) + + +def _rows_need_walk(rows: Any, tagged: bool) -> bool: + """Whether the types of the rows leave a check that the JSON text cannot show: only exact dict rows with str keys and values of + the exact scalar types (and, tagged, the kinds bulk_tag_default checks) are decided by the text. The census runs in C: one pass + over the keys and one over the values, no Python-level loop per row. + """ + if not set(map(type, rows)) <= _DICT_ONLY: + return True + if not set(map(type, itertools.chain.from_iterable(rows))) <= _STR_ONLY: + return True + extra = ( + set(map(type, itertools.chain.from_iterable(map(dict.values, rows)))) + - _BULK_SCALAR_TYPES + ) + if not extra: + return False + return not (tagged and all(map(_kind_is_tagged, extra))) + + +def _dumps_once(rows: Any, tagged: bool) -> str: + """One json.dumps pass for the bulk paths, with the checks of `_check_rows` decided from what that pass shows. + + A str or int the text would change cannot be seen by a json.dumps hook (the encoder writes both itself), so the text decides: + a surrogate escape (a lone surrogate) or an integer token of 2**63 and above sends the rows to the walk, and so do rows whose + types the text cannot vouch for (`_rows_need_walk`). When json.dumps itself refuses the rows (NaN, an unknown type, a + subnormal, a hook error) the walk runs first, as the validation loop used to run before json.dumps, so the error of the first + bad row in row order is still the walk's; if the walk finds nothing, json.dumps's own error stands. A false alarm costs the walk + and changes nothing else. + """ + try: + text = ( + _dumps_checked(rows, default=bulk_tag_default) + if tagged + else _dumps_checked(rows) + ) + except (TypeError, ValueError): + _check_rows(rows, tagged) + raise + if ( + _rows_need_walk(rows, tagged) + or _SURROGATE_ESCAPE.search(text) + or _INT64_OVERFLOW.search(text) + ): + _check_rows(rows, tagged) + return text + + +def json_bulk_dumps(rows: Any) -> str: + """json.dumps for the bulk paths: raises ValueError (or TypeError) when + any value in `rows` would change on the way to the engine. + + `rows` is a list of dicts (one per record). The rows are written once; see `_dumps_once` for how the values are checked. + """ + return _dumps_once(rows, False) # --------------------------------------------------------------------------- @@ -933,43 +1032,7 @@ def json_bulk_dumps_tagged(rows: Any) -> str: through, written by bulk_tag_default. Raises TypeError or ValueError exactly where json_bulk_dumps does for every other value (a datetime inside a list or map included), so the caller can send that row down the per-row path. """ - int_min, int_max = _INT64_MIN, _INT64_MAX - tagged = _TAGGED_KINDS - for row in rows: - for key in row: - if type(key) is not str: - raise ValueError("a non-str key cannot go through the JSON bulk path") - for value in row.values(): - kind = type(value) - if kind is str: - if not value.isascii() and not _str_survives_json(value): - raise ValueError( - "a string with a lone surrogate cannot go through the JSON bulk path" - ) - elif kind is int: - if not int_min <= value <= int_max: - raise ValueError( - f"an integer beyond 64 bits cannot go through the JSON bulk path: {value}" - ) - elif kind is float or kind is bool or value is None: - continue - elif kind in tagged: - continue # bulk_tag_default checks it as json.dumps writes it - elif kind is dict and _TAG in value: - raise ValueError( - "a map keyed by the tag marker cannot go through the JSON bulk path" - ) - elif ( - isinstance(value, (datetime, date, Decimal, bytes, bytearray)) - or (_np is not None and isinstance(value, _np.generic)) - or _is_pandas_null(kind) - ): - continue # a subclass (pandas.Timestamp), a numpy scalar, pandas NA or NaT - else: - _json_bulk_check( - value - ) # nested lists and dicts, and subclasses of the types above - return _dumps_checked(rows, default=bulk_tag_default) + return _dumps_once(rows, True) def bulk_row_ok(row: Any, skip=()) -> bool: @@ -983,6 +1046,81 @@ def bulk_row_ok(row: Any, skip=()) -> bool: return True +_ROWS_OR_NONE = frozenset((dict, type(None))) + + +def _column(rows: Any, key: str): + """The value of `key` in every non-empty row (None where the row lacks it), as one C-level iterator.""" + return map(dict.get, filter(None, rows), itertools.repeat(key)) + + +def classify_vertex_rows(rows: Any, declared: Any): + """What GraphBatch.create_vertices must know about its rows before it writes any chunk, decided by types. + + Returns None when the rows must all go the property-matrix way (a property name that is not a str, or a value of a type the + tagged JSON does not carry: a set, a numpy or pandas value, a list or tuple of only numbers, any other object); otherwise True when + some value needs the tagged text (json_bulk_dumps_tagged) and False when json_bulk_dumps carries every row. A value of an exact + scalar type is not judged here: whether the text carries it (a NaN, an integer beyond 64 bits, a lone surrogate) is decided in + the dumps of its chunk. The property of `declared` (ARRAY_OF_FLOATS) that holds a list or a numpy array is left to + split_float_vectors. The census over the keys and the values runs in C; a Python-level loop runs only over a column that + holds a type other than the exact scalars. + """ + if not set(map(type, rows)) <= _ROWS_OR_NONE: + return _classify_walk( + rows, declared + ) # a row that is no dict: the walk raises what it always raised + keys = set(itertools.chain.from_iterable(filter(None, rows))) + if not set(map(type, keys)) <= _STR_ONLY: + return None + tagged = False + for key in keys: + odd = set(map(type, _column(rows, key))) - _BULK_SCALAR_TYPES + if not odd: + continue + if key in declared and all( + kind is list or (_np is not None and issubclass(kind, _np.ndarray)) + for kind in odd + ): + continue # the vectors of a declared column + for value in _column(rows, key): + if type(value) in _BULK_SCALAR_TYPES or ( + key in declared and is_vector_candidate(value) + ): + continue + if not json_bulk_scalar_ok(value): + if not json_bulk_shape_ok(value): + return None + tagged = True + return tagged + + +def _classify_walk(rows: Any, declared: Any): + tagged = False + for row in rows: + if row: + for key, value in row.items(): + if type(key) is not str: + return None + if key in declared and is_vector_candidate(value): + continue + if not json_bulk_scalar_ok(value): + if not json_bulk_shape_ok(value): + return None + tagged = True + return tagged + + +def declared_scalars_ok(rows: Any, declared: Any) -> bool: + """False when a declared ARRAY_OF_FLOATS property still holds a list (or any non-scalar) after the vector split.""" + for key in declared: + if set(map(type, _column(rows, key))) <= _BULK_SCALAR_TYPES: + continue # scalars: the dumps of the chunk decides whether the text carries them + for row in rows: + if row and key in row and not json_bulk_scalar_ok(row[key]): + return False + return True + + _FLOAT_VECTORS = None # the bridge's FloatVectors class once resolved; False when the bridge jar lacks it diff --git a/tests/conftest.py b/tests/conftest.py index ec3147c7..3ed7133f 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -119,6 +119,7 @@ "test_query_vector_params_jvm.py": "the java.util objects the JVM build's parameter conversion builds", "test_insert_many_gc_pause.py": "the cyclic collector pause is the JVM build's (the native insert_many allocates nothing the collector tracks)", "test_graph_batch_tagged_shapes.py": "the JVM wheel's graph batch bulk path (bridge VertexBatcher and EdgeBatcher, the property-matrix fallback)", + "test_graph_batch_single_pass.py": "the JVM wheel's graph batch bulk path (GraphBatch._create_vertices_json_bulk, the property-matrix fallback)", "test_async_proxy_cache.py": "the JPype proxy classes of the JVM build's async executor", "test_read_crossings.py": "crossing counts of the JVM build's read path (JPype proxies of the Java objects and the bridge classes)", "test_reader_crossings.py": "crossing counts of the JVM build's record and row readers (JPype proxies of the Java objects and the bridge classes)", diff --git a/tests/test_bulk_json_paths.py b/tests/test_bulk_json_paths.py index 8bb7bb07..1c801d00 100644 --- a/tests/test_bulk_json_paths.py +++ b/tests/test_bulk_json_paths.py @@ -104,7 +104,7 @@ def test_a_lone_surrogate_raises_instead_of_becoming_a_question_mark( _write(db, entry, "a\ud800b") -def test_ordinary_rows_stay_on_the_bulk_path(temp_db_path): +def test_ordinary_rows_stay_on_the_bulk_path(): rows = [ {"id": i, "name": f"n{i}", "score": i * 0.5, "flag": i % 2 == 0, "note": None} for i in range(5) @@ -113,14 +113,7 @@ def test_ordinary_rows_stay_on_the_bulk_path(temp_db_path): assert all( type_conversion.json_bulk_scalar_ok(v) for row in rows for v in row.values() ) - with arcadedb.create_database(temp_db_path) as db: - _create_types(db) - with db.graph_batch(parallel_flush=False) as batch: - assert len(batch._create_vertices_json_bulk("V", rows)) == len(rows) - assert batch._create_vertices_json_bulk("V", rows + [{"id": 2**70}]) is None - assert ( - batch._create_vertices_json_bulk("V", rows + [{"x": math.nan}]) is None - ) + # the graph batch's side of this (the JVM wheel's _create_vertices_json_bulk) is in test_graph_batch_single_pass.py @pytest.mark.parametrize( diff --git a/tests/test_bulk_single_pass_validation.py b/tests/test_bulk_single_pass_validation.py new file mode 100644 index 00000000..2610c6d8 --- /dev/null +++ b/tests/test_bulk_single_pass_validation.py @@ -0,0 +1,418 @@ +"""insert_many and create_vertices validate their rows in the one json.dumps pass, not in a Python loop over the rows before it. + +`json_bulk_dumps` and `json_bulk_dumps_tagged` used to walk every value of every row before json.dumps walked it again. Now json.dumps +writes the rows once and the check comes from what that pass shows (the text and the types of the rows), and a walk runs only when it +may find something. The contract is unchanged: the same text for rows that carry, and for a row that does not, the same exception type +and message as the old loop gave for the first bad row in row order. The old loop is copied here as the private reference, so these +tests fail on the old code (they spy on `_check_rows`, which only the new code has) and compare the new code with that reference. +The old messages name no row index, so the first bad row of a large batch is told from a later one by the value in its message. +""" + +import datetime as dt +import decimal +import enum +import json +import math +import random + +import numpy as np +import pytest + +import arcadedb +from arcadedb import type_conversion as tc + +INT64_MAX = 2**63 - 1 +INT64_MIN = -(2**63) + + +def _old_dumps(rows, tagged): + """The json_bulk_dumps and json_bulk_dumps_tagged of 26.11.1: the validation loop, then json.dumps.""" + for row in rows: + for key in row: + if type(key) is not str: + raise ValueError("a non-str key cannot go through the JSON bulk path") + for value in row.values(): + kind = type(value) + if kind is str: + if not value.isascii() and not tc._str_survives_json(value): + raise ValueError( + "a string with a lone surrogate cannot go through the JSON bulk path" + ) + elif kind is int: + if not INT64_MIN <= value <= INT64_MAX: + raise ValueError( + f"an integer beyond 64 bits cannot go through the JSON bulk path: {value}" + ) + elif kind is float or kind is bool or value is None: + continue + elif tagged and kind in tc._TAGGED_KINDS: + continue + elif tagged and kind is dict and tc._TAG in value: + raise ValueError( + "a map keyed by the tag marker cannot go through the JSON bulk path" + ) + elif tagged and ( + isinstance( + value, (dt.datetime, dt.date, decimal.Decimal, bytes, bytearray) + ) + or isinstance(value, np.generic) + or tc._is_pandas_null(kind) + ): + continue + else: + tc._json_bulk_check(value) + if tagged: + return tc._dumps_checked(rows, default=tc.bulk_tag_default) + return tc._dumps_checked(rows) + + +def _outcome(call, rows): + try: + return ("text", call(rows)) + except ( + BaseException + ) as e: # noqa: BLE001 - the type and the message are what is compared + return (type(e).__name__, str(e)) + + +class Colour(enum.IntEnum): + RED = 5 + + +class Name(str): + pass + + +def _scalar_rows(): + return [ + {"id": 1, "name": "alpha", "score": 0.5, "flag": True, "note": None}, + {"id": 2, "name": "", "score": -1e-30, "flag": False, "note": "x"}, + ] + + +CASES = { + "scalars": _scalar_rows(), + "empty": [], + "empty_rows": [{}, {}], + "bounds_fit": [{"a": INT64_MAX, "b": INT64_MIN, "c": 0}], + "above_max": [{"a": INT64_MAX + 1}], + "below_min": [{"a": INT64_MIN - 1}], + "big_negative": [{"a": -(10**30)}], + "huge_digits": [{"a": 10**5000}], + "non_bmp": [{"a": "emoji \U0001f600 pair", "b": "\U00010000"}], + "cjk_and_hangul": [{"a": "中文한글퟿"}], + "nul_and_controls": [{"a": "a\x00b\x1f", "b": "
"}], + "lone_high": [{"a": "a\ud800b"}], + "lone_low": [{"a": "\udc00"}], + "two_lone_halves": [{"a": "𐀀"}], + "literal_backslash_u": [{"a": "\\ud800", "b": "\\u0041"}], + "digits_in_text": [{"a": "9999999999999999999", "b": "1234567890123456789012"}], + "nan": [{"a": math.nan}], + "inf": [{"a": -math.inf}], + "subnormal": [{"a": 5e-324}], + "small_floats": [{"a": 0.00012345678901234568, "b": 1.2345678901234567e19}], + "int_key": [{1: "a"}], + "bool_key": [{True: "a"}], + "none_key": [{None: "a"}], + "list_ok": [{"a": [1, "x", None, [2.5]]}], + "list_big": [{"a": [1, 2**70]}], + "list_nan": [{"a": [1.0, math.nan]}], + "list_lone": [{"a": ["\ud800"]}], + "list_datetime": [{"a": [dt.datetime(2026, 1, 2)]}], + "dict_ok": [{"a": {"b": [1], "c": "d"}}], + "dict_int_key": [{"a": {1: 2}}], + "dict_tag_key": [{"a": {tc._TAG: 1}}], + "tuple_value": [{"a": (1, 2)}], + "datetime": [{"a": dt.datetime(2026, 1, 2, 3, 4, 5, 6)}], + "aware_datetime": [ + {"a": dt.datetime(2026, 1, 2, tzinfo=dt.timezone(dt.timedelta(hours=9)))} + ], + "date": [{"a": dt.date(2026, 1, 2)}], + "decimal": [{"a": decimal.Decimal("1.50")}], + "decimal_nan": [{"a": decimal.Decimal("NaN")}], + "bytes": [{"a": b"\x00\xffab", "b": bytearray(b"x")}], + "numpy_scalars": [ + { + "a": np.int64(5), + "b": np.float64(1.5), + "c": np.bool_(True), + "d": np.float32(2.5), + } + ], + "numpy_big": [{"a": np.uint64(2**64 - 1)}], + "numpy_nan": [{"a": np.float64(math.nan)}], + "numpy_datetime": [{"a": np.datetime64("2026-01-02T03:04:05")}], + "numpy_str": [{"a": np.str_("q")}], + "int_enum": [{"a": Colour.RED}], + "str_subclass": [{"a": Name("s")}], + "str_subclass_lone": [{"a": Name("\ud800")}], + "set_value": [{"a": {1, 2}}], + "object_value": [{"a": object()}], + "none_row": [None], + "list_row": [[1, 2]], + "string_row": ["ab"], + "mixed_good_then_bad": [{"a": 1}, {"a": 2**64}, {"a": math.nan}], +} + + +@pytest.mark.parametrize("tagged", [False, True], ids=["untagged", "tagged"]) +@pytest.mark.parametrize("name", sorted(CASES)) +def test_same_text_or_same_error_as_the_validation_loop(name, tagged): + rows = CASES[name] + new = tc.json_bulk_dumps_tagged if tagged else tc.json_bulk_dumps + assert _outcome(new, rows) == _outcome(lambda r: _old_dumps(r, tagged), rows) + + +@pytest.mark.parametrize("tagged", [False, True], ids=["untagged", "tagged"]) +def test_random_rows_agree_with_the_validation_loop(tagged): + atoms = [ + None, True, 0, -1, INT64_MAX, INT64_MIN, INT64_MAX + 1, 10**30, 1.5, 0.00012345678901234568, + 5e-324, math.nan, math.inf, "", "abc", "a\x00b", "é中", "\U0001f600", "a\ud800b", "\udc00", + "𐀀", "9999999999999999999", [1, 2], [math.nan], {"a": 1}, {1: 2}, (1, 2), + dt.datetime(2026, 1, 2), dt.date(2026, 1, 2), decimal.Decimal("1.5"), b"ab", np.int64(7), + np.float64(math.nan), Colour.RED, Name("s"), {1, 2}, object(), + ] # fmt: skip + rnd = random.Random(318) + new = tc.json_bulk_dumps_tagged if tagged else tc.json_bulk_dumps + for _ in range(3000): + rows = [] + for _ in range(rnd.choice([1, 2, 3, 8])): + keys = rnd.sample(["a", "b", "c", "d"], rnd.randint(0, 4)) + if rnd.random() < 0.03: + keys.append(1) + rows.append({k: rnd.choice(atoms) for k in keys}) + assert _outcome(new, rows) == _outcome(lambda r: _old_dumps(r, tagged), rows) + + +def _batch(n, bad): + """n scalar rows of the shape of a TPC-H part table; `bad` maps a row index to its replacement row.""" + rows = [ + { + "p_partkey": i, + "p_name": f"part {i}", + "p_size": i % 50, + "p_retailprice": 900.0 + (i % 1000) / 100, + "p_comment": None if i % 7 == 0 else "carefully final deposits", + } + for i in range(n) + ] + for index, row in bad.items(): + rows[index] = row + return rows + + +# kind -> (first bad row, a later bad row of another value), so the message tells which row raised +BAD_KINDS = { + "int_beyond_64_bits": ({"p_size": 2**63 + 11}, {"p_size": 2**70 + 22}), + "negative_beyond_64_bits": ({"p_size": INT64_MIN - 33}, {"p_size": 2**64 + 44}), + "lone_surrogate": ({"p_name": "a\ud800"}, {"p_size": 2**65}), + "non_str_key": ({7: "x"}, {"p_size": 2**65}), + "nested_int": ({"p_name": [1, 2**66]}, {"p_size": 2**65 + 55}), + "nan": ({"p_retailprice": math.nan}, {"p_size": 2**65}), + "subnormal": ({"p_retailprice": 5e-324}, {"p_size": 2**65}), +} + + +@pytest.mark.parametrize("tagged", [False, True], ids=["untagged", "tagged"]) +@pytest.mark.parametrize("kind", sorted(BAD_KINDS)) +def test_the_first_bad_row_of_a_large_batch_gives_the_old_error(kind, tagged): + first, later = BAD_KINDS[kind] + rows = _batch(20_000, {12_345: first, 19_000: later}) + new = tc.json_bulk_dumps_tagged if tagged else tc.json_bulk_dumps + expected = _outcome(lambda r: _old_dumps(r, tagged), rows) + assert expected[0] in ("ValueError", "TypeError"), expected + assert _outcome(new, rows) == expected + # the later row alone raises a message of its own. A NaN or a subnormal is refused by json.dumps, after the old loop had walked + # every row, so the old error there is the later row's; for every other kind the first bad row's error comes first + only_later = _batch(20_000, {19_000: later}) + assert _outcome(new, only_later) == _outcome( + lambda r: _old_dumps(r, tagged), only_later + ) + assert (_outcome(new, only_later) == expected) == (kind in ("nan", "subnormal")) + + +@pytest.mark.parametrize("tagged", [False, True], ids=["untagged", "tagged"]) +def test_a_clean_large_batch_is_the_plain_json_text(tagged): + rows = _batch(20_000, {}) + new = tc.json_bulk_dumps_tagged if tagged else tc.json_bulk_dumps + assert new(rows) == json.dumps(rows) + + +def test_scalar_rows_are_not_walked_when_the_text_carries_them(monkeypatch): + # the insert_many shape of the paper's l1tpc Part load and the create_vertices shape of l2, e2 and l3d: dict rows of str, int, float, + # bool and None. Numbers that look alike but fit: 19-digit ints below 2**63, floats whose text has a long run of digits after the + # point, text with digits, Hangul and CJK (BMP characters that json.dumps writes as \uXXXX, some of them as \ud...). + walks = [] + real = tc._check_rows + monkeypatch.setattr( + tc, + "_check_rows", + lambda rows, tagged: walks.append(tagged) or real(rows, tagged), + ) + rows = _batch(2_000, {}) + rows += [ + { + "ts": 1_700_000_000_123_456_789, + "hi": INT64_MAX, + "lo": INT64_MIN + 1, + "f": 0.00012345678901234568, + }, + { + "f": 1.2345678901234567e19, + "s": "id-123456789012345678", + "han": "한글퟿中文", + "nul": "a\x00b", + }, + {"e": "", "n": None, "b": True, "neg": -123456789012345678}, + ] + assert tc.json_bulk_dumps(rows) == json.dumps(rows) + assert tc.json_bulk_dumps_tagged(rows) == json.dumps(rows) + assert walks == [] + + +def test_tagged_kinds_are_not_walked_either(monkeypatch): + walks = [] + real = tc._check_rows + monkeypatch.setattr( + tc, + "_check_rows", + lambda rows, tagged: walks.append(tagged) or real(rows, tagged), + ) + rows = [ + {"d": dt.datetime(2026, 1, 2, 3, 4, 5), "day": dt.date(2026, 1, 2), "m": decimal.Decimal("1.50"), "b": b"ab", "n": np.int64(5), "f": np.float32(2.5)} + for _ in range(100) + ] # fmt: skip + assert tc.json_bulk_dumps_tagged(rows) + assert walks == [] + + +@pytest.mark.parametrize( + "row", + [ + {"a": 2**63}, + {"a": 10**20}, + {"a": INT64_MIN - 1}, + {"a": "a\ud800"}, + {"a": "\U0001f600"}, + {7: "x"}, + {"a": [1]}, + {"a": 5e-324}, + {"a": Colour.RED}, + ], +) +def test_a_value_that_may_be_bad_runs_the_walk(monkeypatch, row): + walks = [] + real = tc._check_rows + monkeypatch.setattr( + tc, + "_check_rows", + lambda rows, tagged: walks.append(tagged) or real(rows, tagged), + ) + _outcome(tc.json_bulk_dumps, [{"id": 1}, row]) + assert walks, "the walk must decide a row the text cannot vouch for" + + +def test_int64_overflow_pattern_matches_exactly_the_tokens_from_2_to_the_63(): + rnd = random.Random(63) + values = [ + 0, + 1, + 10**18, + 2**63 - 2, + 2**63 - 1, + 2**63, + 2**63 + 1, + 10**19 - 1, + 10**19, + 10**20, + 9 * 10**18, + 9_999_999_999_999_999_999, + ] + values += [rnd.randint(10**18, 2**65) for _ in range(5000)] # fmt: skip + for value in values: + for text in (str(value), str(-value), f"[{value},1]", f'{{"a":{value}}}'): + hit = tc._INT64_OVERFLOW.search(text) is not None + assert hit == (value >= 2**63), text + # digits after the point of a float and inside an exponent are no integer token + assert tc._INT64_OVERFLOW.search("[0.00012345678901234568]") is None + assert tc._INT64_OVERFLOW.search("[1.2345678901234567e+19]") is None + + +# --------------------------------------------------------------------------- +# the entry points, on whichever wheel is installed + + +def _types(db): + db.command("sql", "CREATE DOCUMENT TYPE Part") + db.command("sql", "CREATE VERTEX TYPE PartV") + + +def _stored(db, type_name): + rows = db.query("sql", f"SELECT FROM {type_name}").to_list() + return sorted( + ( + r["p_partkey"], + r["p_name"], + r["p_size"], + r["p_retailprice"], + r.get("p_comment"), + ) + for r in rows + ) + + +def _expected(rows): + return sorted( + ( + r["p_partkey"], + r["p_name"], + r["p_size"], + r["p_retailprice"], + r.get("p_comment"), + ) + for r in rows + ) + + +def _rows_with_text(): + rows = _batch(25_000, {}) + rows[10]["p_name"] = "emoji \U0001f600 \U00010000" + rows[11]["p_name"] = "" + rows[12]["p_name"] = "한글 中文 ퟿" + rows[13]["p_size"] = INT64_MAX + rows[14]["p_size"] = INT64_MIN + return rows + + +def test_insert_many_stores_a_scalar_batch_as_before(temp_db_path): + rows = _rows_with_text() + with arcadedb.create_database(temp_db_path) as db: + _types(db) + assert db.insert_many("Part", rows) == len(rows) + assert _stored(db, "Part") == _expected(rows) + + +def test_create_vertices_stores_a_scalar_batch_as_before(temp_db_path): + rows = _rows_with_text() + with arcadedb.create_database(temp_db_path) as db: + _types(db) + with db.graph_batch(parallel_flush=False) as batch: + assert len(batch.create_vertices("PartV", rows)) == len(rows) + assert _stored(db, "PartV") == _expected(rows) + + +@pytest.mark.parametrize("entry", ["insert_many", "create_vertices"]) +@pytest.mark.parametrize("bad", [2**63 + 11, "a\ud800"], ids=["int", "surrogate"]) +def test_a_bad_row_inside_a_large_batch_is_refused(temp_db_path, entry, bad): + column = "p_size" if isinstance(bad, int) else "p_name" + rows = _batch(25_000, {12_345: {column: bad}, 19_000: {"p_size": 2**70 + 22}}) + with arcadedb.create_database(temp_db_path) as db: + _types(db) + with pytest.raises( + Exception + ): # noqa: B017 - the entry points raise their own types + if entry == "insert_many": + db.insert_many("Part", rows) + else: + with db.graph_batch(parallel_flush=False) as batch: + batch.create_vertices("PartV", rows) diff --git a/tests/test_graph_batch_single_pass.py b/tests/test_graph_batch_single_pass.py new file mode 100644 index 00000000..2415c012 --- /dev/null +++ b/tests/test_graph_batch_single_pass.py @@ -0,0 +1,84 @@ +"""GraphBatch.create_vertices (the JVM wheel) decides its rows by their types and writes each chunk once. + +Before, `_create_vertices_json_bulk` called `json_bulk_scalar_ok` on every value of every row before the first chunk was written, and +the chunk was then walked again by json.dumps. Now `type_conversion.classify_vertex_rows` takes a census of the types in C, and the +value checks come from the one json.dumps pass of each chunk (`json_bulk_dumps`, as `json_bulk_dumps_tagged` already did for the tagged +text). What stays: the whole input goes the property-matrix way for a property name that is not a str or a type the text does not carry +(`_create_vertices_json_bulk` returns None); what changes: a scalar the text would change (a subnormal, a NaN, an integer beyond 64 bits, +a lone surrogate) sends its own chunk, not the whole input, through the matrix path, as it already did for a tagged chunk. +""" + +import pytest + +import arcadedb +from arcadedb import graph_batch as gbm +from arcadedb import type_conversion as tc +from arcadedb.graph_batch import GraphBatch + + +def _rows(n): + return [ + {"id": i, "name": f"n{i}", "score": i * 0.5, "flag": i % 2 == 0, "note": None} + for i in range(n) + ] + + +def test_scalar_rows_are_not_judged_value_by_value(temp_db_path, monkeypatch): + def refuse(value): + raise AssertionError("a per-value gate ran over a scalar table") + + monkeypatch.setattr(tc, "json_bulk_scalar_ok", refuse) + monkeypatch.setattr(gbm, "json_bulk_scalar_ok", refuse) + monkeypatch.setattr(tc, "json_bulk_shape_ok", refuse) + monkeypatch.setattr(gbm, "json_bulk_shape_ok", refuse) + rows = _rows(2_000) + with arcadedb.create_database(temp_db_path) as db: + db.command("sql", "CREATE VERTEX TYPE V") + with db.graph_batch(parallel_flush=False) as batch: + assert len(batch._create_vertices_json_bulk("V", rows)) == len(rows) + count = db.query("sql", "SELECT count(*) AS c FROM V").first().get("c") + assert count == len(rows) + + +def test_the_whole_input_still_goes_the_matrix_way_for_a_name_or_type_the_text_does_not_carry( + temp_db_path, +): + rows = _rows(5) + with arcadedb.create_database(temp_db_path) as db: + db.command("sql", "CREATE VERTEX TYPE V") + with db.graph_batch(parallel_flush=False) as batch: + assert len(batch._create_vertices_json_bulk("V", rows)) == len(rows) + assert batch._create_vertices_json_bulk("V", rows + [{1: "a"}]) is None + assert batch._create_vertices_json_bulk("V", rows + [{"k": {1, 2}}]) is None + + +@pytest.mark.parametrize("value", [5e-324, float("nan"), 2**70, "a\ud800b"]) +def test_a_scalar_the_text_changes_sends_its_chunk_through_the_matrix_path( + temp_db_path, monkeypatch, value +): + monkeypatch.setattr(GraphBatch, "_BULK_CHUNK", 100) + calls = [] + real = GraphBatch._to_java_property_matrix + + def spy(self, chunk): + calls.append(len(chunk)) + return real(self, chunk) + + monkeypatch.setattr(GraphBatch, "_to_java_property_matrix", spy) + rows = [{"id": i, "x": 1.5} for i in range(250)] + rows[150] = {"id": 150, "x": value} + with arcadedb.create_database(temp_db_path) as db: + db.command("sql", "CREATE VERTEX TYPE V") + with db.graph_batch(parallel_flush=False) as batch: + try: + rids = batch.create_vertices("V", rows) + except ( + Exception + ): # noqa: BLE001 - the matrix path refuses a value that no entry point stores + rids = None + assert calls == [100], "only the chunk of row 150 takes the matrix path" + if value == 5e-324: + # stored as the double every other entry point stores; the text of a subnormal was read as a DECIMAL + assert len(rids) == 250 + stored = db.query("sql", "SELECT x FROM V WHERE id = 150").first().get("x") + assert stored == 5e-324 and type(stored) is float diff --git a/tests/test_graph_batch_tagged_shapes.py b/tests/test_graph_batch_tagged_shapes.py index 2da99c18..560b2d98 100644 --- a/tests/test_graph_batch_tagged_shapes.py +++ b/tests/test_graph_batch_tagged_shapes.py @@ -438,6 +438,8 @@ def shape(*a, **k): monkeypatch.setattr(gbm, "json_bulk_dumps_tagged", dumps) monkeypatch.setattr(gbm, "json_bulk_shape_ok", shape) + # create_vertices decides the shapes in type_conversion.classify_vertex_rows, which calls the gate there + monkeypatch.setattr(type_conversion, "json_bulk_shape_ok", shape) return calls From 68bc0a9a113ac5e3e71f556933185cf0a8d2bfe4 Mon Sep 17 00:00:00 2001 From: Taewoon Kim Date: Mon, 9 Nov 2026 18:58:29 +0900 Subject: [PATCH 08/49] Native: column frames for to_arrow, to_columns, to_dataframe and to_polars, read by the decoder the JVM wheel uses (refs #312) The native library answers a read of a cursor as one column frame of about 1 MiB (arcadedb_n_next_col, ColumnFrames.java), in the layout that ColumnBatcher writes for the JVM wheel: int64, float64, bool bytes, epoch milliseconds or microseconds, float32 vectors, Arrow strings (int32 offsets and a UTF-8 blob), DECIMAL as text, and a tagged JSON column for what no fixed layout carries without changing a value (nested values, records, a mixed column, an integer beyond 2**53 next to a float). arcadedb._columnar holds the one decoder and the merge of frames that both wheels call; results.py keeps only its JPype transport and the text of its json columns. The first reply of a query is still JSON for the row APIs, and the cursor keeps those rows (at most 256, until its next read by row) so that a columnar read replays them into its first frame. A result that the first reply held whole has no cursor: its dicts go through the old rules once, as one more chunk. The old JSON read path stays for to_list, to_json_list and row iteration in this commit. Tests: tests/test_column_frames.py (the decoder on hand-built frames, and the stored table through every columnar API, both wheels), tests/test_native_column_frames_reference.py (the old route copied into the test as the reference, native only). Co-Authored-By: Claude Sonnet 5.5 --- .../org/humemai/nativelib/ArcadeNative.java | 86 +++- .../org/humemai/nativelib/ColumnFrames.java | 382 +++++++++++++++ .../org/humemai/nativelib/NativeCodec.java | 7 + src/arcadedb/_columnar.py | 397 ++++++++++++++++ src/arcadedb/_native/colframes.py | 130 ++++++ src/arcadedb/_native/database.py | 96 +++- src/arcadedb/_native/lib.py | 27 ++ src/arcadedb/results.py | 408 ++-------------- tests/conftest.py | 2 + tests/test_column_frames.py | 338 ++++++++++++++ tests/test_native_column_frames_reference.py | 434 ++++++++++++++++++ 11 files changed, 1908 insertions(+), 399 deletions(-) create mode 100644 native/src/main/java/org/humemai/nativelib/ColumnFrames.java create mode 100644 src/arcadedb/_columnar.py create mode 100644 src/arcadedb/_native/colframes.py create mode 100644 tests/test_column_frames.py create mode 100644 tests/test_native_column_frames_reference.py diff --git a/native/src/main/java/org/humemai/nativelib/ArcadeNative.java b/native/src/main/java/org/humemai/nativelib/ArcadeNative.java index 9b5acb91..52f9d3c5 100644 --- a/native/src/main/java/org/humemai/nativelib/ArcadeNative.java +++ b/native/src/main/java/org/humemai/nativelib/ArcadeNative.java @@ -20,6 +20,7 @@ import com.arcadedb.engine.ComponentFile; import com.arcadedb.exception.DuplicatedKeyException; import com.arcadedb.exception.NeedRetryException; +import com.arcadedb.query.sql.executor.Result; import com.arcadedb.query.sql.executor.ResultSet; import com.arcadedb.schema.VertexType; import com.arcadedb.server.ArcadeDBServer; @@ -36,6 +37,7 @@ import java.nio.ByteBuffer; import java.nio.ByteOrder; import java.nio.charset.StandardCharsets; +import java.util.ArrayList; import java.util.List; import java.util.Map; import java.util.concurrent.ConcurrentHashMap; @@ -45,6 +47,17 @@ public final class ArcadeNative { static final Map DBS = new ConcurrentHashMap<>(); private static final Map RESULT_SETS = new ConcurrentHashMap<>(); + /** + * The rows of the first reply of an open cursor, kept until the cursor is read again or closed: that reply is JSON for the row APIs, + * and a columnar read (arcadedb_n_next_col) that starts from it replays these rows into its first frame instead of losing them. + * At most the first reply (256 rows) of a cursor that has more rows. + */ + private static final Map> HEADS = new ConcurrentHashMap<>(); + + private static ResultSet removeCursor(final long id) { + HEADS.remove(id); + return RESULT_SETS.remove(id); + } private static final Map SERVERS = new ConcurrentHashMap<>(); static final AtomicLong IDS = new AtomicLong(1); @@ -288,9 +301,10 @@ public static CCharPointer next(final IsolateThread thread, final long rsId, fin rs = RESULT_SETS.get(rsId); if (rs == null) throw new IllegalArgumentException("unknown result set " + rsId); - return out(batchOf(rsId, rs, max)); + HEADS.remove(rsId); + return out(batchOf(rsId, rs, max, null)); } catch (final Throwable t) { - RESULT_SETS.remove(rsId); + removeCursor(rsId); closeQuietly(rs); return err(t); } @@ -309,17 +323,65 @@ public static CCharPointer nextBin(final IsolateThread thread, final long rsId, if (rs == null) throw new IllegalArgumentException("unknown result set " + rsId); final ResultSet cursor = rs; - return withFloatTail(() -> batchOf(rsId, cursor, max)); + HEADS.remove(rsId); + return withFloatTail(() -> batchOf(rsId, cursor, max, null)); } catch (final Throwable t) { - RESULT_SETS.remove(rsId); + removeCursor(rsId); closeQuietly(rs); return err(t); } } + /** + * The next rows of an open cursor as one column frame (see ColumnFrames), written as an int64 LE byte count and the frame; a failure is + * the negated byte count of the error JSON and that JSON. max: rows at most (a frame also ends at about 1 MiB of values). columns: a + * JSON array of the column names to read, or "" for the union of the names of the rows. flags bit 0: datetimes as microseconds (the + * to_dataframe form). skip: of the rows that the first reply of this cursor already held (HEADS), the number the caller has taken; the + * first frame starts after them. The frame that ends the rows closes the cursor. The reply is freed with arcadedb_free. + */ + @CEntryPoint(name = "arcadedb_n_next_col") + @SuppressWarnings("unchecked") + public static CCharPointer nextCol(final IsolateThread thread, final long rsId, final int max, final CCharPointer columns, + final int flags, final int skip) { + ResultSet rs = null; + try { + rs = RESULT_SETS.get(rsId); + if (rs == null) + throw new IllegalArgumentException("unknown result set " + rsId); + final List head = HEADS.remove(rsId); + String[] pinned = null; + final String spec = str(columns); + if (spec != null && !spec.isEmpty()) { + final List names = (List) Json.parse(spec); + pinned = new String[names.size()]; + for (int i = 0; i < pinned.length; i++) + pinned[i] = (String) names.get(i); + } + final ColumnFrames.Frame frame = ColumnFrames.next(rs, head, skip, max, pinned, (flags & 1) != 0); + if (frame.done) { + removeCursor(rsId); + closeQuietly(rs); + } + return frameOut(frame.bytes.length, frame.bytes); + } catch (final Throwable t) { + removeCursor(rsId); + closeQuietly(rs); + final byte[] e = errorJson(t).getBytes(StandardCharsets.UTF_8); + return frameOut(-e.length, e); + } + } + + private static CCharPointer frameOut(final int length, final byte[] b) { + final CCharPointer p = UnmanagedMemory.malloc(8 + b.length); + final ByteBuffer bb = CTypeConversion.asByteBuffer(p, 8 + b.length).order(ByteOrder.LITTLE_ENDIAN); + bb.putLong(length); + bb.put(b, 0, b.length); + return p; + } + @CEntryPoint(name = "arcadedb_n_rs_close") public static CCharPointer rsClose(final IsolateThread thread, final long rsId) { - closeQuietly(RESULT_SETS.remove(rsId)); + closeQuietly(removeCursor(rsId)); return OK(); } @@ -328,16 +390,20 @@ private static String batch(final ResultSet rs, final int max) { return "{\"ok\":true,\"rs\":0,\"rows\":[],\"meta\":[],\"drained\":true}"; final long id = IDS.getAndIncrement(); RESULT_SETS.put(id, rs); - return batchOf(id, rs, max); + final List keep = new ArrayList<>(); + final String reply = batchOf(id, rs, max, keep); + if (RESULT_SETS.containsKey(id)) + HEADS.put(id, keep); + return reply; } - private static String batchOf(final long id, final ResultSet rs, final int max) { + private static String batchOf(final long id, final ResultSet rs, final int max, final List keep) { final StringBuilder sb = new StringBuilder("{\"ok\":true,\"rs\":"); final int mark = sb.length(); sb.append(id).append(','); - final boolean drained = NativeCodec.appendBatch(sb, rs, max); + final boolean drained = NativeCodec.appendBatch(sb, rs, max, keep); if (drained) { - RESULT_SETS.remove(id); + removeCursor(id); closeQuietly(rs); sb.replace(mark, mark + String.valueOf(id).length(), "0"); } @@ -650,7 +716,7 @@ public static CCharPointer lastError(final IsolateThread thread) { public static void rsCloseMany(final IsolateThread thread, final CLongPointer ids, final int n) { try { for (int i = 0; i < n; i++) - closeQuietly(RESULT_SETS.remove(ids.read(i))); + closeQuietly(removeCursor(ids.read(i))); } catch (final Throwable ignored) { // closing is best effort, as in arcadedb_n_rs_close } diff --git a/native/src/main/java/org/humemai/nativelib/ColumnFrames.java b/native/src/main/java/org/humemai/nativelib/ColumnFrames.java new file mode 100644 index 00000000..0c507346 --- /dev/null +++ b/native/src/main/java/org/humemai/nativelib/ColumnFrames.java @@ -0,0 +1,382 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * Result rows as one column frame for the Python native backend (arcadedb_n_next_col): the layout the JVM wheel's ColumnBatcher + * writes, so one Python decoder (arcadedb/_columnar.py) reads both. A frame is + * + * int32 LE header length, header JSON, then for each column its null bitmap and its buffer + * + * The header is {"count":n,"done":bool,"cols":[{"name":..,"type":..,"nulls":bytes,"bytes":bytes[,"dim":d]},..]} ("done" is an addition + * to ColumnBatcher's header, which the decoder ignores). A null bitmap has bit i of byte i/8 set for a null row. A column type is + * + * i8 int64 LE (Long, Integer, Short, Byte) f8 float64 LE (Double, Float; a Float is its exact double) + * b1 one byte per row dt int64 epoch milliseconds, dtu: epoch microseconds (see micros) + * f4v dim float32 LE per row (float[]) f8v dim float64 LE per row (double[]) + * str int32 LE offsets (n + 1) then UTF-8 bytes dec a BigDecimal as text in the str layout + * json one JSON array of the column in the tagged encoding of NativeCodec: what no fixed layout carries without changing a value + * (nested lists and maps, records and RIDs, BigInteger, a column that mixes kinds, a DATE or a time-zone-aware temporal next to + * datetimes with micros, a ragged vector, a Long beyond 2**53 next to a floating value) + * + * A frame holds at most {@code max} rows and stops once the rows read add up to about FRAME_BYTES, so the working set of a read is a frame + * and not the table. + */ +package org.humemai.nativelib; + +import com.arcadedb.query.sql.executor.Result; +import com.arcadedb.query.sql.executor.ResultSet; + +import java.io.ByteArrayOutputStream; +import java.math.BigDecimal; +import java.nio.ByteBuffer; +import java.nio.ByteOrder; +import java.nio.charset.StandardCharsets; +import java.time.Instant; +import java.time.LocalDate; +import java.time.LocalDateTime; +import java.time.OffsetDateTime; +import java.time.ZoneOffset; +import java.time.ZonedDateTime; +import java.util.ArrayList; +import java.util.Calendar; +import java.util.Collection; +import java.util.Date; +import java.util.LinkedHashSet; +import java.util.List; +import java.util.Map; +import java.util.Set; + +final class ColumnFrames { + /** The rows of a frame add up to about this many bytes of values. */ + static final long FRAME_BYTES = 1L << 20; + private static final long TWO53 = 1L << 53; + + private ColumnFrames() { + } + + /** One frame and whether the rows are over (the cursor was read to its end). */ + static final class Frame { + final byte[] bytes; + final boolean done; + + Frame(final byte[] bytes, final boolean done) { + this.bytes = bytes; + this.done = done; + } + } + + private static long toEpochMillis(final Object v) { + if (v instanceof Date d) + return d.getTime(); + if (v instanceof Calendar c) + return c.getTimeInMillis(); + if (v instanceof LocalDateTime ldt) + return ldt.toInstant(ZoneOffset.UTC).toEpochMilli(); + if (v instanceof LocalDate ld) + return ld.atStartOfDay().toInstant(ZoneOffset.UTC).toEpochMilli(); + if (v instanceof Instant i) + return i.toEpochMilli(); + if (v instanceof ZonedDateTime z) + return z.toInstant().toEpochMilli(); + if (v instanceof OffsetDateTime o) + return o.toInstant().toEpochMilli(); + throw new IllegalArgumentException("not a temporal: " + v.getClass()); + } + + private static long toEpochMicros(final Object v) { + if (v instanceof LocalDateTime ldt) + return Math.addExact(Math.multiplyExact(ldt.toEpochSecond(ZoneOffset.UTC), 1_000_000L), ldt.getNano() / 1000); + if (v instanceof Date d) + return Math.multiplyExact(d.getTime(), 1000L); + if (v instanceof Calendar c) + return Math.multiplyExact(c.getTimeInMillis(), 1000L); + throw new IllegalArgumentException("not a naive datetime: " + v.getClass()); + } + + private static boolean yearFits(final LocalDateTime ldt) { + return ldt.getYear() >= 1 && ldt.getYear() <= 9999; + } + + /** About how many bytes the values of a row take in a frame (a stop rule, not an exact size). */ + private static long estimate(final Result row) { + long n = 0; + for (final String name : row.getPropertyNames()) { + final Object v = row.getProperty(name); + if (v == null) + n += 1; + else if (v instanceof String s) + n += s.length() + 4; + else if (v instanceof float[] a) + n += 4L * a.length; + else if (v instanceof double[] a) + n += 8L * a.length; + else if (v instanceof Number || v instanceof Boolean || v instanceof Date || v instanceof LocalDate || v instanceof LocalDateTime + || v instanceof Instant) + n += 8; + else if (v instanceof Collection c) + n += 16 + 16L * c.size(); + else if (v instanceof Map m) + n += 16 + 32L * m.size(); + else + n += 32; + } + return n; + } + + /** + * The next frame. {@code head} holds rows that were already read from the cursor when its first reply was encoded as JSON (the + * frame starts at {@code head[skip]}, then goes on with the cursor), or is null. {@code pinned} is the column set, or null to take + * the union of the names of the rows of this frame, in order of first appearance. With {@code micros} a naive datetime column is + * epoch microseconds ("dtu") and a column with a DATE, a time-zone-aware temporal or a year Python cannot hold is "json". + */ + static Frame next(final ResultSet rs, final List head, final int skip, final int max, final String[] pinned, + final boolean micros) { + final List results = new ArrayList<>(Math.min(Math.max(max, 1), 1 << 14)); + int h = Math.max(skip, 0); + long budget = 0; + boolean done = false; + while (results.size() < max && budget < FRAME_BYTES) { + final Result row; + if (head != null && h < head.size()) + row = head.get(h++); + else if (rs.hasNext()) + row = rs.next(); + else { + done = true; + break; + } + results.add(row); + budget += estimate(row); + } + if (!done) + done = (head == null || h >= head.size()) && !rs.hasNext(); + + String[] columns = pinned; + if (columns == null) { + final Set names = new LinkedHashSet<>(); + String[] previous = null; + for (final Result row : results) { + final Set rowNames = row.getPropertyNames(); + if (previous != null && rowNames.size() == previous.length) { + int i = 0; + for (final String name : rowNames) + if (!name.equals(previous[i++])) { + i = -1; + break; + } + if (i >= 0) + continue; + } + names.addAll(rowNames); + previous = rowNames.toArray(new String[0]); + } + columns = names.toArray(new String[0]); + } + final int n = columns.length; + final int count = results.size(); + final List rows = new ArrayList<>(count); + for (final Result row : results) { + final Object[] vals = new Object[n]; + for (int c = 0; c < n; c++) + vals[c] = row.getProperty(columns[c]); + rows.add(vals); + } + + final StringBuilder hdr = new StringBuilder("{\"count\":").append(count).append(",\"done\":").append(done).append(",\"cols\":["); + final ByteArrayOutputStream body = new ByteArrayOutputStream(Math.max(count * n * 8, 16)); + + for (int c = 0; c < n; c++) { + final byte[] nulls = new byte[(count + 7) / 8]; + String type = null; + boolean sawInt = false, sawFloating = false, bigInt = false; + for (int r = 0; r < count; r++) { + final Object v = rows.get(r)[c]; + if (v == null) + continue; + final String t; + if (v instanceof Long || v instanceof Integer || v instanceof Short || v instanceof Byte) { + t = "i8"; + sawInt = true; + final long l = ((Number) v).longValue(); + if (l > TWO53 || l < -TWO53) + bigInt = true; + } else if (v instanceof Double || v instanceof Float) { + t = "f8"; + sawFloating = true; + } else if (v instanceof Boolean) + t = "b1"; + else if (v instanceof Date || v instanceof Calendar || v instanceof LocalDateTime || v instanceof LocalDate + || v instanceof Instant || v instanceof ZonedDateTime || v instanceof OffsetDateTime) { + if (!micros) + t = "dt"; + else if (v instanceof LocalDateTime ldt ? yearFits(ldt) : v instanceof Date || v instanceof Calendar) + t = "dtu"; + else + t = "json"; + } else if (v instanceof String || v instanceof Character) + t = "str"; + else if (v instanceof BigDecimal) + t = "dec"; + else if (v instanceof float[]) + t = "f4v"; + else if (v instanceof double[]) + t = "f8v"; + else + t = "json"; + if (type == null) + type = t; + else if (!type.equals(t)) { + if ((type.equals("i8") && t.equals("f8")) || (type.equals("f8") && t.equals("i8"))) + type = "f8"; + else { + type = "json"; + break; + } + } + } + if (type == null) + type = "str"; // an all-null column + // an integer beyond 2**53 next to a floating value has no exact double: the values stay what they are, in a json column + if (type.equals("f8") && sawInt && sawFloating && bigInt) + type = "json"; + + int vdim = -1; + if (type.equals("f4v") || type.equals("f8v")) { + for (int r = 0; r < count && vdim != -2; r++) { + final Object v = rows.get(r)[c]; + if (v == null) + continue; + final int len = type.equals("f4v") ? ((float[]) v).length : ((double[]) v).length; + if (vdim == -1) + vdim = len; + else if (vdim != len) + vdim = -2; + } + if (vdim < 0 && vdim != -1) + type = "json"; // ragged + else if (vdim == -1) + vdim = 0; + } + + final byte[] colBuf; + if (type.equals("i8") || type.equals("dt") || type.equals("dtu")) { + final ByteBuffer bb = ByteBuffer.allocate(count * 8).order(ByteOrder.LITTLE_ENDIAN); + for (int r = 0; r < count; r++) { + final Object v = rows.get(r)[c]; + if (v == null) { + nulls[r >> 3] |= (1 << (r & 7)); + bb.putLong(0); + } else if (type.equals("dt")) + bb.putLong(toEpochMillis(v)); + else if (type.equals("dtu")) + bb.putLong(toEpochMicros(v)); + else + bb.putLong(((Number) v).longValue()); + } + colBuf = bb.array(); + } else if (type.equals("f8")) { + final ByteBuffer bb = ByteBuffer.allocate(count * 8).order(ByteOrder.LITTLE_ENDIAN); + for (int r = 0; r < count; r++) { + final Object v = rows.get(r)[c]; + if (v == null) { + nulls[r >> 3] |= (1 << (r & 7)); + bb.putDouble(Double.NaN); + } else + bb.putDouble(((Number) v).doubleValue()); + } + colBuf = bb.array(); + } else if (type.equals("b1")) { + final ByteBuffer bb = ByteBuffer.allocate(count); + for (int r = 0; r < count; r++) { + final Object v = rows.get(r)[c]; + if (v == null) { + nulls[r >> 3] |= (1 << (r & 7)); + bb.put((byte) 0); + } else + bb.put((byte) (((Boolean) v) ? 1 : 0)); + } + colBuf = bb.array(); + } else if (type.equals("f4v")) { + final ByteBuffer bb = ByteBuffer.allocate(count * vdim * 4).order(ByteOrder.LITTLE_ENDIAN); + for (int r = 0; r < count; r++) { + final Object v = rows.get(r)[c]; + if (v == null) { + nulls[r >> 3] |= (1 << (r & 7)); + for (int k = 0; k < vdim; k++) + bb.putFloat(Float.NaN); + } else + for (final float x : (float[]) v) + bb.putFloat(x); + } + colBuf = bb.array(); + } else if (type.equals("f8v")) { + final ByteBuffer bb = ByteBuffer.allocate(count * vdim * 8).order(ByteOrder.LITTLE_ENDIAN); + for (int r = 0; r < count; r++) { + final Object v = rows.get(r)[c]; + if (v == null) { + nulls[r >> 3] |= (1 << (r & 7)); + for (int k = 0; k < vdim; k++) + bb.putDouble(Double.NaN); + } else + for (final double x : (double[]) v) + bb.putDouble(x); + } + colBuf = bb.array(); + } else if (type.equals("json")) { + final StringBuilder arr = new StringBuilder(); + arr.append('['); + for (int r = 0; r < count; r++) { + final Object v = rows.get(r)[c]; + if (r > 0) + arr.append(','); + if (v == null) + nulls[r >> 3] |= (1 << (r & 7)); + NativeCodec.append(arr, v); + } + arr.append(']'); + colBuf = arr.toString().getBytes(StandardCharsets.UTF_8); + } else { // str and dec: int32 offsets (count + 1) then the UTF-8 bytes + final ByteArrayOutputStream chars = new ByteArrayOutputStream(Math.max(count * 16, 16)); + final ByteBuffer offs = ByteBuffer.allocate((count + 1) * 4).order(ByteOrder.LITTLE_ENDIAN); + int pos = 0; + offs.putInt(0); + for (int r = 0; r < count; r++) { + final Object v = rows.get(r)[c]; + if (v == null) + nulls[r >> 3] |= (1 << (r & 7)); + else { + final String text = v instanceof BigDecimal d ? d.toPlainString() : v.toString(); + final byte[] b = text.getBytes(StandardCharsets.UTF_8); + chars.write(b, 0, b.length); + pos += b.length; + } + offs.putInt(pos); + } + final byte[] charBytes = chars.toByteArray(); + final ByteBuffer bb = ByteBuffer.allocate(offs.capacity() + charBytes.length); + bb.put(offs.array()); + bb.put(charBytes); + colBuf = bb.array(); + } + + if (c > 0) + hdr.append(','); + hdr.append("{\"name\":"); + NativeCodec.quote(hdr, columns[c]); + hdr.append(",\"type\":\"").append(type).append("\",\"nulls\":").append(nulls.length).append(",\"bytes\":").append(colBuf.length); + if (vdim > 0 && (type.equals("f4v") || type.equals("f8v"))) + hdr.append(",\"dim\":").append(vdim); + hdr.append('}'); + body.write(nulls, 0, nulls.length); + body.write(colBuf, 0, colBuf.length); + } + hdr.append("]}"); + + final byte[] headerBytes = hdr.toString().getBytes(StandardCharsets.UTF_8); + final byte[] bodyBytes = body.toByteArray(); + final ByteBuffer out = ByteBuffer.allocate(4 + headerBytes.length + bodyBytes.length).order(ByteOrder.LITTLE_ENDIAN); + out.putInt(headerBytes.length); + out.put(headerBytes); + out.put(bodyBytes); + return new Frame(out.array(), done); + } +} diff --git a/native/src/main/java/org/humemai/nativelib/NativeCodec.java b/native/src/main/java/org/humemai/nativelib/NativeCodec.java index 59b7c94c..5d037a50 100644 --- a/native/src/main/java/org/humemai/nativelib/NativeCodec.java +++ b/native/src/main/java/org/humemai/nativelib/NativeCodec.java @@ -52,6 +52,11 @@ private NativeCodec() { * for a projection, or [rid, typeName, kind] for a record row (kind v, e or d). */ static boolean appendBatch(final StringBuilder sb, final ResultSet rs, final int max) { + return appendBatch(sb, rs, max, null); + } + + /** As above; {@code keep}, when not null, receives each row that was encoded (a columnar read replays them, see ArcadeNative.HEADS). */ + static boolean appendBatch(final StringBuilder sb, final ResultSet rs, final int max, final java.util.List keep) { final StringBuilder meta = new StringBuilder("["); sb.append("\"rows\":["); int n = 0; @@ -78,6 +83,8 @@ static boolean appendBatch(final StringBuilder sb, final ResultSet rs, final int throw t; } n++; + if (keep != null) + keep.add(row); if (sb.length() >= MAX_BATCH_CHARS) break; } diff --git a/src/arcadedb/_columnar.py b/src/arcadedb/_columnar.py new file mode 100644 index 00000000..7cb62e36 --- /dev/null +++ b/src/arcadedb/_columnar.py @@ -0,0 +1,397 @@ +"""The column frame of both wheels, and the one decoder that reads it. + +A frame is what ``ColumnBatcher`` (the JVM wheel's bridge jar) writes and what the native library's ``arcadedb_n_next_col`` writes: a +little-endian ``int32`` header length, a JSON header (``count``, ``cols``: name, type, null-bitmap bytes, data bytes and, for a +vector, ``dim``), then for each column its null bitmap (bit i of byte i/8 set means row i is null) and its buffer. A column type is +``i8`` (int64), ``f8`` (float64), ``b1`` (one byte per bool), ``dt`` and ``dtu`` (int64 epoch milliseconds and microseconds), +``f4v`` and ``f8v`` (``dim`` floats per row), ``str`` (int32 offsets, count + 1 of them, then the UTF-8 bytes: Arrow's own string +layout), ``dec`` (a BigDecimal as text in the string layout) and ``json`` (a JSON array of the column, for values no fixed layout +carries: nested lists and maps, RIDs, a DATE next to datetimes, a mixed column). + +The interface and the format are shared, the plumbing is per wheel: what differs is how the bytes get here (JPype or ctypes) and the +text of a ``json`` column (the JVM wheel's tagged JSON with a side array of Java objects, the native library's tagged JSON), so the +decoders take ``json_values(data, side, count)`` from the wheel. Nothing here imports JPype. +""" + +import json +from typing import Any, Callable, Dict, List, Tuple + + +def _cast_all_to_string(arrs: list, pa) -> list: + """ + Cast every array to a string type. + + Tolerates a type pyarrow cannot cast directly (e.g. a vector/list column) + by going through Python objects instead. + """ + unified = [] + for a in arrs: + try: + unified.append(a.cast(pa.string())) + except pa.ArrowException: + unified.append( + pa.array( + [None if v is None else str(v) for v in a.to_pylist()], + type=pa.string(), + ) + ) + return unified + + +def _decode_strings(data, count: int, mask) -> list: + """The strings of one column buffer: int32 offsets, then the UTF-8 blob. + + ``mask`` is the null mask (True = null) or None when the column has no nulls. + """ + import numpy as np + + offs = np.frombuffer(data[: (count + 1) * 4], dtype=" list: + """The DECIMAL values of one column buffer, as ``Decimal`` objects (None for + null). The bridge writes each ``BigDecimal`` in full as text, so no digit is + lost to a double.""" + from decimal import Decimal + + return [ + None if s is None else Decimal(s) for s in _decode_strings(data, count, mask) + ] + + +def _missing_column_part(count: int, template): + """A batch of ``count`` rows that lack a column some other batch has: nulls in + the shape ``to_columns`` uses for that column (NaN for numbers, NaT for + datetimes, None for everything else).""" + import numpy as np + + if isinstance(template, np.ndarray): + if template.ndim == 2: + return np.full((count, template.shape[1]), np.nan, dtype=template.dtype) + if template.dtype.kind in "iuf": + return np.full(count, np.nan) + if template.dtype.kind == "M": + return np.full(count, np.datetime64("NaT"), dtype=template.dtype) + if template.dtype.kind == "O": + return np.full(count, None, dtype=object) + return [None] * count + + +def _is_untyped_arrow_chunk(a, pa) -> bool: + """True for a chunk that says nothing about its column's type: all values + null (a batch whose rows lack the property, or carry it as null), or a list + column holding only empty lists, which Arrow types ``list``.""" + if len(a) == 0 or a.null_count == len(a) or pa.types.is_null(a.type): + return True + return pa.types.is_list(a.type) and pa.types.is_null(a.type.value_type) + + +def _common_decimal_type(types: list, pa): + """One decimal type every given decimal type fits in (decimal128 up to 38 + digits, decimal256 up to 76), or None when it would need more.""" + scale = max(t.scale for t in types) + precision = max(t.precision - t.scale for t in types) + scale + if precision <= 38: + return pa.decimal128(precision, scale) + if precision <= 76: + return pa.decimal256(precision, scale) + return None + + +def _unify_arrow_chunk_types(arrs: list, pa) -> list: + """ + Unify a column's per-batch Arrow arrays onto one Arrow type. + + ArcadeDB is schemaless per document, so a property's Java type can + legally vary row to row; ColumnBatcher infers each batch's column type + independently, so two batches of the same result set can produce + incompatible Arrow types for the same column (#7108) - e.g. int64 in one + batch, float64 in the next. ``pa.chunked_array`` requires every chunk to + share one type, so leaving them as-is raises at concatenation time on a + result set that is otherwise entirely legal. + + Numeric types widen (int64 -> float64, matching to_columns' pandas-style + promotion) when every value survives the cast exactly; pyarrow's cast is + safe by default and raises rather than silently lose precision, which an + int64 outside float64's +-2**53 exact range would (code review on #7108), + so that case - and anything else, including a json or vector column that + varied - degrades to string instead, so the column always ends up with + exactly one type rather than an exception. + + int64/float64 is the only numeric pair worth special-casing: ColumnBatcher + (src/java/.../ColumnBatcher.java) already widens every + Java integer type (Long/Integer/Short/Byte) to the same "i8" wire encoding + and every Java floating type (Double/Float) to "f8" *within* one batch, so + across batches the only numeric type mismatch this ever has to unify is + exactly this pair - there is no int32/int64 or bool/int64 case to widen. + """ + types = {a.type for a in arrs} + if len(types) <= 1: + return arrs + + # A chunk that carries no type information (every value null, or only empty + # lists) takes the type the other chunks agree on, so the column's type does + # not depend on where the batch boundaries fall (#114). + untyped = [_is_untyped_arrow_chunk(a, pa) for a in arrs] + if any(untyped) and not all(untyped): + typed = _unify_arrow_chunk_types( + [a for a, u in zip(arrs, untyped) if not u], pa + ) + target = typed[0].type + filled = iter(typed) + out = [] + for a, u in zip(arrs, untyped): + if not u: + out.append(next(filled)) + elif a.null_count == len(a): + out.append(pa.nulls(len(a), type=target)) + else: + try: + out.append(a.cast(target)) + except pa.ArrowException: + return _cast_all_to_string(arrs, pa) + return out + + if types <= {pa.int64(), pa.float64()}: + try: + return [a.cast(pa.float64()) for a in arrs] + except pa.ArrowException: + pass + + if all(pa.types.is_decimal(t) for t in types): + common = _common_decimal_type(list(types), pa) + if common is not None: + return [a.cast(common) for a in arrs] + + return _cast_all_to_string(arrs, pa) + + +def _null_mask(buf, pos: int, nulls_len: int, count: int): + """``(null_bits, mask)`` of a column: ``mask`` is a bool array (True = null) or None when no row is null.""" + import numpy as np + + null_bits = np.frombuffer(buf[pos : pos + nulls_len], dtype=np.uint8) + if not null_bits.any(): + return null_bits, None + return null_bits, np.unpackbits(null_bits, bitorder="little")[:count].astype(bool) + + +def read_header(buf) -> Tuple[Dict[str, Any], int]: + """``(header, position of the first column)`` of a frame.""" + hlen = int.from_bytes(buf[:4], "little") + return json.loads(bytes(buf[4 : 4 + hlen])), 4 + hlen + + +def decode_columns_frame( + buf, side, json_values: Callable, micros: bool = False +) -> Tuple[int, Dict[str, Any], Dict[str, Any]]: + """``(count, parts, header)`` of one frame as ``to_columns`` columns: ``parts`` maps a column name to a numpy array + (int64, float64, bool, datetime64) or a Python list (text, JSON-typed values, a bool column with nulls); a count of 0 has + no parts. With ``micros`` a naive datetime column is ``datetime64[us]`` (``dtu``). + """ + import numpy as np + + header, pos = read_header(buf) + count = header["count"] + if count == 0: + return 0, {}, header + parts: Dict[str, Any] = {} + for col in header["cols"]: + name, ctype = col["name"], col["type"] + nulls_len = col["nulls"] + _, mask = _null_mask(buf, pos, nulls_len, count) + has_nulls = mask is not None + pos += nulls_len + data = buf[pos : pos + col["bytes"]] + pos += col["bytes"] + + if ctype == "i8": + arr = np.frombuffer(data, dtype=" 2-D array (count, dim); the writer puts NaN rows into null slots; + # an empty vector has no "dim" in the header + dim = col.get("dim", 0) + dt = " Dict[str, Any]: + """The ``to_columns`` dict of the parts of every frame: a column a frame lacks gets nulls (NaN, NaT or None) there.""" + import numpy as np + + out: Dict[str, Any] = {} + for name in names: + template = next(p[name] for _, p in batches if name in p) + parts = [ + p[name] if name in p else _missing_column_part(c, template) + for c, p in batches + ] + np_parts = [p for p in parts if not isinstance(p, list)] + if parts and len(np_parts) == len(parts): + try: + out[name] = np.concatenate(parts) if len(parts) > 1 else parts[0] + continue + except Exception: # nosec B110 - fall through to generic merge + pass + column: list = [] + for p in parts: + column.extend(p if isinstance(p, list) else p.tolist()) + out[name] = column + return out + + +def decode_arrow_frame( + buf, side, json_values: Callable +) -> Tuple[int, Dict[str, Any], Dict[str, Any]]: + """``(count, chunks, header)`` of one frame as ``to_arrow`` columns: ``chunks`` maps a column name to a ``pyarrow.Array``. + Integers keep their type and their nulls (``to_columns`` would make them float64 with NaN), strings wrap the frame's own + offsets and bytes, vectors are fixed-size lists of float32 or float64.""" + import numpy as np + import pyarrow as pa + + header, pos = read_header(buf) + count = header["count"] + if count == 0: + return 0, {}, header + chunks: Dict[str, Any] = {} + for col in header["cols"]: + name, ctype = col["name"], col["type"] + nulls_len = col["nulls"] + null_bits, mask = _null_mask(buf, pos, nulls_len, count) + has_nulls = mask is not None + pos += nulls_len + data = buf[pos : pos + col["bytes"]] + pos += col["bytes"] + + if ctype == "i8": + # kept as int64 WITH validity, where to_columns promotes the whole column to float64/NaN + arr = pa.array(np.frombuffer(data, dtype="`` array of a column of float lists that float32 holds, or None.""" + import pyarrow as pa + + vec = _rows._float_vectors(values) + if vec is None: + return None + arr = pa.FixedSizeListArray.from_arrays(pa.array(vec.reshape(-1)), vec.shape[1]) + if any(v is None for v in values): + arr = pa.array(arr.to_pylist(), type=arr.type, mask=[v is None for v in values]) + return arr + + +def arrow_result(head_rows, columns, batches): + """The ``pyarrow.Table`` of the buffered rows (dicts) followed by the decoded frames ``batches`` (a list of ``(count, chunks)``).""" + import pyarrow as pa + + names = [] + chunks_list = [] + if head_rows: + table = _rows.to_arrow(head_rows, columns) + chunks = {} + for name in table.column_names: + column = table.column(name) + arr = ( + column.chunk(0) + if column.num_chunks == 1 + else pa.concat_arrays(column.chunks) + ) + if pa.types.is_null(arr.type): + # a frame types an all-null column as text + arr = pa.nulls(len(arr), pa.string()) + elif pa.types.is_list(arr.type) and arr.type.value_type == pa.float64(): + other = _first_part(batches, name) + if other is None or pa.types.is_fixed_size_list(other.type): + vector = _vector_arrow([row.get(name) for row in head_rows]) + if vector is not None: + arr = vector + chunks[name] = arr + names.extend(chunks) + chunks_list.append((len(head_rows), chunks)) + for count, chunks in batches: + for name in chunks: + if name not in names: + names.append(name) + chunks_list.append((count, chunks)) + return _columnar.merge_arrow(names, chunks_list) diff --git a/src/arcadedb/_native/database.py b/src/arcadedb/_native/database.py index f30dc526..71d178eb 100644 --- a/src/arcadedb/_native/database.py +++ b/src/arcadedb/_native/database.py @@ -210,6 +210,9 @@ def __init__(self, database, reply: dict, tagged: bool = True): self._rs = reply["rs"] self._buf = deque(zip(reply["rows"], reply["meta"])) self._more = not reply["drained"] + # the rows of this reply that the cursor still holds, for a columnar read to replay (arcadedb_n_next_col): the first reply of + # a cursor with more rows. Any later read of the cursor by row (arcadedb_n_next) lets the library drop them + self._head_n = len(reply["rows"]) if self._more else 0 self._closed = False self._exhausted = False # the engine computes lazily: an error met while the first rows were read is raised when the rows before it @@ -224,6 +227,7 @@ def _lazy(cls, database, rs_id: int) -> "NativeResultSet": self._rs = rs_id self._buf = deque() self._tagged = False + self._head_n = 0 self._more = rs_id > 0 self._closed = False self._exhausted = False @@ -258,6 +262,7 @@ def _fill(self) -> bool: raise _read_error(err) if not self._more: return False + self._head_n = 0 try: if lib.NEXT_BIN and _BINARY_REPLY: reply, tagged = loads_tagged( @@ -407,33 +412,100 @@ def iter_json_batches(self, batch_size: int = 10_000, **kwargs): # returns as it is. The flag is read after the batch is built, so it covers every reply the batch's rows came from yield [json_native(row) for row in batch] if self._tagged else batch + def _read_frames(self, batch_size, columns, decode, flags=0): + """The rows still to be read, in the two forms a result has: ``(head_rows, batches)``. + + ``head_rows`` are rows that this object holds as dicts (the first reply, when the cursor no longer has them); + ``batches`` is the list of ``(count, parts)`` that ``decode(frame_bytes)`` makes of each column frame the cursor + yields (about 1 MiB of values each, ``batch_size`` rows at most; ``flags`` 1 asks for datetimes as microseconds). The result + is finished when this returns. + """ + if int(batch_size) < 1: + raise ValueError("batch_size must be at least 1") + head_rows: list = [] + batches: list = [] + if not self._readable(): + return head_rows, batches + spec = json.dumps(list(columns)).encode("utf-8") if columns is not None else b"" + skip = 0 + if self._head_n and self._more and self._rs: + # the cursor still holds the rows of the first reply: it replays the ones this object has not handed out + skip = self._head_n - len(self._buf) + self._buf.clear() + else: + head_rows = [props for props, _ in self._buf] + self._buf.clear() + if self._pending_error is not None: + self._fill() # the rows before the error are consumed: raise it + while self._more and self._rs: + try: + raw = lib.call_frame(self._rs, int(batch_size), spec, flags, skip) + except lib.ColumnFrameError as exc: + # the library closed the cursor + self._more = False + self._rs = 0 + self._closed = True + self._exhausted = True + raise _read_error(exc.reply) from None + skip = 0 + self._head_n = 0 + count, parts, header = decode(raw) + if header.get("done"): + self._more = False # the library closed the cursor with the frame that ended the rows + self._rs = 0 + if count: + batches.append((count, parts)) + self._finish() + return head_rows, batches + def to_dataframe(self, convert_types: bool = True): + """The rows as a ``pandas.DataFrame``: the values ``to_list()`` returns (a DATETIME keeps its microseconds, a DATE is a + ``datetime.date`` in an object column, a ``Decimal`` stays one), read as column frames, not as one dict per row. + """ import pandas as pd - return pd.DataFrame(self._drain_dicts()) + columns = self._columns(25_000, None, True) + # pandas rejects 2-D ndarrays as column values; vector columns become object columns of row slices + columns = { + k: (list(v) if getattr(v, "ndim", 1) > 1 else v) for k, v in columns.items() + } + return pd.DataFrame(columns) + + def _columns(self, batch_size, columns, micros): + from . import colframes + + head_rows, batches = self._read_frames( + batch_size, + columns, + lambda raw: colframes.decode_columns(raw, micros), + 1 if micros else 0, + ) + return colframes.columns_result(head_rows, columns, micros, batches) def to_columns(self, batch_size: int = 25_000, columns=None): """Columns as the JPype path returns them (numpy int64/float64/bool/datetime64[ms] arrays, lists for text and - JSON-typed values, an object array of Decimal), built in Python from the rows: a columnar JSON path, no - binary buffer. None when numpy is missing.""" + JSON-typed values, an object array of Decimal), decoded from column frames by the decoder both builds share. None when + numpy is missing.""" try: - import numpy # noqa: F401 + __import__("numpy") # availability check except ImportError: return None - from .columns import to_columns - - return to_columns(self._drain_dicts(), columns) + return self._columns(batch_size, columns, False) def to_arrow(self, batch_size: int = 25_000, columns=None): - """A ``pyarrow.Table`` with the types of :meth:`to_columns` but nulls kept (None when pyarrow is missing).""" + """A ``pyarrow.Table`` with the types of :meth:`to_columns` but nulls kept (None when pyarrow is missing), decoded from + column frames by the decoder both builds share.""" try: - import numpy # noqa: F401 - import pyarrow # noqa: F401 + __import__("numpy") # availability check + __import__("pyarrow") except ImportError: return None - from .columns import to_arrow + from . import colframes - return to_arrow(self._drain_dicts(), columns) + head_rows, batches = self._read_frames( + batch_size, columns, colframes.decode_arrow + ) + return colframes.arrow_result(head_rows, columns, batches) def to_polars(self, batch_size: int = 25_000, columns=None): try: diff --git a/src/arcadedb/_native/lib.py b/src/arcadedb/_native/lib.py index 58b261ba..dc3f1e98 100644 --- a/src/arcadedb/_native/lib.py +++ b/src/arcadedb/_native/lib.py @@ -232,6 +232,7 @@ class _Params(ctypes.Structure): "arcadedb_n_insert_colp": [P, L, S, S, P, I], "arcadedb_n_next": [P, L, I], "arcadedb_n_next_bin": [P, L, I], + "arcadedb_n_next_col": [P, L, I, S, I, I], "arcadedb_n_rs_close": [P, L], "arcadedb_n_cmd": [P, L, I, S, S, S, S, L, I], "arcadedb_n_last_error": [P], @@ -365,6 +366,32 @@ def call_bin(name: str, *args): return raw.decode("utf-8"), floats +def call_frame(rs_id: int, max_rows: int, spec: bytes, flags: int, skip: int) -> bytes: + """One column frame of an open cursor (``arcadedb_n_next_col``): its bytes, in the layout ``arcadedb._columnar`` reads. + + The reply is an int64 byte count and the frame. A negative count is the negated length of an error JSON: it is raised as + :class:`ColumnFrameError` with the decoded reply (the cursor is already closed on the Java side then). + """ + lib = _lib or _load() + th = _thread_handle() + ptr = lib.arcadedb_n_next_col(th, rs_id, max_rows, spec, flags, skip) + try: + n = int.from_bytes(ctypes.string_at(ptr, 8), "little", signed=True) + if n < 0: + raise ColumnFrameError(json.loads(ctypes.string_at(ptr + 8, -n))) + return ctypes.string_at(ptr + 8, n) + finally: + lib.arcadedb_free(th, ptr) + + +class ColumnFrameError(Exception): + """The error reply (the dict ``errorJson`` writes) of a failed ``arcadedb_n_next_col``.""" + + def __init__(self, reply: dict): + super().__init__(reply.get("error")) + self.reply = reply + + def alloc(size: int): """Memory from the library's allocator (``arcadedb_n_alloc``), for the reply of a callback. diff --git a/src/arcadedb/results.py b/src/arcadedb/results.py index ebb344d0..945ab21e 100644 --- a/src/arcadedb/results.py +++ b/src/arcadedb/results.py @@ -12,6 +12,12 @@ import jpype as _jpype from jpype import JException +from ._columnar import ( + decode_arrow_frame, + decode_columns_frame, + merge_arrow, + merge_columns, +) from ._logging import get_logger from .callbacks import raise_pending_interrupt from .exceptions import ArcadeDBError @@ -92,6 +98,11 @@ def hook(obj): return hook +def _typed_json_values(data, side, count): + """The values of a ``json`` column of a ColumnBatcher frame: the tagged JSON (TypedRows) with its side array of Java objects.""" + return json.loads(bytes(data), object_hook=_typed_hook(side)) + + def _loads_typed_row(java_row): """The dict of one row or record that ``ReaderCalls.rowJson`` / ``documentJson`` wrote. @@ -156,162 +167,6 @@ def _java_class_name(value: Any) -> str: return str(value.getClass().getName()) -def _cast_all_to_string(arrs: list, pa) -> list: - """ - Cast every array to a string type. - - Tolerates a type pyarrow cannot cast directly (e.g. a vector/list column) - by going through Python objects instead. - """ - unified = [] - for a in arrs: - try: - unified.append(a.cast(pa.string())) - except pa.ArrowException: - unified.append( - pa.array( - [None if v is None else str(v) for v in a.to_pylist()], - type=pa.string(), - ) - ) - return unified - - -def _decode_strings(data, count: int, mask) -> list: - """The strings of one column buffer: int32 offsets, then the UTF-8 blob. - - ``mask`` is the null mask (True = null) or None when the column has no nulls. - """ - import numpy as np - - offs = np.frombuffer(data[: (count + 1) * 4], dtype=" list: - """The DECIMAL values of one column buffer, as ``Decimal`` objects (None for - null). The bridge writes each ``BigDecimal`` in full as text, so no digit is - lost to a double.""" - from decimal import Decimal - - return [ - None if s is None else Decimal(s) for s in _decode_strings(data, count, mask) - ] - - -def _missing_column_part(count: int, template): - """A batch of ``count`` rows that lack a column some other batch has: nulls in - the shape ``to_columns`` uses for that column (NaN for numbers, NaT for - datetimes, None for everything else).""" - import numpy as np - - if isinstance(template, np.ndarray): - if template.ndim == 2: - return np.full((count, template.shape[1]), np.nan, dtype=template.dtype) - if template.dtype.kind in "iuf": - return np.full(count, np.nan) - if template.dtype.kind == "M": - return np.full(count, np.datetime64("NaT"), dtype=template.dtype) - if template.dtype.kind == "O": - return np.full(count, None, dtype=object) - return [None] * count - - -def _is_untyped_arrow_chunk(a, pa) -> bool: - """True for a chunk that says nothing about its column's type: all values - null (a batch whose rows lack the property, or carry it as null), or a list - column holding only empty lists, which Arrow types ``list``.""" - if len(a) == 0 or a.null_count == len(a) or pa.types.is_null(a.type): - return True - return pa.types.is_list(a.type) and pa.types.is_null(a.type.value_type) - - -def _common_decimal_type(types: list, pa): - """One decimal type every given decimal type fits in (decimal128 up to 38 - digits, decimal256 up to 76), or None when it would need more.""" - scale = max(t.scale for t in types) - precision = max(t.precision - t.scale for t in types) + scale - if precision <= 38: - return pa.decimal128(precision, scale) - if precision <= 76: - return pa.decimal256(precision, scale) - return None - - -def _unify_arrow_chunk_types(arrs: list, pa) -> list: - """ - Unify a column's per-batch Arrow arrays onto one Arrow type. - - ArcadeDB is schemaless per document, so a property's Java type can - legally vary row to row; ColumnBatcher infers each batch's column type - independently, so two batches of the same result set can produce - incompatible Arrow types for the same column (#7108) - e.g. int64 in one - batch, float64 in the next. ``pa.chunked_array`` requires every chunk to - share one type, so leaving them as-is raises at concatenation time on a - result set that is otherwise entirely legal. - - Numeric types widen (int64 -> float64, matching to_columns' pandas-style - promotion) when every value survives the cast exactly; pyarrow's cast is - safe by default and raises rather than silently lose precision, which an - int64 outside float64's +-2**53 exact range would (code review on #7108), - so that case - and anything else, including a json or vector column that - varied - degrades to string instead, so the column always ends up with - exactly one type rather than an exception. - - int64/float64 is the only numeric pair worth special-casing: ColumnBatcher - (src/java/.../ColumnBatcher.java) already widens every - Java integer type (Long/Integer/Short/Byte) to the same "i8" wire encoding - and every Java floating type (Double/Float) to "f8" *within* one batch, so - across batches the only numeric type mismatch this ever has to unify is - exactly this pair - there is no int32/int64 or bool/int64 case to widen. - """ - types = {a.type for a in arrs} - if len(types) <= 1: - return arrs - - # A chunk that carries no type information (every value null, or only empty - # lists) takes the type the other chunks agree on, so the column's type does - # not depend on where the batch boundaries fall (#114). - untyped = [_is_untyped_arrow_chunk(a, pa) for a in arrs] - if any(untyped) and not all(untyped): - typed = _unify_arrow_chunk_types( - [a for a, u in zip(arrs, untyped) if not u], pa - ) - target = typed[0].type - filled = iter(typed) - out = [] - for a, u in zip(arrs, untyped): - if not u: - out.append(next(filled)) - elif a.null_count == len(a): - out.append(pa.nulls(len(a), type=target)) - else: - try: - out.append(a.cast(target)) - except pa.ArrowException: - return _cast_all_to_string(arrs, pa) - return out - - if types <= {pa.int64(), pa.float64()}: - try: - return [a.cast(pa.float64()) for a in arrs] - except pa.ArrowException: - pass - - if all(pa.types.is_decimal(t) for t in types): - common = _common_decimal_type(list(types), pa) - if common is not None: - return [a.cast(common) for a in arrs] - - return _cast_all_to_string(arrs, pa) - - class ResultSet: """Iterator wrapper for ArcadeDB query results. @@ -816,98 +671,20 @@ def _read_columns(self, batch_size, columns, micros): column of ``date`` objects, which is what ``to_dataframe()`` needs to return the values ``to_list()`` does.""" try: - import numpy as np + __import__("numpy") # availability check except ImportError: return None - import json - column_batcher = _bridge_class("ColumnBatcher") if column_batcher is None: return None # Every batch reports the union of its own rows' property names. The - # result's columns are the union over the batches, in order of first - # appearance (#113); a batch lacking one gets nulls for it when merged. + # result's columns are the union over the batches, in order of + # first appearance (#113); a batch lacking one gets nulls for it when merged. names: List[str] = [] batches: List[Tuple[int, Dict[str, Any]]] = [] - def decode_batch(buf, side): - hlen = int.from_bytes(buf[:4], "little") - header = json.loads(bytes(buf[4 : 4 + hlen])) - count = header["count"] - if count == 0: - return 0 - batch_parts: Dict[str, Any] = {} - pos = 4 + hlen - for col in header["cols"]: - name, ctype = col["name"], col["type"] - nulls_len = col["nulls"] - null_bits = np.frombuffer(buf[pos : pos + nulls_len], dtype=np.uint8) - has_nulls = bool(null_bits.any()) - if has_nulls: - mask = np.unpackbits(null_bits, bitorder="little")[:count].astype( - bool - ) - pos += nulls_len - data = buf[pos : pos + col["bytes"]] - pos += col["bytes"] - - if ctype == "i8": - arr = np.frombuffer(data, dtype=" 2-D array (count, dim); - # Java already writes NaN rows into null slots - # an empty vector has no "dim" in the header - dim = col.get("dim", 0) - dt = " 1 else parts[0] - continue - except Exception: # nosec B110 - fall through to generic merge - pass - column = [] - for p in parts: - column.extend(p if isinstance(p, list) else p.tolist()) - out[name] = column - return out + return merge_columns(names, batches) def to_arrow( self, batch_size: int = 25_000, columns: Optional[Sequence[str]] = None @@ -983,13 +747,11 @@ def to_arrow( falls back to the row-based paths. """ try: - import numpy as np + __import__("numpy") # availability check import pyarrow as pa except ImportError: return None - import json - column_batcher = _bridge_class("ColumnBatcher") if column_batcher is None: return None @@ -1000,107 +762,6 @@ def to_arrow( names: List[str] = [] batches: List[Tuple[int, Dict[str, Any]]] = [] - def validity(null_bits, count, has_nulls): - """Arrow validity is 1=valid; the bridge writes 1=null, so invert. - Bits past `count` are padding and Arrow ignores them.""" - if not has_nulls: - return None - return pa.py_buffer(bytes(np.invert(null_bits).tobytes())) - - def decode_batch(buf, side): - hlen = int.from_bytes(buf[:4], "little") - header = json.loads(bytes(buf[4 : 4 + hlen])) - count = header["count"] - if count == 0: - return 0 - batch_chunks: Dict[str, Any] = {} - pos = 4 + hlen - for col in header["cols"]: - name, ctype = col["name"], col["type"] - nulls_len = col["nulls"] - null_bits = np.frombuffer(buf[pos : pos + nulls_len], dtype=np.uint8) - has_nulls = bool(null_bits.any()) - mask = None - if has_nulls: - mask = np.unpackbits(null_bits, bitorder="little")[:count].astype( - bool - ) - pos += nulls_len - data = buf[pos : pos + col["bytes"]] - pos += col["bytes"] - - if ctype == "i8": - # kept as int64 WITH validity, where to_columns would have - # promoted the whole column to float64/NaN - arr = pa.array(np.frombuffer(data, dtype=" Python upcalls (Python functions in SQL) exist on the native build only collect_ignore.append("test_native_udf.py") # how find_nearest hands the query vector to the native library (the codec's side buffer) diff --git a/tests/test_column_frames.py b/tests/test_column_frames.py new file mode 100644 index 00000000..a2e51a22 --- /dev/null +++ b/tests/test_column_frames.py @@ -0,0 +1,338 @@ +"""The column frame decoder both wheels share (``arcadedb._columnar``), and the columnar readers that use it (refs #312). + +The first half builds frames by hand, in the layout of ``ColumnBatcher`` (JVM wheel) and ``arcadedb_n_next_col`` (native wheel), and +decodes them: no database, so a decoder change is checked on its own. The second half reads one stored table through ``to_arrow``, +``to_columns``, ``to_dataframe`` and ``to_polars`` and compares each value with what was stored, on whichever wheel runs the tests: +nulls kept in Arrow, one type per column whatever the batch size, a result of more than one frame, column subsets, an empty result, +one row, many columns, vectors of several lengths. There is no branch on ``arcadedb.__backend__``: a literal that holds on both wheels +is the parity check. +""" + +import json +import math +import struct +from datetime import datetime + +import numpy as np +import pyarrow as pa +import pytest + +from arcadedb import _columnar + + +def _json_values(data, side, count): + return json.loads(bytes(data)) + + +def _bits(count, nulls): + out = bytearray((count + 7) // 8) + for i in nulls: + out[i >> 3] |= 1 << (i & 7) + return bytes(out) + + +def _frame(count, columns): + """``columns``: ``(name, type, null_rows, data_bytes[, dim])`` each; the frame as ColumnBatcher writes it.""" + cols, body = [], b"" + for name, ctype, null_rows, data, *dim in columns: + nulls = _bits(count, null_rows) + head = {"name": name, "type": ctype, "nulls": len(nulls), "bytes": len(data)} + if dim: + head["dim"] = dim[0] + cols.append(head) + body += nulls + data + header = json.dumps({"count": count, "cols": cols}).encode("utf-8") + return memoryview(struct.pack(" 1 or t.num_rows == 8000 + + +def test_to_columns_of_the_stored_rows(table): + db, rows = table + c = db.query("sql", _sel()).to_columns() + assert c["big"].dtype == np.int64 and list(c["big"]) == [r["big"] for r in rows] + assert c["n"].dtype == np.float64 # an int column with nulls + assert c["d"].dtype == np.float64 and c["ts"].dtype == np.dtype("datetime64[ms]") + assert c["v9"].dtype == np.float32 and c["v9"].shape == (5000, 9) + assert c["v128"].shape == (5000, 128) + np.testing.assert_array_equal( + c["v9"], np.array([r["v9"] for r in rows], dtype=np.float32) + ) + assert c["s"] == [r["s"] for r in rows] + assert c["ok"] == [r["ok"] for r in rows] # a bool column with nulls is a list + + +def test_column_subset_and_order(table): + db, rows = table + t = db.query("sql", _sel()).to_arrow(columns=["s", "big"]) + assert t.column_names == ["s", "big"] + assert t.column("big").to_pylist() == [r["big"] for r in rows] + c = db.query("sql", _sel()).to_columns(columns=["v3"]) + assert list(c) == ["v3"] and c["v3"].shape == (5000, 3) + + +def test_empty_and_one_row_results(table): + db, rows = table + empty = db.query("sql", "SELECT n, s FROM T WHERE big < 0").to_arrow() + assert empty.num_rows == 0 + assert db.query("sql", "SELECT n, s FROM T WHERE big < 0").to_columns() == {} + one = db.query( + "sql", "SELECT n, s, v3 FROM T WHERE big = ?", rows[1]["big"] + ).to_arrow() + assert one.num_rows == 1 + assert one.column("s").to_pylist() == [rows[1]["s"]] + assert one.column("v3").to_pylist() == [rows[1]["v3"]] + assert db.query("sql", "SELECT FROM T WHERE big < 0").to_arrow().num_rows == 0 + + +def test_a_cursor_read_by_row_first_then_to_arrow(table): + db, rows = table + rs = db.query("sql", _sel()) + first = [next(rs) for _ in range(10)] + assert [r.get("big") for r in first] == [r["big"] for r in rows[:10]] + t = rs.to_arrow() + assert t.column("big").to_pylist() == [r["big"] for r in rows[10:]] + + +def test_many_columns(temp_db): + db = temp_db + db.command("sql", "CREATE DOCUMENT TYPE Many") + names = [f"c{k}" for k in range(80)] + for k, name in enumerate(names): + db.command( + "sql", + f"CREATE PROPERTY Many.{name} {['INTEGER', 'STRING', 'DOUBLE'][k % 3]}", + ) + rows = [ + { + name: (i + k if k % 3 == 0 else f"{i}-{k}" if k % 3 == 1 else (i + k) / 4) + for k, name in enumerate(names) + } + for i in range(600) + ] + db.insert_many("Many", rows) + t = db.query("sql", "SELECT FROM Many ORDER BY c0").to_arrow(columns=names) + assert t.column_names == names + for name in ("c0", "c1", "c2", "c79"): + assert t.column(name).to_pylist() == [r[name] for r in rows] + + +def test_to_dataframe_and_to_polars(table): + db, rows = table + df = db.query("sql", _sel()).to_dataframe() + assert list(df.columns) == [n for n, _ in TYPES] + assert list(df["big"]) == [r["big"] for r in rows] + assert df["s"].tolist() == [r["s"] for r in rows] + assert [list(v) for v in df["v3"]] == [r["v3"] for r in rows] + assert df["ts"].iloc[1] == datetime(2026, 1, 2, 3, 4, 5, 1000) + import polars as pl + + p = db.query("sql", _sel()).to_polars() + assert isinstance(p, pl.DataFrame) and p.height == 5000 + assert p["big"].to_list() == [r["big"] for r in rows] diff --git a/tests/test_native_column_frames_reference.py b/tests/test_native_column_frames_reference.py new file mode 100644 index 00000000..19b46cab --- /dev/null +++ b/tests/test_native_column_frames_reference.py @@ -0,0 +1,434 @@ +"""The native column frames give the table the JSON row path gave (refs #312). + +Before the frames, ``NativeResultSet.to_arrow``, ``to_columns`` and ``to_dataframe`` read every row as JSON text into one Python dict and +built the columns from those dicts. The module below (``REF_SOURCE``) is that assembly, copied from ``arcadedb._native.columns`` as it was +the day the frames came, and kept here as a private reference: each case reads the same query twice, once through the old route (the +rows drained as dicts, then the reference rules) and once through the frames, and compares the two column by column, value by value. + +Equal output is the bar. Two types are different on purpose, because the JVM wheel's frames type them so and both wheels now read frames +with one decoder: a column of ``float[]`` is a ``fixed_size_list`` (the old rules gave ``list``, the same values: a +float32 widened to float64 is exact), and a column of only nulls is a string column (the old rules gave the null type). + +conftest.py collects this file on the native backend only (the old route is the native build's JSON reply). +""" + +import math +import types +from datetime import date, datetime +from decimal import Decimal + +import numpy as np +import pandas as pd +import pyarrow as pa +import pytest + +REF_SOURCE = '''"""Rows to columns for the native backend (the columnar path is Python over the rows; the JPype path decodes a Java buffer).""" + +from datetime import date, datetime, timezone +from decimal import Decimal + + +def _names(rows, columns): + if columns is not None: + return list(columns) + seen = {} + for row in rows: + for k in row: + seen.setdefault(k, None) + return list(seen) + + +def _kind(values): + """'b', 'i', 'f', 'dt', 'dec', 'obj' for the non-null values of a column.""" + kinds = set() + for v in values: + if v is None: + continue + t = type(v) + if t is bool: + kinds.add("b") + elif t is int: + kinds.add("i") + elif t is float: + kinds.add("f") + elif t is datetime or t is date: + kinds.add("dt") + elif t is Decimal: + kinds.add("dec") + else: + kinds.add("obj") + if not kinds: + return "obj" + if len(kinds) == 1: + return kinds.pop() + if kinds == {"i", "f"}: + return "f" + return "obj" + + +def _ms(v): + if isinstance(v, datetime): + if v.tzinfo is not None: + v = v.astimezone(timezone.utc).replace(tzinfo=None) + return v + return datetime(v.year, v.month, v.day) + + +def _float_vectors(values): + """A column of equal-length float lists (None allowed) that float32 holds exactly, as a (count, dim) float32 array with + NaN rows for the nulls; None for any other column. Rows cannot say float[] from List, so only a column whose every + value survives float32 unchanged is read as a vector column (the JPype path decides on the Java element type). + """ + import numpy as np + + dim = 0 + for v in values: + if v is None: + continue + if type(v) is not list or not v or not all(type(x) is float for x in v): + return None + if dim == 0: + dim = len(v) + elif len(v) != dim: + return None + if dim == 0: + return None + arr = np.full((len(values), dim), np.nan, dtype=np.float32) + for i, v in enumerate(values): + if v is not None: + arr[i] = v + if not np.array_equal(arr[i].astype(np.float64), np.array(v)): + return None + return arr + + +def to_columns(rows, columns=None): + import numpy as np + + out = {} + for name in _names(rows, columns): + values = [row.get(name) for row in rows] + if _kind(values) == "obj": + vectors = _float_vectors(values) + if vectors is not None: + out[name] = vectors + continue + has_null = any(v is None for v in values) + kind = _kind(values) + if kind == "i": + out[name] = ( + np.array([np.nan if v is None else v for v in values], dtype=np.float64) + if has_null + else np.array(values, dtype=np.int64) + ) + elif kind == "f": + out[name] = np.array( + [np.nan if v is None else v for v in values], dtype=np.float64 + ) + elif kind == "b": + out[name] = values if has_null else np.array(values, dtype=bool) + elif kind == "dt": + out[name] = np.array( + [ + ( + np.datetime64("NaT", "ms") + if v is None + else np.datetime64(_ms(v), "ms") + ) + for v in values + ], + dtype="datetime64[ms]", + ) + elif kind == "dec": + arr = np.empty(len(values), dtype=object) + arr[:] = values + out[name] = arr + else: + out[name] = values + return out + + +def to_arrow(rows, columns=None): + import pyarrow as pa + + arrays = {} + for name in _names(rows, columns): + values = [row.get(name) for row in rows] + kind = _kind(values) + if kind == "i": + arrays[name] = pa.array(values, type=pa.int64()) + elif kind == "f": + try: + arrays[name] = pa.array(values, type=pa.float64()) + except ( + pa.ArrowInvalid + ): # an int beyond +-2**53 next to floats: the column degrades to exact text + arrays[name] = pa.array( + [None if v is None else str(v) for v in values], type=pa.string() + ) + elif kind == "b": + arrays[name] = pa.array(values, type=pa.bool_()) + elif kind == "dt": + arrays[name] = pa.array( + [None if v is None else _ms(v) for v in values], type=pa.timestamp("ms") + ) + elif kind == "dec": + try: + arrays[name] = pa.array(values) + except ( + pa.ArrowInvalid + ): # wider than decimal256 (76 digits): an exact string + arrays[name] = pa.array( + [None if v is None else str(v) for v in values], type=pa.string() + ) + else: + try: + arrays[name] = pa.array([None if v is None else v for v in values]) + except (pa.ArrowInvalid, pa.ArrowTypeError): + arrays[name] = pa.array( + [None if v is None else str(v) for v in values], type=pa.string() + ) + return pa.table(arrays) +''' + +REF = types.ModuleType("columns_reference") +exec( + compile(REF_SOURCE, "columns_reference", "exec"), REF.__dict__ +) # nosec B102 - the test's own copy of the old rules + + +def _f32(x): + return float(np.float32(x)) + + +def _row(i): + null = i % 7 == 3 + row = { + "n": None if null else i, + "big": (1 << 40) + i, + "sh": None if null else (i % 2000) - 1000, + "by": None if null else (i % 200) - 100, + "d": i / 3, + "f": None if null else _f32(i / 7), + "s": None if null else f"row {i} \U0001f600 \x00 \u00e9", + "ok": None if null else i % 2 == 0, + "ts": datetime(2026, 1, 2, 3, 4, 5, (i % 1000) * 1000), + "day": date(2000 + i % 30, 1 + i % 12, 1 + i % 28), + "m": Decimal(f"{i}.25"), + "v1": [_f32(i)], + "v3": [_f32(i), _f32(i / 2), 0.5], + "v9": None if null else [_f32(i + k / 10) for k in range(9)], + "v128": [_f32(i * 0.5 + k) for k in range(128)], + "pad": "p" * 400, + "nothing": None, + } + return row + + +TYPES = [ + ("n", "INTEGER"), + ("big", "LONG"), + ("sh", "SHORT"), + ("by", "BYTE"), + ("d", "DOUBLE"), + ("f", "FLOAT"), + ("s", "STRING"), + ("ok", "BOOLEAN"), + ("ts", "DATETIME"), + ("day", "DATE"), + ("m", "DECIMAL"), + ("v1", "ARRAY_OF_FLOATS"), + ("v3", "ARRAY_OF_FLOATS"), + ("v9", "ARRAY_OF_FLOATS"), + ("v128", "ARRAY_OF_FLOATS"), + ("pad", "STRING"), + ("nothing", "STRING"), +] +NAMES = [n for n, _ in TYPES] +ROWS = 5000 + + +@pytest.fixture(scope="module") +def ref_db(tmp_path_factory): + import arcadedb + + path = str(tmp_path_factory.mktemp("frames_ref") / "db") + db = arcadedb.create_database(path) + db.command("sql", "CREATE DOCUMENT TYPE R") + for name, type_name in TYPES: + db.command("sql", f"CREATE PROPERTY R.{name} {type_name}") + db.insert_many("R", [_row(i) for i in range(ROWS)]) + yield db + db.close() + + +def _norm(v): + """A value as a plain Python object, NaN, NaT and None alike (pandas turns a None into NaN in some columns).""" + if isinstance(v, np.ndarray): + return [_norm(x) for x in v.tolist()] + if isinstance(v, (list, tuple)): + return [_norm(x) for x in v] + if v is None or v is pd.NaT: + return None + if isinstance(v, float) and math.isnan(v): + return None + if isinstance(v, np.datetime64): + return None if np.isnat(v) else v.astype("datetime64[us]").item() + if isinstance(v, np.generic): + return v.item() + if isinstance(v, pd.Timestamp): + return v.to_pydatetime() + return v + + +def _old_rows(db, sql, *args): + return db.query("sql", sql, *args)._drain_dicts() + + +def _assert_arrow_equal(new, old): + assert new.column_names == old.column_names + assert new.num_rows == old.num_rows + for name in old.column_names: + have, want = new.column(name), old.column(name) + if pa.types.is_null(want.type): + assert have.type == pa.string() and have.null_count == len(have), name + elif pa.types.is_list(want.type) and want.type.value_type == pa.float64(): + assert pa.types.is_fixed_size_list(have.type) or pa.types.is_list( + have.type + ), name + assert have.type.value_type in (pa.float32(), pa.float64()), name + else: + assert have.type == want.type, (name, have.type, want.type) + assert [_norm(x) for x in have.to_pylist()] == [ + _norm(x) for x in want.to_pylist() + ], name + + +def _assert_columns_equal(new, old): + assert list(new) == list(old) + for name, want in old.items(): + have = new[name] + if isinstance(want, np.ndarray): + assert isinstance(have, np.ndarray), (name, type(have)) + assert have.dtype == want.dtype, (name, have.dtype, want.dtype) + assert have.shape == want.shape, name + else: + assert isinstance(have, list), (name, type(have)) + assert _norm(have) == _norm(want), name + + +def _assert_frame_equal(new, old_rows, columns=None): + old = pd.DataFrame(old_rows) + if columns is not None: + old = old.reindex(columns=list(columns)) + assert list(new.columns) == list(old.columns) + for name in old.columns: + assert _norm(new[name].tolist()) == _norm(old[name].tolist()), name + + +QUERIES = [ + ("full", "SELECT FROM R ORDER BY big", None), + ( + "projection", + "SELECT n, big, sh, by, d, f, s, ok, ts, day, m FROM R ORDER BY big", + None, + ), + ("vectors", "SELECT big, v1, v3, v9, v128 FROM R ORDER BY big", None), + ("limit 1", "SELECT FROM R ORDER BY big LIMIT 1", None), + ("limit 100, one reply", "SELECT FROM R ORDER BY big LIMIT 100", None), + ("limit 300, a reply and a cursor", "SELECT FROM R ORDER BY big LIMIT 300", None), + ("empty", "SELECT FROM R WHERE big < 0", None), + ("subset", "SELECT FROM R ORDER BY big", ["s", "big", "v3", "ts"]), + ("subset with a column nobody has", "SELECT FROM R ORDER BY big", ["n", "absent"]), + ("subset on a short result", "SELECT FROM R ORDER BY big LIMIT 20", ["m", "ok"]), +] + + +@pytest.mark.parametrize("label,sql,columns", QUERIES, ids=[q[0] for q in QUERIES]) +def test_to_arrow_equals_the_old_route(ref_db, label, sql, columns): + old = REF.to_arrow(_old_rows(ref_db, sql), columns) + new = ref_db.query("sql", sql).to_arrow(columns=columns) + _assert_arrow_equal(new, old) + + +@pytest.mark.parametrize("label,sql,columns", QUERIES, ids=[q[0] for q in QUERIES]) +def test_to_columns_equals_the_old_route(ref_db, label, sql, columns): + old = REF.to_columns(_old_rows(ref_db, sql), columns) + new = ref_db.query("sql", sql).to_columns(columns=columns) + _assert_columns_equal(new, old) + + +@pytest.mark.parametrize( + "label,sql,columns", QUERIES[:7], ids=[q[0] for q in QUERIES[:7]] +) +def test_to_dataframe_equals_the_old_route(ref_db, label, sql, columns): + rows = _old_rows(ref_db, sql) + new = ref_db.query("sql", sql).to_dataframe() + _assert_frame_equal(new, rows) + + +@pytest.mark.parametrize("batch_size", [1, 64, 255, 256, 257, 1000, 25000]) +def test_the_frames_do_not_depend_on_the_batch_size(ref_db, batch_size): + sql = "SELECT FROM R ORDER BY big" + old = REF.to_arrow(_old_rows(ref_db, sql)) + _assert_arrow_equal(ref_db.query("sql", sql).to_arrow(batch_size=batch_size), old) + + +def test_a_result_larger_than_one_frame_is_read_in_several(ref_db): + from unittest import mock + + from arcadedb._native import lib + + sizes = [] + real = lib.call_frame + + def spy(*args): + raw = real(*args) + sizes.append(len(raw)) + return raw + + # 5000 rows of about 2.4 KiB of values: several frames of about 1 MiB each, and the table is the same as one read of the rows + with mock.patch.object(lib, "call_frame", spy): + t = ref_db.query("sql", "SELECT FROM R ORDER BY big").to_arrow() + assert t.num_rows == ROWS + assert len(sizes) >= 4, sizes + assert max(sizes) < 4 * (1 << 20), sizes # a frame, not the table + assert t.column("big").to_pylist() == [(1 << 40) + i for i in range(ROWS)] + + +def test_rows_read_by_iteration_first_then_the_rest_as_columns(ref_db): + sql = "SELECT FROM R ORDER BY big" + old = REF.to_arrow(_old_rows(ref_db, sql)).slice(40) + rs = ref_db.query("sql", sql) + taken = [next(rs).get("big") for _ in range(40)] + assert taken == [(1 << 40) + i for i in range(40)] + _assert_arrow_equal(rs.to_arrow(), old) + + +def test_rows_read_past_the_first_reply_then_the_rest_as_columns(ref_db): + sql = "SELECT FROM R ORDER BY big" + old = REF.to_arrow(_old_rows(ref_db, sql)).slice(300) + rs = ref_db.query("sql", sql) + for _ in range(300): + next(rs) + _assert_arrow_equal(rs.to_arrow(), old) + + +def test_a_result_is_read_once(ref_db): + rs = ref_db.query("sql", "SELECT FROM R ORDER BY big LIMIT 500") + assert rs.to_arrow().num_rows == 500 + assert rs.to_arrow().num_rows == 0 + + +def test_to_list_still_reads_the_json_rows(ref_db): + rows = ref_db.query("sql", "SELECT FROM R ORDER BY big LIMIT 300").to_list() + assert len(rows) == 300 and rows[299]["big"] == (1 << 40) + 299 + + +def test_the_cursor_keeps_no_rows_after_a_row_read(ref_db): + rs = ref_db.query("sql", "SELECT FROM R ORDER BY big") + assert rs._head_n == 256 + for _ in range(300): + next(rs) + assert rs._head_n == 0 + rs.close() From 71bad36ed15678d78e96b62a81d0d03229483738 Mon Sep 17 00:00:00 2001 From: Taewoon Kim Date: Mon, 9 Nov 2026 19:00:08 +0900 Subject: [PATCH 09/49] Tests: the frames test builds its SQL as one literal (bandit B608) Co-Authored-By: Claude Sonnet 5.5 --- tests/test_column_frames.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/test_column_frames.py b/tests/test_column_frames.py index a2e51a22..0265a17a 100644 --- a/tests/test_column_frames.py +++ b/tests/test_column_frames.py @@ -197,8 +197,8 @@ def table(temp_db): return db, rows -def _sel(extra=""): - return "SELECT n, big, d, f, s, ok, ts, v3, v9, v128 FROM T ORDER BY big" + extra +def _sel(): + return "SELECT n, big, d, f, s, ok, ts, v3, v9, v128 FROM T ORDER BY big" def _same(have, want): From 4b42be7f64383395e001bbac94882e9ace7f1e27 Mon Sep 17 00:00:00 2001 From: Taewoon Kim Date: Mon, 9 Nov 2026 19:00:18 +0900 Subject: [PATCH 10/49] JVM wheel: a default CDS archive for the bundled JRE, dumped with the flags the wheel starts with (#313) jlink makes a JRE without a class-data-sharing archive. scripts/dump_cds_archive.sh dumps the JVM's default archive as the last step of the JRE build (Docker and the native macOS and Windows builds), with -XX:+UseCompactObjectHeaders as jvm.py sets it, and checks it with -Xshare:on. The AOT cache stays opt-in. Co-Authored-By: Claude Sonnet 5.5 --- docs/modules/ROOT/pages/installation.adoc | 5 ++ scripts/Dockerfile.build | 4 ++ scripts/build-native.sh | 5 ++ scripts/dump_cds_archive.sh | 46 ++++++++++++++++ src/arcadedb/jvm.py | 4 +- tests/conftest.py | 1 + tests/test_jvm_cds.py | 65 +++++++++++++++++++++++ 7 files changed, 129 insertions(+), 1 deletion(-) create mode 100755 scripts/dump_cds_archive.sh create mode 100644 tests/test_jvm_cds.py diff --git a/docs/modules/ROOT/pages/installation.adoc b/docs/modules/ROOT/pages/installation.adoc index 45e29999..a3945ada 100644 --- a/docs/modules/ROOT/pages/installation.adoc +++ b/docs/modules/ROOT/pages/installation.adoc @@ -155,6 +155,11 @@ WARNING: *One JVM configuration per process.* JVM options are locked after the J Each Python process starts its own JVM, and each JVM sees the whole container limit. With several Python processes in one container, give each process its own `heap_size`. +[[jvm-cds]] +=== Class-data sharing + +The bundled JRE ships the JVM's default class-data-sharing archive, dumped with `-XX:+UseCompactObjectHeaders`, the flag that `start_jvm` sets. The JVM finds the archive by itself and maps the JDK's classes from it instead of loading and verifying them at every start. You do nothing to use it. The JVM ignores the archive, and starts as it did before, when your `jvm_args` change a setting that the archive depends on (`-XX:-UseCompactObjectHeaders`, `-Xshare:off`) or when the heap is larger than 32 GB (the default is half of the RAM, so a machine with more than 64 GB of RAM starts without the archive unless you set `heap_size`). The archive holds the JDK's classes only, not the engine's jars. + [[jvm-aot]] === Faster start-up with the AOT cache diff --git a/scripts/Dockerfile.build b/scripts/Dockerfile.build index f9ca2684..23712ca6 100644 --- a/scripts/Dockerfile.build +++ b/scripts/Dockerfile.build @@ -181,6 +181,10 @@ RUN echo "🔍 Analyzing JARs with jdeps..." && \ exit 1; \ fi +# The default class-data-sharing archive of the JRE, dumped last with the flags the wheel starts the JVM with (#313) +COPY scripts/dump_cds_archive.sh /build/dump_cds_archive.sh +RUN bash /build/dump_cds_archive.sh /build/jre + # Stage 3: Build Python wheel FROM python:${PYTHON_VERSION}-slim AS python-builder diff --git a/scripts/build-native.sh b/scripts/build-native.sh index ee6f72d4..3505eaab 100755 --- a/scripts/build-native.sh +++ b/scripts/build-native.sh @@ -275,6 +275,11 @@ if [[ "$PLATFORM" == windows/* ]]; then echo -e "${CYAN}📊 JRE size after Windows cleanup: ${YELLOW}${JRE_SIZE}${NC}" fi +# The default class-data-sharing archive of the JRE, dumped last with the flags the wheel starts the JVM with (#313) +bash "$SCRIPT_DIR/dump_cds_archive.sh" "$PY_BINDINGS_DIR/temp_jre" +JRE_SIZE=$(du -sh "$PY_BINDINGS_DIR/temp_jre" | cut -f1) +echo -e "${CYAN}📊 JRE size with the CDS archive: ${YELLOW}${JRE_SIZE}${NC}" + # Step 3: Copy JRE to package (JARs already filtered in place) echo -e "${CYAN}📦 Preparing package...${NC}" diff --git a/scripts/dump_cds_archive.sh b/scripts/dump_cds_archive.sh new file mode 100755 index 00000000..35d17fd1 --- /dev/null +++ b/scripts/dump_cds_archive.sh @@ -0,0 +1,46 @@ +#!/usr/bin/env bash +# SPDX-FileCopyrightText: 2021-present Arcade Data Ltd (info@arcadedata.com) and HumemAI (info@humem.ai) +# SPDX-License-Identifier: Apache-2.0 +# +# Dump the default class-data-sharing (CDS) archive of the bundled JRE, so the JVM maps the JDK's classes at start instead of +# loading and verifying them (#313). A JRE made by jlink has no archive; the JVM then starts without one. +# +# Usage: dump_cds_archive.sh JRE_DIR +# +# The archive is the JVM's default one (lib/server, or bin/server on Windows): the JVM looks for it by itself at every start and +# ignores it, without an error, when it was dumped for other settings. So it must be dumped with the settings the wheel starts +# with: -XX:+UseCompactObjectHeaders, which src/arcadedb/jvm.py sets (an archive dumped without it is ignored by a JVM that has it, +# and the JVM names the default archive after it). -Xmx1g keeps compressed oops on whatever the RAM of this build host is: the +# archive is the one a heap below 32 GB uses. A start with a larger heap (jvm.py gives the JVM 50 percent of the RAM) takes no +# archive, as before. The archive holds the JDK's classes only: the engine's jars are named by absolute path and checked by +# mtime in an archive, which an installed wheel cannot match. +# +# Run it last, on the finished JRE: the archive records the size of lib/modules. It checks its own result with -Xshare:on, which +# fails the JVM start when the archive cannot be used. +set -euo pipefail + +JRE_DIR="${1:?usage: dump_cds_archive.sh JRE_DIR}" +JAVA="${JRE_DIR}/bin/java" +CDS_FLAGS=(-XX:+UseCompactObjectHeaders -Xmx1g) + +if [[ ! -x "$JAVA" && ! -x "${JAVA}.exe" ]]; then + echo "dump_cds_archive.sh: no java in ${JRE_DIR}/bin" >&2 + exit 1 +fi + +echo "Dumping the default CDS archive of ${JRE_DIR} with: ${CDS_FLAGS[*]}" +"$JAVA" "${CDS_FLAGS[@]}" -Xshare:dump + +ARCHIVES="$(find "$JRE_DIR" -name '*.jsa' -type f)" +if [[ -z "$ARCHIVES" ]]; then + echo "dump_cds_archive.sh: the dump wrote no .jsa file under ${JRE_DIR}" >&2 + exit 1 +fi +echo "CDS archive:" +while IFS= read -r archive; do + echo " ${archive} ($(du -k "$archive" | cut -f1) KiB)" +done <<< "$ARCHIVES" + +# -Xshare:on: no archive, or one for other settings, is an error here instead of a quiet start without it +"$JAVA" "${CDS_FLAGS[@]}" -Xshare:on -version +echo "The JVM starts with the archive (-Xshare:on)" diff --git a/src/arcadedb/jvm.py b/src/arcadedb/jvm.py index 2289ccf6..f38067a6 100644 --- a/src/arcadedb/jvm.py +++ b/src/arcadedb/jvm.py @@ -889,7 +889,9 @@ def _build_jvm_args( ): merged_args.append("-Darcadedb.polyglotEngineEnabled=false") - # Enable compact object headers (JDK 25+) to reduce heap footprint + # Enable compact object headers (JDK 25+) to reduce heap footprint. + # The bundled JRE's class-data-sharing archive is dumped with this flag (scripts/dump_cds_archive.sh, #313): the JVM ignores an + # archive dumped with another setting of it, so a change here is a change there, and tests/test_jvm_cds.py fails when they differ. if not any("UseCompactObjectHeaders" in arg for arg in merged_args): merged_args.append("-XX:+UseCompactObjectHeaders") diff --git a/tests/conftest.py b/tests/conftest.py index ec3147c7..4a3419a2 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -110,6 +110,7 @@ "test_server_packaging.py": "JVM wheel contents", "test_runtime_cache.py": "JRE runtime cache of the JVM wheel", "test_aot.py": "JVM class-data cache", + "test_jvm_cds.py": "class-data-sharing archive of the JVM wheel's bundled JRE", "test_stage_jars.py": "staging of the JVM wheel's jars", "test_extract_version.py": "version of the JVM wheel's engine jar", "test_extract_version_classifiers.py": "version of the JVM wheel's engine jar", diff --git a/tests/test_jvm_cds.py b/tests/test_jvm_cds.py new file mode 100644 index 00000000..033f87ec --- /dev/null +++ b/tests/test_jvm_cds.py @@ -0,0 +1,65 @@ +"""The bundled JRE ships a class-data-sharing (CDS) archive that the JVM uses at start (#313). + +A JRE made by jlink has no archive, so every start loaded and verified the JDK's classes again. ``scripts/dump_cds_archive.sh`` dumps the +JVM's default archive as the last step of the JRE build, with ``-XX:+UseCompactObjectHeaders`` (which ``jvm.py`` sets: the JVM ignores an +archive dumped with another setting, and it does so without an error). These tests run on the installed wheel, on every platform and +Python of the JVM matrix: the archive is in the wheel, the JVM maps it with the flags ``start_jvm`` builds (``-Xshare:on`` turns an +unusable archive into a failed start; ``-Xlog:class+load`` shows a JDK class coming from the shared objects file), and an archive does not +serve a JVM with other settings. The archive is the JVM's own default one, so a missing or mismatched file only means a start without it; +the JDK AOT cache (``aot.py``, ``test_aot.py``) stays opt-in. +""" + +import subprocess # nosec B404 - runs the bundled JRE's java with fixed arguments +import sys + +from arcadedb import jvm + + +def _java(): + name = "java.exe" if sys.platform == "win32" else "java" + return str(jvm._extract_runtime_resource("jre") / "bin" / name) + + +def _flags(tmp_path): + """The JVM arguments ``start_jvm`` passes, with a heap that keeps compressed oops on whatever the RAM of this host is.""" + return jvm._build_jvm_args( + heap_size="1g", disable_xml_limits=False, jvm_args=None, log_dir=tmp_path + ) + + +def _run(*args): + return subprocess.run( # nosec B603 - the bundled java, arguments of this file + [_java(), *args], capture_output=True, text=True, timeout=300 + ) + + +def test_the_wheel_ships_the_archive(): + archives = list(jvm._extract_runtime_resource("jre").rglob("*.jsa")) + assert archives, "no .jsa file in the bundled JRE" + assert all(a.stat().st_size > 1_000_000 for a in archives) + + +def test_the_jvm_maps_the_archive_with_the_flags_the_wheel_starts_with(tmp_path): + flags = _flags(tmp_path) + assert "-XX:+UseCompactObjectHeaders" in flags + # -Xshare:on: no usable archive is a failed start, not a quiet start without it + result = _run(*flags, "-Xshare:on", "-version") + assert result.returncode == 0, result.stderr + result = _run(*flags, "-Xshare:auto", "-Xlog:class+load:stdout", "-version") + assert result.returncode == 0, result.stderr + assert "java.lang.Object source: shared objects file" in result.stdout + + +def test_the_default_start_does_not_turn_the_archive_off(tmp_path): + for flag in _flags(tmp_path): + assert not flag.startswith(("-Xshare:", "-XX:-UseSharedSpaces")) + + +def test_an_archive_does_not_serve_a_jvm_with_other_settings(tmp_path): + flags = [f for f in _flags(tmp_path) if f != "-XX:+UseCompactObjectHeaders"] + flags.append("-XX:-UseCompactObjectHeaders") + # the archive was dumped with compact object headers: without them -Xshare:on cannot use it, and a plain start goes on without it + assert _run(*flags, "-Xshare:on", "-version").returncode != 0 + result = _run(*flags, "-Xshare:auto", "-Xlog:class+load:stdout", "-version") + assert result.returncode == 0, result.stderr + assert "java.lang.Object source: shared objects file" not in result.stdout From 3d092217b98c0f491e0e02ad5933b5b679474fcb Mon Sep 17 00:00:00 2001 From: Taewoon Kim Date: Mon, 9 Nov 2026 19:02:46 +0900 Subject: [PATCH 11/49] Test: keep the nosec of the stale-record test on the line bandit reports Co-Authored-By: Claude Sonnet 5.5 --- tests/test_native_one_row_crossings.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/tests/test_native_one_row_crossings.py b/tests/test_native_one_row_crossings.py index 227404b2..f694445b 100644 --- a/tests/test_native_one_row_crossings.py +++ b/tests/test_native_one_row_crossings.py @@ -193,9 +193,8 @@ def test_a_stale_record_is_still_refused_before_anything_is_written(db): rid = _saved(db, a=1, b=1) mine = db.lookup_by_rid(rid).modify() with db.transaction(): - db.command( - "sql", f"UPDATE {rid} SET b = 9" - ).close() # nosec B608 - a record id this test read + bump_sql = f"UPDATE {rid} SET b = 9" # nosec B608 - a record id this test read + db.command("sql", bump_sql).close() mine.set("a", 2) with pytest.raises(ArcadeDBError, match="ConcurrentModificationException"): with db.transaction(): From aa4504a6b9ad472e6a3e9efefc4eeeecc8e6ecbd Mon Sep 17 00:00:00 2001 From: Taewoon Kim Date: Mon, 9 Nov 2026 19:04:02 +0900 Subject: [PATCH 12/49] JVM insert_columns: a column of str and None crosses as one UTF-8 blob plus int32 offsets (refs #324) A string column used to cross JPype one element at a time: convert_python_to_java per value and an Object[] of Java Strings built element by element, all alive at once. A column of nothing but str and None is now one byte[] (the UTF-8 text of all rows), one int[] of row + 1 offsets and, when there is a null, one boolean[], wrapped in the new bridge class StringColumn; DocumentBatcher.insertColumns and insertColumnsParallel decode each row when they build its document, so the Java Strings are short-lived. The blob crosses through bytes into JArray(JByte), a copy; nothing relies on a JPype view of Python memory. Kept as before: a mixed column (an int, bytes, a date-time) and an empty column convert per element; a column with a lone surrogate (no UTF-8 form) and one beyond an int offset keep the per-element path. insert_dataframe and insert_arrow reach the change through insert_columns. Stored values are the same on both wheels (tests compare with the per-element path and with the values given). Co-Authored-By: Claude Sonnet 5.5 --- src/arcadedb/core.py | 64 ++++++- .../com/arcadedb/python/DocumentBatcher.java | 14 +- .../com/arcadedb/python/StringColumn.java | 61 ++++++ tests/conftest.py | 1 + tests/test_insert_columns.py | 2 +- tests/test_insert_columns_string_blob.py | 177 ++++++++++++++++++ tests/test_insert_columns_strings.py | 87 +++++++++ 7 files changed, 397 insertions(+), 9 deletions(-) create mode 100644 src/java/com/arcadedb/python/StringColumn.java create mode 100644 tests/test_insert_columns_string_blob.py create mode 100644 tests/test_insert_columns_strings.py diff --git a/src/arcadedb/core.py b/src/arcadedb/core.py index c1777814..a1024e91 100644 --- a/src/arcadedb/core.py +++ b/src/arcadedb/core.py @@ -6,6 +6,7 @@ import gc import itertools +import operator from collections.abc import Mapping from datetime import date from decimal import Decimal @@ -186,13 +187,61 @@ def _java_class(name): return cls +_NONE_TYPE = type(None) +_STR_OR_NONE = frozenset((str, _NONE_TYPE)) + + +def _string_column(values): + """A column of ``str`` and ``None`` as one ``StringColumn`` (the UTF-8 text of all rows in one ``byte[]``, one ``int[]`` of row + 1 + offsets, and one ``boolean[]`` of the null rows when there is a null), or None when the column is not that: empty, or holding any + other type (a mixed column converts per element), or too large for an int offset, or a ``str`` with no UTF-8 form (a lone + surrogate), which the per-element path keeps handling as before. The blob crosses through ``bytes`` into ``JArray(JByte)``, a + copy; nothing relies on a JPype view of Python memory. The type census, the lengths and the join run in C. + """ + n = len(values) + if _np is None or not n: + return None + kinds = set(map(type, values)) + if str not in kinds or not kinds <= _STR_OR_NONE: + return None + mask = None + if _NONE_TYPE in kinds: + mask = _np.fromiter( + map(operator.is_, values, itertools.repeat(None)), dtype=_np.bool_, count=n + ) + values = list(values) # the caller's sequence is not touched + for i in _np.flatnonzero(mask).tolist(): + values[i] = "" + if all(map(str.isascii, values)): + blob = "".join(values).encode("ascii") + lengths = _np.fromiter(map(len, values), dtype=_np.int64, count=n) + else: + try: + encoded = list(map(str.encode, values)) + except UnicodeEncodeError: + return None + blob = b"".join(encoded) + lengths = _np.fromiter(map(len, encoded), dtype=_np.int64, count=n) + del encoded + if len(blob) > _JAVA_INT_MAX: + return None + offsets = _np.zeros(n + 1, dtype=_np.int32) + _np.cumsum(lengths, dtype=_np.int32, out=offsets[1:]) + return _java_class("com.arcadedb.python.StringColumn")( + jpype.JArray(jpype.JByte)(blob), + jpype.JArray(jpype.JInt)(offsets), + None if mask is None else jpype.JArray(jpype.JBoolean)(mask), + ) + + def _column_to_java(name, values): """One column of ``Database.insert_columns`` as ``(java array, length)``. A numpy array of an integer, float, or bool kind crosses as ONE buffer copy into a ``long[]``, ``double[]``, or ``boolean[]``; a string or object array - and any other sequence convert per element into an ``Object[]`` (a ``str`` - reuses one Java String per distinct value, ``None`` is a null). A pandas + and any other sequence of mixed values convert per element into an ``Object[]`` (a ``str`` + reuses one Java String per distinct value, ``None`` is a null); a column of nothing but ``str`` and ``None`` crosses instead as one + ``StringColumn`` (``_string_column``). A pandas ``Series`` is read through ``to_numpy``, with a nullable dtype's ```` as ``None``. A 2-D ``float32`` array is a column of vectors: it crosses as one ``float[]`` and each document gets one row of it. Kinds that do not cross natively raise ``TypeError`` rather @@ -265,6 +314,9 @@ def _column_to_java(name, values): values = list(values.astype(object, copy=False)) elif not isinstance(values, (list, tuple)): values = list(values) + column = _string_column(values) + if column is not None: + return column, len(values) converted = [] seen = {} for value in values: @@ -1082,7 +1134,10 @@ def insert_columns( bool as ``boolean[]``, one buffer copy each) or any sequence of Python values (``str``, ``int``, ``float``, ``bool``, ``None``, and the types ``insert_many``'s per-row fallback accepts), which - converts per element. ``None`` is a null; a float ``NaN`` in a + converts per element, except a sequence of nothing but ``str`` + and ``None``, which crosses as one UTF-8 buffer with its offsets + (a string with no UTF-8 form, a lone surrogate, keeps the per-element + path). ``None`` is a null; a float ``NaN`` in a numpy float column is stored as NaN, not as null (use a sequence with ``None`` for nulls). A pandas nullable column (``Int64``, ``string``) converts with its ```` as null. @@ -1211,7 +1266,8 @@ def insert_dataframe( takes that table directly and has the type mapping, the declared-type rules, and the failure contract). Each column becomes a typed array that crosses the bridge once per batch: integers, floats, and booleans - without nulls as one buffer copy, anything else per element. A document + without nulls as one buffer copy, a string column as one UTF-8 buffer + with its offsets, anything else per element. A document type gets documents; a vertex type gets vertices. An edge type is refused: load the vertices, then buffer the edges with ``graph_batch().new_edges()``. diff --git a/src/java/com/arcadedb/python/DocumentBatcher.java b/src/java/com/arcadedb/python/DocumentBatcher.java index bbcae4f8..793e0c26 100644 --- a/src/java/com/arcadedb/python/DocumentBatcher.java +++ b/src/java/com/arcadedb/python/DocumentBatcher.java @@ -99,15 +99,17 @@ public static long insertManyJson(final Database db, final String typeName, fina /** * Columnar insert: the documents are built Java-side from whole columns, so each column crosses the FFI once (a long[] or - * double[] copied from a numpy buffer, a boolean[], or an Object[] of Strings, boxed numbers, and nulls) instead of one + * double[] copied from a numpy buffer, a boolean[], a StringColumn (one UTF-8 byte[] plus the int[] offsets), or an Object[] of + * boxed numbers, Strings of a mixed column, and nulls) instead of one * JSON text per batch (bindings issue #150: 2.24x over insertManyJson on the first * 2,000,000 TPC-H SF1 line items, nine typed properties, commit every 10,000, same sums and count). * * Same failure contract as insertManyJson (#7882): batch commits and the failure rollback apply only to a transaction * opened here; a caller's own transaction is the caller's to commit or roll back. * - * @param columns one array per name, each of length n: long[], double[], boolean[] or Object[]; a null element sets - * the property to null, as a JSON null does on the insertManyJson path + * @param columns one array per name, each of length n: long[], double[], boolean[], Object[], a FloatVectors.Matrix or a + * StringColumn; a null element (or null row of a StringColumn) sets the property to null, as a JSON null does + * on the insertManyJson path */ public static long insertColumns(final Database db, final String typeName, final String[] names, final Object[] columns, final int n, final int commitEvery) { @@ -195,9 +197,11 @@ else if (col instanceof Object[] a) len = a.length; else if (col instanceof FloatVectors.Matrix m) len = m.rows(); + else if (col instanceof StringColumn s) + len = s.rows(); else throw new IllegalArgumentException("column '" + names[c] + "' is a " + (col == null ? "null" : col.getClass().getName()) - + ", not a long[], double[], boolean[], Object[] or FloatVectors.Matrix"); + + ", not a long[], double[], boolean[], Object[], FloatVectors.Matrix or StringColumn"); if (len != n) throw new IllegalArgumentException("column '" + names[c] + "' has " + len + " values for " + n + " rows"); } @@ -215,6 +219,8 @@ else if (col instanceof boolean[] a) v = a[i]; else if (col instanceof FloatVectors.Matrix m) v = m.row(i); + else if (col instanceof StringColumn s) + v = s.get(i); else v = ((Object[]) col)[i]; doc.set(names[c], v); diff --git a/src/java/com/arcadedb/python/StringColumn.java b/src/java/com/arcadedb/python/StringColumn.java new file mode 100644 index 00000000..0946f125 --- /dev/null +++ b/src/java/com/arcadedb/python/StringColumn.java @@ -0,0 +1,61 @@ +/* + * Python-bindings bridge: a string column of Database.insert_columns that crosses as one UTF-8 blob plus an int32 offsets array. + * + * The JPype guidance names one conversion per element as the slow way across the boundary, and an Object[] of Strings costs one + * conversion per row (and one Java String per distinct value, all alive at once). Here the whole column is ONE byte[] (the UTF-8 + * text of every row, back to back), ONE int[] of row + 1 offsets into it, and optionally ONE boolean[] marking the null rows. + * Row i is blob[offsets[i], offsets[i + 1]), decoded when the document of that row is built, so the Strings are short-lived. + * + * The decoded String is the one Python's own str holds: UTF-8 round-trips every code point, so a character beyond the BMP becomes + * its surrogate pair, U+0000 stays a NUL and an empty row is the empty String. Python sends no column it cannot encode (a lone + * surrogate has no UTF-8 form): that column keeps the per-element path. + * + * Compiled into arcadedb-python-bridge.jar during the wheel build and consumed by DocumentBatcher.insertColumns() and + * insertColumnsParallel(). + */ +package com.arcadedb.python; + +import java.nio.charset.StandardCharsets; + +public final class StringColumn { + private final byte[] blob; + private final int[] offsets; + private final boolean[] nulls; + private final int rows; + + /** + * @param blob the UTF-8 text of all rows, back to back + * @param offsets rows + 1 non-decreasing offsets into {@code blob}, the first 0 and the last at most {@code blob.length} + * @param nulls null for a column without a null, else one flag per row (a flagged row has an empty range and reads as null) + */ + public StringColumn(final byte[] blob, final int[] offsets, final boolean[] nulls) { + if (offsets.length < 1) + throw new IllegalArgumentException("a string column needs rows + 1 offsets, got none"); + this.rows = offsets.length - 1; + if (offsets[0] != 0) + throw new IllegalArgumentException("the first offset of a string column must be 0, got " + offsets[0]); + for (int i = 0; i < rows; i++) + if (offsets[i + 1] < offsets[i]) + throw new IllegalArgumentException("the offsets of a string column must not decrease (row " + i + ")"); + if (offsets[rows] > blob.length) + throw new IllegalArgumentException( + "the last offset " + offsets[rows] + " of a string column is beyond its " + blob.length + " bytes"); + if (nulls != null && nulls.length != rows) + throw new IllegalArgumentException("a string column has " + nulls.length + " null flags for " + rows + " rows"); + this.blob = blob; + this.offsets = offsets; + this.nulls = nulls; + } + + public int rows() { + return rows; + } + + /** The String of row {@code i}, or null for a null row. */ + public String get(final int i) { + if (nulls != null && nulls[i]) + return null; + final int from = offsets[i]; + return new String(blob, from, offsets[i + 1] - from, StandardCharsets.UTF_8); + } +} diff --git a/tests/conftest.py b/tests/conftest.py index ec3147c7..bd2a40e6 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -119,6 +119,7 @@ "test_query_vector_params_jvm.py": "the java.util objects the JVM build's parameter conversion builds", "test_insert_many_gc_pause.py": "the cyclic collector pause is the JVM build's (the native insert_many allocates nothing the collector tracks)", "test_graph_batch_tagged_shapes.py": "the JVM wheel's graph batch bulk path (bridge VertexBatcher and EdgeBatcher, the property-matrix fallback)", + "test_insert_columns_string_blob.py": "the bridge's StringColumn and the JVM wheel's core._column_to_java (JPype arrays, a spy on the per-element conversion)", "test_async_proxy_cache.py": "the JPype proxy classes of the JVM build's async executor", "test_read_crossings.py": "crossing counts of the JVM build's read path (JPype proxies of the Java objects and the bridge classes)", "test_reader_crossings.py": "crossing counts of the JVM build's record and row readers (JPype proxies of the Java objects and the bridge classes)", diff --git a/tests/test_insert_columns.py b/tests/test_insert_columns.py index 34ca2ada..482d1228 100644 --- a/tests/test_insert_columns.py +++ b/tests/test_insert_columns.py @@ -79,7 +79,7 @@ def test_nulls_in_a_sequence_column_are_nulls(self, temp_db): assert _count(temp_db, "Col", "WHERE label IS NULL") == 2 assert _count(temp_db, "Col", "WHERE label = 'c'") == 1 - def test_a_numpy_string_array_and_an_object_array_with_none_cross_per_element( + def test_a_numpy_string_array_and_an_object_array_with_none_are_stored( self, temp_db ): _make_type(temp_db) diff --git a/tests/test_insert_columns_string_blob.py b/tests/test_insert_columns_string_blob.py new file mode 100644 index 00000000..497e755b --- /dev/null +++ b/tests/test_insert_columns_string_blob.py @@ -0,0 +1,177 @@ +"""The JVM wheel sends a column of str and None as one UTF-8 blob plus int32 offsets, not as one JPype conversion per element. + +`_column_to_java` used to convert every element of a string column (`convert_python_to_java`, then an `Object[]` of Java Strings built +element by element). Now a column of nothing but `str` and `None` is one `byte[]`, one `int[]` of row + 1 offsets and, when there is +a null, one `boolean[]`, wrapped in the bridge's `StringColumn`, which `DocumentBatcher.insertColumns` decodes row by row as it builds +each document. The blob crosses through `bytes` into `JArray(JByte)`, a copy; no JPype view of Python memory is relied on. A mixed +column, an empty column, a column with a lone surrogate (no UTF-8 form) and a column too large for an int offset keep the +per-element path. These tests spy on which path ran (they fail on the code before) and compare the stored table with the one the +per-element path stores for the same columns. +""" + +import jpype +import pytest + +np = pytest.importorskip("numpy") + +from arcadedb import core # noqa: E402 + +TEXTS = [ + "plain", + "", + "emoji \U0001f600 and \U00010000 and \U0010ffff", + "embedded\x00nul", + "\x00", + "line
separator
", + "中文 한글 ퟿ é", + "x" * 5_000, +] + + +def _string_column_class(): + return jpype.JClass("com.arcadedb.python.StringColumn") + + +def _make_type(db, name): + db.command("sql", f"CREATE DOCUMENT TYPE {name}") + db.command("sql", f"CREATE PROPERTY {name}.id LONG") + db.command("sql", f"CREATE PROPERTY {name}.label STRING") + + +def _stored(db, name): + rows = db.query("sql", f"SELECT FROM {name} ORDER BY id").to_list() # nosec B608 + return [(r["id"], r.get("label")) for r in rows] + + +@pytest.mark.parametrize( + "values", + [ + ["a", "bb", ""], + ["a", None, "c"], + ("tuple", "of", "str"), + np.array(["x", "yy"]), + np.array(["x", None, "z"], dtype=object), + TEXTS, + ], + ids=["list", "list_with_none", "tuple", "numpy_str", "object_array", "edge_texts"], +) +def test_a_column_of_str_and_none_is_one_string_column(values): + array, n = core._column_to_java("label", values) + assert n == len(values) + assert isinstance(array, _string_column_class()) + + +@pytest.mark.parametrize( + "values", + [ + ["a", 1], + ["a", b"bytes"], + ["a", 1.5, None], + [1, 2, 3], + [None, None], + ["ok", "lone \ud800 surrogate"], + [], + ], + ids=[ + "str_int", + "str_bytes", + "str_float", + "ints", + "only_none", + "lone_surrogate", + "empty", + ], +) +def test_any_other_column_keeps_the_per_element_path(values): + array, n = core._column_to_java("label", values) + assert n == len(values) + assert not isinstance(array, _string_column_class()) + + +def test_a_string_column_makes_no_per_element_conversion(temp_db, monkeypatch): + _make_type(temp_db, "S") + calls = [] + real = core.convert_python_to_java + monkeypatch.setattr( + core, "convert_python_to_java", lambda v: calls.append(v) or real(v) + ) + n = 2_000 + label = [TEXTS[i % len(TEXTS)] if i % 7 else None for i in range(n)] + temp_db.insert_columns("S", {"id": np.arange(n, dtype=np.int64), "label": label}) + assert calls == [], "a string column was converted element by element" + assert _stored(temp_db, "S") == [(i, label[i]) for i in range(n)] + + +@pytest.mark.parametrize("parallel", [False, True], ids=["sync", "parallel"]) +def test_the_blob_stores_the_table_the_per_element_path_stores( + temp_db, monkeypatch, parallel +): + n = 4_000 + label = [ + None if i % 5 == 3 else TEXTS[i % len(TEXTS)] + str(i % 3) for i in range(n) + ] + cols = {"id": np.arange(n, dtype=np.int64), "label": label} + _make_type(temp_db, "Blob") + _make_type(temp_db, "Ref") + temp_db.insert_columns("Blob", cols, parallel=parallel) + with monkeypatch.context() as mp: + mp.setattr(core, "_string_column", lambda values: None) # the path before + array, _n = core._column_to_java("label", label) + assert not isinstance(array, _string_column_class()) + temp_db.insert_columns("Ref", cols, parallel=parallel) + assert _stored(temp_db, "Blob") == _stored(temp_db, "Ref") + assert _stored(temp_db, "Blob") == [(i, label[i]) for i in range(n)] + + +def test_a_lone_surrogate_column_is_handled_as_before(temp_db, monkeypatch): + # no UTF-8 form: the column takes the per-element path whatever that path does with it, so the outcome equals the old one + values = ["ok", "lone \ud800 surrogate", "fine"] + cols = {"id": np.arange(3, dtype=np.int64), "label": values} + _make_type(temp_db, "Blob") + _make_type(temp_db, "Ref") + outcomes = [] + for name, force_old in (("Blob", False), ("Ref", True)): + with monkeypatch.context() as mp: + if force_old: + mp.setattr(core, "_string_column", lambda values: None) + try: + temp_db.insert_columns(name, cols) + outcomes.append(("stored", _stored(temp_db, name))) + except ( + Exception + ) as e: # noqa: BLE001 - the type and message are what is compared + outcomes.append((type(e).__name__, str(e).replace(name, "T"))) + assert outcomes[0] == outcomes[1] + + +@pytest.mark.parametrize( + "blob, offsets, nulls", + [ + (b"ab", [1, 2], None), + (b"ab", [0, 2, 1], None), + (b"ab", [0, 3], None), + (b"ab", [0, 1, 2], [False]), + (b"ab", [], None), + ], + ids=[ + "first_not_zero", + "decreasing", + "past_the_end", + "short_null_flags", + "no_offsets", + ], +) +def test_the_java_side_refuses_a_malformed_column(blob, offsets, nulls): + with pytest.raises(Exception): # noqa: B017 - a Java IllegalArgumentException + _string_column_class()( + jpype.JArray(jpype.JByte)(blob), + jpype.JArray(jpype.JInt)(offsets), + None if nulls is None else jpype.JArray(jpype.JBoolean)(nulls), + ) + + +def test_a_column_beyond_an_int_offset_keeps_the_per_element_path(monkeypatch): + monkeypatch.setattr(core, "_JAVA_INT_MAX", 10) + array, n = core._column_to_java("label", ["0123456", "0123456"]) + assert n == 2 + assert not isinstance(array, _string_column_class()) diff --git a/tests/test_insert_columns_strings.py b/tests/test_insert_columns_strings.py new file mode 100644 index 00000000..fbaf6a23 --- /dev/null +++ b/tests/test_insert_columns_strings.py @@ -0,0 +1,87 @@ +"""insert_columns stores a string column exactly, on both wheels, whatever way it crosses. + +The JVM wheel sends a column of nothing but str and None as one UTF-8 blob plus int32 offsets (test_insert_columns_string_blob.py); +the native wheel reads the column in place. A user must not see a difference: a character beyond the BMP, a NUL, a line separator, +an empty string and a null are stored as they were given, in one column and in a mixed column, in the synchronous and the parallel +mode, from a list, a numpy string array and an object array. +""" + +import pytest + +np = pytest.importorskip("numpy") + +TEXTS = [ + "plain", + "", + "emoji \U0001f600 and \U00010000 and \U0010ffff", + "embedded\x00nul", + "\x00", + "line
separator
", + "中文 한글 ퟿ é", + "x" * 5_000, + 'tab\tnewline\nquote"backslash\\', +] + + +def _make_type(db, name): + db.command("sql", f"CREATE DOCUMENT TYPE {name}") + db.command("sql", f"CREATE PROPERTY {name}.id LONG") + db.command("sql", f"CREATE PROPERTY {name}.label STRING") + db.command("sql", f"CREATE PROPERTY {name}.other STRING") + + +def _stored(db, name): + rows = db.query("sql", f"SELECT FROM {name} ORDER BY id").to_list() # nosec B608 + return [(r["id"], r.get("label"), r.get("other")) for r in rows] + + +def _columns(n): + label = [TEXTS[i % len(TEXTS)] for i in range(n)] + other = [None if i % 4 == 1 else f"o{i}" for i in range(n)] + return {"id": np.arange(n, dtype=np.int64), "label": label, "other": other} + + +@pytest.mark.parametrize("parallel", [False, True], ids=["sync", "parallel"]) +def test_strings_are_stored_as_given(temp_db, parallel): + _make_type(temp_db, "S") + cols = _columns(3_000) + assert temp_db.insert_columns("S", cols, parallel=parallel) == 3_000 + assert _stored(temp_db, "S") == [ + (i, cols["label"][i], cols["other"][i]) for i in range(3_000) + ] + + +@pytest.mark.parametrize("kind", ["numpy_str", "object_array"]) +def test_other_containers_of_strings_are_stored_as_given(temp_db, kind): + _make_type(temp_db, "S") + # a numpy "U" array drops a trailing NUL by itself, so its texts have none + texts = [t for t in TEXTS if "\x00" not in t] + label = [texts[i % len(texts)] for i in range(40)] + column = np.array(label) if kind == "numpy_str" else np.array(label, dtype=object) + expected = label + temp_db.insert_columns("S", {"id": np.arange(40, dtype=np.int64), "label": column}) + assert [row[1] for row in _stored(temp_db, "S")] == expected + + +def test_a_mixed_column_keeps_every_value(temp_db): + _make_type(temp_db, "S") + temp_db.command("sql", "CREATE PROPERTY S.n LONG") + labels = ["a", "emoji \U0001f600", None, "", "z"] + temp_db.insert_columns( + "S", + { + "id": np.arange(5, dtype=np.int64), + "label": labels, + "n": [1, None, 3, 4, 5], + }, + ) + assert [row[1] for row in _stored(temp_db, "S")] == labels + assert _stored(temp_db, "S")[0][0] == 0 + + +def test_a_single_row_column(temp_db): + _make_type(temp_db, "S") + temp_db.insert_columns( + "S", {"id": np.array([10], dtype=np.int64), "label": ["one"]} + ) + assert _stored(temp_db, "S") == [(10, "one", None)] From 8d4ec4a4e5f8e0f49efed8faee8db31c1a53b838 Mon Sep 17 00:00:00 2001 From: Taewoon Kim Date: Mon, 9 Nov 2026 19:07:55 +0900 Subject: [PATCH 13/49] tests: nosec B608 on the test-owned count query, which bandit flagged after black wrapped it Co-Authored-By: Claude Sonnet 5.5 --- tests/test_native_columnar_parallel.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/tests/test_native_columnar_parallel.py b/tests/test_native_columnar_parallel.py index 0f5d80d9..90d6f79a 100644 --- a/tests/test_native_columnar_parallel.py +++ b/tests/test_native_columnar_parallel.py @@ -45,9 +45,8 @@ def spy(monkeypatch): def _count(db, name): - return db.query("sql", f"SELECT count(*) AS n FROM {name}").to_list()[0][ - "n" - ] # nosec B608 - test-owned type name + sql = f"SELECT count(*) AS n FROM {name}" # nosec B608 - test-owned type name + return db.query("sql", sql).to_list()[0]["n"] def _types(db, *names, kind="DOCUMENT", buckets=3, unique=False): From 7fddca0a65e864346bc8d12d81a9b8e3fa7feaa4 Mon Sep 17 00:00:00 2001 From: Taewoon Kim Date: Mon, 9 Nov 2026 20:02:08 +0900 Subject: [PATCH 14/49] Tests: bandit annotations for the seeded corpus and the test-controlled query (refs #318) Co-Authored-By: Claude Sonnet 5.5 --- tests/test_bulk_single_pass_validation.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/tests/test_bulk_single_pass_validation.py b/tests/test_bulk_single_pass_validation.py index 2610c6d8..5d9afe72 100644 --- a/tests/test_bulk_single_pass_validation.py +++ b/tests/test_bulk_single_pass_validation.py @@ -172,7 +172,7 @@ def test_random_rows_agree_with_the_validation_loop(tagged): dt.datetime(2026, 1, 2), dt.date(2026, 1, 2), decimal.Decimal("1.5"), b"ab", np.int64(7), np.float64(math.nan), Colour.RED, Name("s"), {1, 2}, object(), ] # fmt: skip - rnd = random.Random(318) + rnd = random.Random(318) # nosec B311 - a seeded corpus, not security new = tc.json_bulk_dumps_tagged if tagged else tc.json_bulk_dumps for _ in range(3000): rows = [] @@ -313,7 +313,7 @@ def test_a_value_that_may_be_bad_runs_the_walk(monkeypatch, row): def test_int64_overflow_pattern_matches_exactly_the_tokens_from_2_to_the_63(): - rnd = random.Random(63) + rnd = random.Random(63) # nosec B311 - a seeded corpus, not security values = [ 0, 1, @@ -348,7 +348,8 @@ def _types(db): def _stored(db, type_name): - rows = db.query("sql", f"SELECT FROM {type_name}").to_list() + query = f"SELECT FROM {type_name}" # nosec B608 - test-controlled + rows = db.query("sql", query).to_list() return sorted( ( r["p_partkey"], From 1ff1e64365a66b9751854a2e1ea736f577b23e3a Mon Sep 17 00:00:00 2001 From: Taewoon Kim Date: Mon, 9 Nov 2026 20:51:56 +0900 Subject: [PATCH 15/49] Native frames: a frame that ends before the replayed first-reply rows keeps the rest for the next call With batch_size=64 the first frame took 64 of the 256 replayed rows and the cursor then dropped the other 192 (test_to_arrow_multi_batch: 108 rows of 300). The library now puts back the unread tail of the head after each frame. Also: the reference test named a property "by" (an SQL keyword), and the dataframe test expected None where pandas 3 shows nan for a missing string. Co-Authored-By: Claude Sonnet 5.5 --- .../src/main/java/org/humemai/nativelib/ArcadeNative.java | 5 ++++- .../src/main/java/org/humemai/nativelib/ColumnFrames.java | 7 +++++-- tests/test_column_frames.py | 7 ++++++- tests/test_native_column_frames_reference.py | 6 +++--- 4 files changed, 18 insertions(+), 7 deletions(-) diff --git a/native/src/main/java/org/humemai/nativelib/ArcadeNative.java b/native/src/main/java/org/humemai/nativelib/ArcadeNative.java index 52f9d3c5..836e40a7 100644 --- a/native/src/main/java/org/humemai/nativelib/ArcadeNative.java +++ b/native/src/main/java/org/humemai/nativelib/ArcadeNative.java @@ -337,7 +337,7 @@ public static CCharPointer nextBin(final IsolateThread thread, final long rsId, * the negated byte count of the error JSON and that JSON. max: rows at most (a frame also ends at about 1 MiB of values). columns: a * JSON array of the column names to read, or "" for the union of the names of the rows. flags bit 0: datetimes as microseconds (the * to_dataframe form). skip: of the rows that the first reply of this cursor already held (HEADS), the number the caller has taken; the - * first frame starts after them. The frame that ends the rows closes the cursor. The reply is freed with arcadedb_free. + * first frame starts after them, and a frame that ends before those rows do leaves the rest in HEADS for the next call (skip 0). The frame that ends the rows closes the cursor. The reply is freed with arcadedb_free. */ @CEntryPoint(name = "arcadedb_n_next_col") @SuppressWarnings("unchecked") @@ -361,6 +361,9 @@ public static CCharPointer nextCol(final IsolateThread thread, final long rsId, if (frame.done) { removeCursor(rsId); closeQuietly(rs); + } else if (head != null && frame.headUsed < head.size()) { + // the frame ended (batch_size rows or about 1 MiB) before the replayed rows did: the next call starts with the rest + HEADS.put(rsId, new ArrayList<>(head.subList(frame.headUsed, head.size()))); } return frameOut(frame.bytes.length, frame.bytes); } catch (final Throwable t) { diff --git a/native/src/main/java/org/humemai/nativelib/ColumnFrames.java b/native/src/main/java/org/humemai/nativelib/ColumnFrames.java index 0c507346..a45aad38 100644 --- a/native/src/main/java/org/humemai/nativelib/ColumnFrames.java +++ b/native/src/main/java/org/humemai/nativelib/ColumnFrames.java @@ -57,10 +57,13 @@ private ColumnFrames() { static final class Frame { final byte[] bytes; final boolean done; + /** The index in {@code head} of the first row this frame did not take (a frame may end before the head does). */ + final int headUsed; - Frame(final byte[] bytes, final boolean done) { + Frame(final byte[] bytes, final boolean done, final int headUsed) { this.bytes = bytes; this.done = done; + this.headUsed = headUsed; } } @@ -377,6 +380,6 @@ else if (type.equals("dtu")) out.putInt(headerBytes.length); out.put(headerBytes); out.put(bodyBytes); - return new Frame(out.array(), done); + return new Frame(out.array(), done, h); } } diff --git a/tests/test_column_frames.py b/tests/test_column_frames.py index 0265a17a..7956b15a 100644 --- a/tests/test_column_frames.py +++ b/tests/test_column_frames.py @@ -14,6 +14,7 @@ from datetime import datetime import numpy as np +import pandas as pd import pyarrow as pa import pytest @@ -328,7 +329,11 @@ def test_to_dataframe_and_to_polars(table): df = db.query("sql", _sel()).to_dataframe() assert list(df.columns) == [n for n, _ in TYPES] assert list(df["big"]) == [r["big"] for r in rows] - assert df["s"].tolist() == [r["s"] for r in rows] + # pandas decides how a missing string is shown (None in an object column, nan in the string dtype of pandas 3): the same on + # both wheels, since both build the frame from a list with None + assert [None if pd.isna(v) else v for v in df["s"].tolist()] == [ + r["s"] for r in rows + ] assert [list(v) for v in df["v3"]] == [r["v3"] for r in rows] assert df["ts"].iloc[1] == datetime(2026, 1, 2, 3, 4, 5, 1000) import polars as pl diff --git a/tests/test_native_column_frames_reference.py b/tests/test_native_column_frames_reference.py index 19b46cab..630a4390 100644 --- a/tests/test_native_column_frames_reference.py +++ b/tests/test_native_column_frames_reference.py @@ -207,7 +207,7 @@ def _row(i): "n": None if null else i, "big": (1 << 40) + i, "sh": None if null else (i % 2000) - 1000, - "by": None if null else (i % 200) - 100, + "bt": None if null else (i % 200) - 100, "d": i / 3, "f": None if null else _f32(i / 7), "s": None if null else f"row {i} \U0001f600 \x00 \u00e9", @@ -229,7 +229,7 @@ def _row(i): ("n", "INTEGER"), ("big", "LONG"), ("sh", "SHORT"), - ("by", "BYTE"), + ("bt", "BYTE"), ("d", "DOUBLE"), ("f", "FLOAT"), ("s", "STRING"), @@ -330,7 +330,7 @@ def _assert_frame_equal(new, old_rows, columns=None): ("full", "SELECT FROM R ORDER BY big", None), ( "projection", - "SELECT n, big, sh, by, d, f, s, ok, ts, day, m FROM R ORDER BY big", + "SELECT n, big, sh, bt, d, f, s, ok, ts, day, m FROM R ORDER BY big", None, ), ("vectors", "SELECT big, v1, v3, v9, v128 FROM R ORDER BY big", None), From 028e8776c11d1c04fe97aae59f2bc070cc6b06b2 Mon Sep 17 00:00:00 2001 From: Taewoon Kim Date: Mon, 9 Nov 2026 21:15:58 +0900 Subject: [PATCH 16/49] tests: ingest integrity, count and digest checks after every ingest path, with mutation proofs (refs #336) One check (tests/ingest_integrity.py) compares the stored type with the rows that were sent: the count read three ways, an order-independent digest of every column read two ways, every key exactly once, and all of it again after close and reopen. test_ingest_integrity.py runs it for insert_many (list and generator), insert_columns, insert_arrow, insert_dataframe, create_vertices, async create_record and command, append_samples, each sequential and parallel where it exists, at every chunk and commit boundary, for edges (endpoint pairs and weights), several writer threads with a reader thread, a bad row mid-batch with the documented outcome exact, rejected and raising async callbacks, and close straight after the last call. Every family also runs with a rows-dropped, duplicated, altered or uncommitted fault and must report it. The native and JVM seams are in their own files. Co-Authored-By: Claude Sonnet 5.5 --- tests/conftest.py | 30 + tests/ingest_integrity.py | 220 +++++ tests/test_ingest_integrity.py | 988 +++++++++++++++++++ tests/test_jvm_ingest_integrity_faults.py | 54 + tests/test_native_ingest_integrity_faults.py | 39 + 5 files changed, 1331 insertions(+) create mode 100644 tests/ingest_integrity.py create mode 100644 tests/test_ingest_integrity.py create mode 100644 tests/test_jvm_ingest_integrity_faults.py create mode 100644 tests/test_native_ingest_integrity_faults.py diff --git a/tests/conftest.py b/tests/conftest.py index ec3147c7..b260e308 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -83,6 +83,8 @@ collect_ignore.append("test_native_point_op_glue.py") # the address table of append_samples (arcadedb_n_async_bin), observed on the native library collect_ignore.append("test_native_timeseries_binary.py") + # the native columnar fast path (arcadedb._native.bulk.insert_columns_fast) with a rows-dropped fault in it, found by the ingest-integrity check (#336) + collect_ignore.append("test_native_ingest_integrity_faults.py") if not os.path.isfile( os.path.join(os.path.dirname(os.path.dirname(__file__)), "docs", "index.md") ): @@ -122,6 +124,7 @@ "test_async_proxy_cache.py": "the JPype proxy classes of the JVM build's async executor", "test_read_crossings.py": "crossing counts of the JVM build's read path (JPype proxies of the Java objects and the bridge classes)", "test_reader_crossings.py": "crossing counts of the JVM build's record and row readers (JPype proxies of the Java objects and the bridge classes)", + "test_jvm_ingest_integrity_faults.py": "rows-dropped faults put into the JVM build's column conversion and bulk vertex path (core._column_to_java, GraphBatch._create_vertices_json_bulk)", } if BACKEND == "native" and not os.environ.get("NATIVE_PARITY_ALL"): collect_ignore.extend(NATIVE_NOT_APPLICABLE) @@ -521,3 +524,30 @@ def pytest_sessionfinish(session, exitstatus): def pytest_terminal_summary(terminalreporter, exitstatus, config): """Terminal summary hook.""" pass + + +@pytest.fixture +def store(temp_db_path): + """A database on disk that a test can close and open again (the ingest-integrity tests, ArcadeData/arcadedb-python#336).""" + from tests.ingest_integrity import Store + + box = Store(temp_db_path) + yield box + box.close() + + +@pytest.fixture +def small_chunks(monkeypatch): + """insert_many reads 7 rows at a time and a graph batch crosses 7 rows at a time (the real chunks are 10,000 and 100,000).""" + from tests.ingest_integrity import CHUNK + + if BACKEND == "native": + from arcadedb._native import database, extras + + monkeypatch.setattr(database, "_INSERT_MANY_CHUNK", CHUNK) + monkeypatch.setattr(extras.NativeGraphBatch, "_BULK_CHUNK", CHUNK) + else: + from arcadedb import core + + monkeypatch.setattr(core, "_INSERT_MANY_CHUNK", CHUNK) + monkeypatch.setattr(arcadedb.GraphBatch, "_BULK_CHUNK", CHUNK) diff --git a/tests/ingest_integrity.py b/tests/ingest_integrity.py new file mode 100644 index 00000000..d41d072f --- /dev/null +++ b/tests/ingest_integrity.py @@ -0,0 +1,220 @@ +"""The one check that every ingest-integrity test calls (ArcadeData/arcadedb-python#336). + +An ingest call must not lose, duplicate or silently alter a row. After a call, ``problems_for_type`` checks three things on the +stored type, each read back through the public API: + +1. the record count, read three ways (``SELECT count(*)``, ``Database.count_type`` and the length of the rows that come back), +2. an order-independent digest of every column: the sum and the xor of a stable 64-bit hash of each row, compared with the digest + of the rows that were sent (two row readers, ``to_list()`` and iterating the result set, must each match it), +3. for a key column, every sent key present exactly once and no other key. + +``Store.reopen`` closes the database right after the call and opens it again from disk, and the same three checks run once more +(durability). A check never looks at what the path under test says it did: the expected side is the data the test generated. + +``MUTANTS`` are the faults the mutation tests inject into the rows a path is given (drop the last row, skip a middle chunk, store +one row twice, change one value, swap two values). A test family is proven able to fail when the check reports each of them. +""" + +import collections +import hashlib + +import arcadedb + +MASK = (1 << 64) - 1 + +CHUNK = 7 # rows per crossing, made small so every boundary is cheap +COMMIT = 5 # commit_every and batch_size + +#: every problem line carries this word, so a test can tell "the check found it" from "the call raised" +FOUND = "MISMATCH" + + +class IntegrityError(AssertionError): + """The stored records are not the sent records.""" + + +def canon(kind, value): + """The canonical text of one cell. ``kind`` is how the property is declared, so 5 and 5.0 are the same DOUBLE.""" + if value is None: + return "N" + if kind == "int": + return "i%d" % int(value) + if kind == "float": + number = float(value) + return "nan" if number != number else "f" + number.hex() + if kind == "bool": + return "b1" if value else "b0" + if kind == "str": + return "s" + str(value) + raise ValueError(f"no canonical form for kind {kind!r}") + + +def row_hash(schema, row): + """A stable 64-bit hash of one row: the column names sorted, each cell length-prefixed, a missing cell the same as a null.""" + parts = [] + for name in sorted(schema): + text = canon(schema[name], row.get(name)) + parts.append("%s:%d:%s" % (name, len(text), text)) + data = "|".join(parts).encode("utf-8", "surrogatepass") + return int.from_bytes(hashlib.blake2b(data, digest_size=8).digest(), "big") + + +def digest(schema, rows): + """``(row count, sum of the row hashes mod 2**64, xor of the row hashes)``: the same for any order of the rows.""" + count = total = mixed = 0 + for row in rows: + h = row_hash(schema, row) + count += 1 + total = (total + h) & MASK + mixed ^= h + return count, total, mixed + + +def _key_problems(label, view, schema, key, want_rows, rows): + kind = schema[key] + wanted = collections.Counter(canon(kind, r.get(key)) for r in want_rows) + stored = collections.Counter(canon(kind, r.get(key)) for r in rows) + missing = sorted(set(wanted) - set(stored)) + extra = sorted(set(stored) - set(wanted)) + twice = sorted(k for k, c in stored.items() if c > 1 and wanted.get(k, 0) < c) + problems = [] + if missing: + problems.append( + f"{FOUND} {label}: {len(missing)} sent key(s) not stored, first {missing[:3]} ({view})" + ) + if extra: + problems.append( + f"{FOUND} {label}: {len(extra)} stored key(s) were never sent, first {extra[:3]} ({view})" + ) + if twice: + problems.append( + f"{FOUND} {label}: {len(twice)} key(s) stored more than once, first {twice[:3]} ({view})" + ) + return problems + + +def verify(label, want_rows, schema, key, counts, readers): + """The problems (a list of text lines, empty when none) between the sent rows and what ``counts`` and ``readers`` saw.""" + want = digest(schema, want_rows) + problems = [] + for name, value in counts.items(): + if value != want[0]: + problems.append( + f"{FOUND} {label}: {name} says {value} record(s), {want[0]} were sent" + ) + first = True + for name, rows in readers.items(): + got = digest(schema, rows) + if got[0] != want[0]: + problems.append( + f"{FOUND} {label}: reading with {name} returned {got[0]} row(s), {want[0]} were sent" + ) + elif got != want: + problems.append( + f"{FOUND} {label}: reading with {name} returned {got[0]} row(s) whose digest differs from the sent data" + f" (a stored value changed)" + ) + if key is not None and first: + problems += _key_problems(label, name, schema, key, want_rows, rows) + first = False + return problems + + +def type_view(db, name, schema): + """What the public API reports for one document, vertex or time-series type: the counts and the rows two ways.""" + names = list(schema) + select = "SELECT FROM " + name # nosec B608 + count_sql = "SELECT count(*) AS n FROM " + name # nosec B608 + counts = { + "SELECT count(*)": int(db.query("sql", count_sql).to_list()[0]["n"]), + "count_type()": int(db.count_type(name)), + } + listed = db.query("sql", select).to_list() + iterated = [{n: r.get(n) for n in names} for r in db.query("sql", select)] + return counts, {"to_list()": listed, "iterating the result": iterated} + + +def problems_for_type(db, name, want_rows, schema, key, label): + counts, readers = type_view(db, name, schema) + return verify(label, want_rows, schema, key, counts, readers) + + +EDGE_SCHEMA = {"a": "int", "b": "int", "w": "float"} + + +def edge_view(db, edge_type): + """The counts of an edge type, and its edges as (source key, destination key, weight) rows.""" + count_sql = "SELECT count(*) AS n FROM " + edge_type # nosec B608 + cypher_count = "MATCH ()-[r:" + edge_type + "]->() RETURN count(r) AS c" + pairs = "MATCH (a)-[r:" + edge_type + "]->(b) RETURN a.k AS a, b.k AS b, r.w AS w" + counts = { + "SELECT count(*)": int(db.query("sql", count_sql).to_list()[0]["n"]), + "Cypher count(r)": int(db.query("opencypher", cypher_count).to_list()[0]["c"]), + } + return counts, {"Cypher MATCH": db.query("opencypher", pairs).to_list()} + + +def problems_for_edges(db, edge_type, want_rows, label): + counts, readers = edge_view(db, edge_type) + return verify(label, want_rows, EDGE_SCHEMA, None, counts, readers) + + +class Store: + """A database on disk that a test can close and open again.""" + + def __init__(self, path): + self.path = path + self.db = arcadedb.create_database(path) + + def reopen(self): + """Close straight after the last call and open the same files again.""" + self.db.close() + self.db = arcadedb.open_database(self.path) + + def close(self): + db, self.db = self.db, None + if db is not None and db.is_open(): + db.close() + + +def _mutant_drop_last(rows): + return rows[:-1] + + +def _mutant_drop_first(rows): + return rows[1:] + + +def _mutant_skip_a_chunk(rows): + third = max(1, len(rows) // 3) + return rows[:third] + rows[2 * third :] + + +def _mutant_duplicate_one(rows): + return rows + [dict(rows[len(rows) // 2])] + + +def _mutant_alter_one_value(rows): + out = [dict(r) for r in rows] + out[len(out) // 2]["val"] = out[len(out) // 2]["val"] + 0.5 + return out + + +def _mutant_swap_two_values(rows): + out = [dict(r) for r in rows] + a, b = 1, len(out) - 2 + out[a]["val"], out[b]["val"] = out[b]["val"], out[a]["val"] + return out + + +MUTANTS = { + "drop_the_last_row": _mutant_drop_last, + "drop_the_first_row": _mutant_drop_first, + "skip_a_chunk": _mutant_skip_a_chunk, + "store_one_row_twice": _mutant_duplicate_one, + "change_one_value": _mutant_alter_one_value, + "swap_two_values": _mutant_swap_two_values, +} + +#: the durability mutant: the rows go into a transaction nobody commits, so they are visible in this session and gone after a reopen +NO_COMMIT = "leave_the_transaction_open" diff --git a/tests/test_ingest_integrity.py b/tests/test_ingest_integrity.py new file mode 100644 index 00000000..006d992c --- /dev/null +++ b/tests/test_ingest_integrity.py @@ -0,0 +1,988 @@ +"""Ingest integrity (ArcadeData/arcadedb-python#336): no ingest path may lose, duplicate or silently alter a row, and a test must be +able to catch it. + +``tests/ingest_integrity.py`` holds the one check. After every ingest call it compares the stored type with the rows that were sent: +the record count (read three ways), an order-independent digest of every column (read two ways), the key of every row exactly once, +and all of it again after the database is closed straight after the call and opened from disk. Both wheels run this file: the API is +the same, only the seams in ``test_native_ingest_integrity_faults.py`` and ``test_jvm_ingest_integrity_faults.py`` differ. + +* ``test_every_size_at_every_boundary`` loads each ingest path (``insert_many`` from a list and from a generator, ``insert_columns``, + ``insert_arrow``, ``insert_dataframe``, ``graph_batch().create_vertices``, ``create_record`` and ``command`` on the async executor, + ``append_samples``; each sequential and, where it exists, with ``parallel=True``) at 0, 1 and 2 rows and one either side of every + multiple of the chunk (7 rows) and of ``commit_every`` / ``batch_size`` (5 rows), and at 255, 256 and 257 rows (the first reply of + the native read path). The chunk sizes are made small so the boundaries are cheap; one load at the real 100,000 chunk is separate. +* the edge, thread, failure, async and ``close()`` tests below check the cases the issue lists, with the documented outcome exact. +* ``test_mutation_*`` run the same code with a rows-dropped (or duplicated, or altered, or uncommitted) fault in the path and assert that the check + reports it, so each family is shown able to fail. + +``wait_completion()`` can return while the executor still reports itself as processing (#210), so no test here asserts idleness: +they assert the stored data. +""" + +import dataclasses +import threading +import time +from typing import Any, Callable, Optional + +import numpy as np +import pandas as pd +import pyarrow as pa +import pytest + +import arcadedb +from arcadedb.exceptions import ArcadeDBError +from tests.ingest_integrity import ( + CHUNK, + COMMIT, + FOUND, + MUTANTS, + NO_COMMIT, + IntegrityError, + problems_for_edges, + problems_for_type, + verify, +) + +FIRST_REPLY = 256 # rows in the first reply of the native read path +MUTATION_SIZE = 25 + +SCHEMA = {"k": "int", "val": "float", "flag": "bool", "txt": "str"} +PROPS = {"k": "LONG", "val": "DOUBLE", "flag": "BOOLEAN", "txt": "STRING"} +TS_SCHEMA = {"host": "str", "val": "float"} + + +def ladder(units, extra=()): + sizes = {0, 1, 2, *extra} + for unit in units: + for k in (1, 2, 3): + sizes.update((k * unit - 1, k * unit, k * unit + 1)) + return sorted(sizes) + + +SIZES = ladder((CHUNK, COMMIT), extra=(FIRST_REPLY - 1, FIRST_REPLY, FIRST_REPLY + 1)) + + +def make_rows(n, start=0): + return [ + { + "k": i, + "val": (i * 37 % 1009) / 7.0 + i, + "flag": i % 3 == 0, + "txt": None if i % 7 == 3 else "row-%d-%s" % (i, "é" * (i % 4)), + } + for i in range(start, start + n) + ] + + +def make_ts_rows(n): + return [{"host": "h%d" % (i % 3), "val": float(i)} for i in range(n)] + + +def make_type(db, name, kind="DOCUMENT", buckets=None, unique=False): + ddl = f"CREATE {kind} TYPE {name}" + if buckets: + ddl += f" BUCKETS {buckets}" + db.command("sql", ddl) + for prop, declared in PROPS.items(): + db.command("sql", f"CREATE PROPERTY {name}.{prop} {declared}") + if unique: + db.command("sql", f"CREATE INDEX ON {name} (k) UNIQUE") + + +def make_ts_type(db, name): + db.command( + "sql", + f"CREATE TIMESERIES TYPE {name} TIMESTAMP ts TAGS (host STRING) FIELDS (val DOUBLE) SHARDS 2", + ) + + +# -- the ingest paths --------------------------------------------------------------------------------------------------------- + + +def _numpy_columns(rows): + return { + "k": np.array([r["k"] for r in rows], dtype=np.int64), + "val": np.array([r["val"] for r in rows], dtype=np.float64), + "flag": np.array([r["flag"] for r in rows], dtype=np.bool_), + "txt": [r["txt"] for r in rows], + } + + +def _list_columns(rows): + return {n: [r[n] for r in rows] for n in SCHEMA} + + +ARROW_SCHEMA = pa.schema( + [ + ("k", pa.int64()), + ("val", pa.float64()), + ("flag", pa.bool_()), + ("txt", pa.string()), + ] +) + + +def _arrow(rows): + return pa.Table.from_pylist(list(rows), schema=ARROW_SCHEMA) + + +def _frame(rows): + return pd.DataFrame( + { + "k": np.array([r["k"] for r in rows], dtype=np.int64), + "val": np.array([r["val"] for r in rows], dtype=np.float64), + "flag": np.array([r["flag"] for r in rows], dtype=np.bool_), + "txt": pd.array([r["txt"] for r in rows], dtype="string"), + } + ) + + +def _doc_of(db, name, row): + doc = db.new_document(name) + for key, value in row.items(): + if value is not None: + doc.set(key, value) + return doc + + +@dataclasses.dataclass +class Ctx: + """The async executor of a database (there is one per database) and what its executor-wide handler was told.""" + + ex: Any + errors: list + + +def _executor(level=None): + def setup(db): + ex = db.async_executor() + if level: + ex.set_parallel_level(level) + ex.set_commit_every(COMMIT) + errors = [] + ex.on_error(errors.append) + return Ctx(ex, errors) + + return setup + + +def _no_setup(db): + return None + + +def _teardown(ctx): + if ctx is not None: + ctx.ex.wait_completion() + ctx.ex.close() + + +def _ing_many(ctx, db, name, rows): + return db.insert_many(name, list(rows), commit_every=COMMIT) + + +def _ing_many_gen(ctx, db, name, rows): + return db.insert_many(name, (r for r in rows), commit_every=COMMIT) + + +def _ing_many_par(ctx, db, name, rows): + return db.insert_many(name, list(rows), parallel=True) + + +def _ing_cols(ctx, db, name, rows): + return db.insert_columns(name, _numpy_columns(rows), commit_every=COMMIT) + + +def _ing_cols_par(ctx, db, name, rows): + return db.insert_columns(name, _numpy_columns(rows), parallel=True) + + +def _ing_arrow(ctx, db, name, rows): + return db.insert_arrow(name, _arrow(rows), batch_size=COMMIT) + + +def _ing_arrow_par(ctx, db, name, rows): + return db.insert_arrow(name, _arrow(rows), batch_size=COMMIT, parallel=True) + + +def _ing_df(ctx, db, name, rows): + return db.insert_dataframe(name, _frame(rows), batch_size=COMMIT) + + +def _ing_df_par(ctx, db, name, rows): + return db.insert_dataframe(name, _frame(rows), batch_size=COMMIT, parallel=True) + + +def _ing_async_create(ctx, db, name, rows): + for row in rows: + ctx.ex.create_record(_doc_of(db, name, row)) + ctx.ex.wait_completion() + return len(rows) + + +def _ing_async_command(ctx, db, name, rows): + head = "INSERT INTO " + name + with_text = head + " SET k = :k, val = :val, flag = :flag, txt = :txt" + without_text = head + " SET k = :k, val = :val, flag = :flag" + for row in rows: + if row["txt"] is None: + ctx.ex.command( + "sql", without_text, k=row["k"], val=row["val"], flag=row["flag"] + ) + else: + ctx.ex.command("sql", with_text, **row) + ctx.ex.wait_completion() + return len(rows) + + +def _ing_vertices(ctx, db, name, rows): + with db.graph_batch(commit_every=CHUNK, batch_size=CHUNK) as batch: + rids = batch.create_vertices(name, [dict(r) for r in rows]) + if len(rids) != len(rows) or len(set(rids)) != len(rids): + raise AssertionError( + f"create_vertices returned {len(rids)} RIDs ({len(set(rids))} distinct) for {len(rows)} rows" + ) + return len(rids) + + +def _ing_timeseries(ctx, db, name, rows): + n = len(rows) + ts = (np.arange(n, dtype=np.int64) + 1) * 1000 + ctx.ex.append_samples( + name, + ts, + [r["host"] for r in rows], + np.array([r["val"] for r in rows], dtype=np.float64), + ) + ctx.ex.wait_completion() + return n + + +@dataclasses.dataclass(frozen=True) +class Family: + id: str + create: Callable + ingest: Callable + setup: Callable = _no_setup + schema: Optional[dict] = None + rows: Callable = make_rows + key: str = "k" + min_n: int = 0 + sync_tx: bool = ( + False # joins a transaction the caller opened (the durability mutant needs one) + ) + + def __post_init__(self): + if self.schema is None: + object.__setattr__(self, "schema", SCHEMA) + + +def _doc(buckets=None): + return lambda db, name: make_type(db, name, buckets=buckets) + + +def _vertex(db, name): + make_type(db, name, kind="VERTEX") + + +FAMILIES = [ + Family("many", _doc(), _ing_many, sync_tx=True), + Family("many_gen", _doc(), _ing_many_gen, sync_tx=True), + Family("many_par", _doc(4), _ing_many_par, setup=_executor()), + Family("cols", _doc(), _ing_cols, sync_tx=True), + Family("cols_par", _doc(4), _ing_cols_par, setup=_executor()), + Family("arrow", _doc(), _ing_arrow, sync_tx=True), + Family("arrow_par", _doc(4), _ing_arrow_par, setup=_executor()), + Family("arrow_vertex", _vertex, _ing_arrow), + Family("df", _doc(), _ing_df, sync_tx=True), + Family("df_par", _doc(4), _ing_df_par, setup=_executor()), + Family("async_create", _doc(), _ing_async_create, setup=_executor(1)), + Family("async_create_par", _doc(4), _ing_async_create, setup=_executor(4)), + Family("async_command", _doc(), _ing_async_command, setup=_executor(1)), + Family("vertices", _vertex, _ing_vertices), + Family( + "timeseries", + lambda db, name: make_ts_type(db, name), + _ing_timeseries, + setup=_executor(), + schema=TS_SCHEMA, + rows=make_ts_rows, + key="val", + min_n=1, + ), +] +BY_ID = {f.id: f for f in FAMILIES} + + +def run_cases(store, fam, sizes, mutant=None): + """Load one type per size through ``fam``, check each after the call, close and reopen, check each again. + + ``mutant`` names a fault: one of ``MUTANTS`` rewrites the rows the path is given (the check still compares with the rows that + were generated), ``NO_COMMIT`` loads inside a transaction nobody commits. Raises ``IntegrityError`` with every problem found. + """ + problems, cases = [], [] + db = store.db + ctx = fam.setup(db) + try: + for n in sizes: + if n < fam.min_n: + continue + name = f"{fam.id}_{n}" + rows = fam.rows(n) + sent = MUTANTS[mutant](rows) if mutant in MUTANTS else rows + fam.create(db, name) + if mutant == NO_COMMIT: + db.begin() + try: + got = fam.ingest(ctx, db, name, sent) + if got != len(sent): + problems.append( + f"{FOUND} n={n}: the call returned {got}, {len(sent)} rows were given" + ) + except ( + Exception + ) as exc: # noqa: BLE001 - reported with the size, the other sizes still run + problems.append(f"RAISED n={n}: {type(exc).__name__}: {exc}") + if db.is_transaction_active() and mutant != NO_COMMIT: + db.rollback() + if ctx is not None and ctx.errors: + problems.append( + f"{FOUND} n={n}: the executor's error handler was told {len(ctx.errors)} time(s): {ctx.errors[:2]}" + ) + ctx.errors.clear() + problems += problems_for_type( + db, name, rows, fam.schema, fam.key, f"n={n} after the call" + ) + cases.append((name, n, rows)) + finally: + _teardown(ctx) + store.reopen() + for name, n, rows in cases: + problems += problems_for_type( + store.db, name, rows, fam.schema, fam.key, f"n={n} after reopen" + ) + if problems: + raise IntegrityError(f"{fam.id}: " + "\n".join(problems)) + + +# -- sizes at every boundary -------------------------------------------------------------------------------------------------- + + +@pytest.mark.parametrize("fam", FAMILIES, ids=lambda f: f.id) +def test_every_size_at_every_boundary(store, small_chunks, fam): + run_cases(store, fam, SIZES) + + +def test_one_load_across_the_real_100000_chunk(store): + db = store.db + make_type(db, "Big", kind="VERTEX") + n = 100_001 + rows = make_rows(n) + with db.graph_batch() as batch: + rids = batch.create_vertices("Big", rows) + assert len(rids) == n and len(set(rids)) == n + problems = problems_for_type(db, "Big", rows, SCHEMA, "k", "after the call") + store.reopen() + problems += problems_for_type(store.db, "Big", rows, SCHEMA, "k", "after reopen") + assert not problems, "\n".join(problems) + + +# -- reading back at every batch size (the read path found the loss of #331) ---------------------------------------------- + + +@pytest.mark.parametrize("batch_size", [1, 2, 7, 255, 256, 257, 600, 25_000]) +def test_reading_back_every_row_at_every_batch_size(store, batch_size): + db = store.db + make_type(db, "Wide") + rows = make_rows(600) + assert db.insert_many("Wide", rows) == 600 + names = list(SCHEMA) + select = "SELECT FROM Wide" + cols = db.query("sql", select).to_columns(batch_size=batch_size, columns=names) + from_columns = [dict(zip(names, vals)) for vals in zip(*(cols[n] for n in names))] + table = db.query("sql", select).to_arrow(batch_size=batch_size, columns=names) + problems = verify( + f"batch_size={batch_size}", + rows, + SCHEMA, + "k", + {"to_columns rows": len(from_columns), "to_arrow rows": table.num_rows}, + {"to_columns()": from_columns, "to_arrow()": table.to_pylist()}, + ) + assert not problems, "\n".join(problems) + + +# -- edges: count and endpoint pairs ------------------------------------------------------------------------------------------ + +EDGE_SIZES = ladder((CHUNK, COMMIT)) +VERTICES = 11 + + +def _endpoints(m): + return [(i % VERTICES, (3 * i + 1) % VERTICES) for i in range(m)] + + +def _load_edges(db, rids, etype, pairs, weights): + batch = db.graph_batch(commit_every=CHUNK, batch_size=CHUNK) + try: + batch.new_edges( + [rids[a] for a, _ in pairs], + etype, + [rids[b] for _, b in pairs], + properties=None if weights is None else [{"w": w} for w in weights], + ) + finally: + batch.close() + + +def _vertex_rids(db): + make_type(db, "Node", kind="VERTEX") + with db.graph_batch(commit_every=CHUNK, batch_size=CHUNK) as batch: + return batch.create_vertices("Node", make_rows(VERTICES)) + + +@pytest.mark.parametrize("with_weights", [False, True], ids=["plain", "weighted"]) +def test_new_edges_store_every_edge_between_its_endpoints( + store, small_chunks, with_weights +): + db = store.db + rids = _vertex_rids(db) + problems, cases = [], [] + for m in EDGE_SIZES: + etype = f"Link{m}" + db.command("sql", f"CREATE EDGE TYPE {etype}") + pairs = _endpoints(m) + weights = [i + 0.25 for i in range(m)] if with_weights else None + _load_edges(db, rids, etype, pairs, weights) + want = [ + {"a": a, "b": b, "w": None if weights is None else weights[i]} + for i, (a, b) in enumerate(pairs) + ] + problems += problems_for_edges(db, etype, want, f"m={m} after close()") + cases.append((etype, m, want)) + store.reopen() + for etype, m, want in cases: + problems += problems_for_edges(store.db, etype, want, f"m={m} after reopen") + assert not problems, "\n".join(problems) + + +@pytest.mark.parametrize( + "fault", ["drop_the_last_edge", "swap_two_destinations", "store_one_edge_twice"] +) +def test_mutation_edges(store, small_chunks, fault): + db = store.db + rids = _vertex_rids(db) + db.command("sql", "CREATE EDGE TYPE Mut") + m = MUTATION_SIZE + pairs = _endpoints(m) + weights = [i + 0.25 for i in range(m)] + sent = list(pairs) + sent_weights = list(weights) + if fault == "drop_the_last_edge": + sent, sent_weights = sent[:-1], sent_weights[:-1] + elif fault == "store_one_edge_twice": + sent.append(sent[4]) + sent_weights.append(sent_weights[4]) + else: + (a1, b1), (a2, b2) = sent[1], sent[8] + sent[1], sent[8] = (a1, b2), (a2, b1) + _load_edges(db, rids, "Mut", sent, sent_weights) + want = [{"a": a, "b": b, "w": weights[i]} for i, (a, b) in enumerate(pairs)] + problems = problems_for_edges(db, "Mut", want, "after close()") + assert problems and all(FOUND in p for p in problems), problems + + +# -- the mutation check: every family, every fault --------------------------------------------------------------------------- + + +def _mutation_cases(): + for fam in FAMILIES: + for mutant in MUTANTS: + yield pytest.param(fam, mutant, id=f"{fam.id}-{mutant}") + if fam.sync_tx: + yield pytest.param(fam, NO_COMMIT, id=f"{fam.id}-{NO_COMMIT}") + + +@pytest.mark.parametrize("fam, mutant", list(_mutation_cases())) +def test_mutation_the_check_reports_the_fault(store, small_chunks, fam, mutant): + with pytest.raises(IntegrityError) as found: + run_cases(store, fam, [MUTATION_SIZE], mutant=mutant) + text = str(found.value) + assert FOUND in text, text + if mutant == NO_COMMIT: + # the rows are visible in this session, so only the reopen can tell + assert "after the call" not in text, text + assert "after reopen" in text, text + else: + assert "after the call" in text and "after reopen" in text, text + + +def _chunk_seam(): + """The function that stores one chunk of insert_many on this wheel: (class, name, arguments before the chunk).""" + if arcadedb.__backend__ == "native": + from arcadedb._native import database + + return database.NativeDatabase, "_insert_chunk" + from arcadedb import core + + return core.Database, "_insert_many_chunk" + + +@pytest.mark.parametrize( + "fault", + ["drop_the_last_row_of_a_chunk", "skip_a_chunk", "store_a_row_twice"], +) +def test_mutation_inside_insert_many(store, small_chunks, monkeypatch, fault): + owner, name = _chunk_seam() + real = getattr(owner, name) + seen = [] + + def faulty(self, type_name, *args): + *lead, chunk = args + seen.append(len(chunk)) + if len(seen) == 2: + if fault == "skip_a_chunk": + return None + if fault == "drop_the_last_row_of_a_chunk": + chunk = chunk[:-1] + else: + chunk = chunk + [chunk[-1]] + return real(self, type_name, *lead, chunk) + + monkeypatch.setattr(owner, name, faulty) + with pytest.raises(IntegrityError) as found: + run_cases(store, BY_ID["many"], [MUTATION_SIZE]) + assert FOUND in str(found.value), found.value + assert len(seen) >= 3, f"the fault was not in the path: {seen}" + + +def test_mutation_an_async_load_that_returns_before_the_wait(store): + """The executor is parked inside a callback, so the queued records cannot have run when the check looks.""" + db = store.db + make_type(db, "Early") + rows = make_rows(MUTATION_SIZE) + ex = db.async_executor().set_parallel_level(1).set_commit_every(COMMIT) + started, release = threading.Event(), threading.Event() + + def park(_row): + started.set() + release.wait(60) + + ex.query("sql", "SELECT 1 AS one", park) + assert started.wait(60), "the parking callback never ran" + try: + for row in rows: + ex.create_record(_doc_of(db, "Early", row)) + # no wait_completion(): this is the fault + problems = problems_for_type(db, "Early", rows, SCHEMA, "k", "before the wait") + finally: + release.set() + ex.wait_completion() + ex.close() + assert problems and all(FOUND in p for p in problems), problems + # and once the wait is done the same check passes, so the fault was the missing wait + assert not problems_for_type(db, "Early", rows, SCHEMA, "k", "after the wait") + + +# -- several writer threads --------------------------------------------------------------------------------------------------- + + +def _run_threads(jobs): + errors = [] + + def run(job): + try: + job() + except BaseException as exc: # noqa: BLE001 - handed to the test thread + errors.append(exc) + + threads = [threading.Thread(target=run, args=(job,)) for job in jobs] + for t in threads: + t.start() + for t in threads: + t.join(180) + assert not [t for t in threads if t.is_alive()], "a writer thread never finished" + return errors + + +class Reader: + """Counts the types in a loop while writers run; the counts of one type may only grow.""" + + def __init__(self, db, names): + self.db = db + self.series = {n: [] for n in names} + self.stop = threading.Event() + self.error = None + self.thread = threading.Thread(target=self.run) + + def count(self, name): + sql = "SELECT count(*) AS n FROM " + name # nosec B608 + return int(self.db.query("sql", sql).to_list()[0]["n"]) + + def run(self): + try: + while not self.stop.is_set(): + for name, values in self.series.items(): + values.append(self.count(name)) + time.sleep(0.002) + except BaseException as exc: # noqa: BLE001 - handed to the test thread + self.error = exc + + def __enter__(self): + self.thread.start() + return self + + def __exit__(self, *exc_info): + self.stop.set() + self.thread.join(60) + + def problems(self, totals): + found = [] + if self.error is not None: + found.append(f"RAISED the reader thread: {self.error!r}") + for name, values in self.series.items(): + if any(b < a for a, b in zip(values, values[1:])): + found.append(f"{FOUND} the count of {name} went down: {values}") + if values and values[-1] > totals[name]: + found.append( + f"{FOUND} the count of {name} reached {values[-1]}, above the {totals[name]} sent" + ) + if self.count(name) != totals[name]: + found.append( + f"{FOUND} the final count of {name} is {self.count(name)}, {totals[name]} were sent" + ) + return found + + +def test_writer_threads_on_one_type(store, small_chunks): + db = store.db + make_type(db, "OneType", buckets=4) + writers, per = 4, 60 + parts = [make_rows(per, start=w * per) for w in range(writers)] + + def job(rows): + return lambda: db.run_in_transaction( + lambda: db.insert_many("OneType", rows), retries=200, backoff_s=0.002 + ) + + with Reader(db, ["OneType"]) as reader: + errors = _run_threads([job(rows) for rows in parts]) + assert not errors, errors + everything = [r for rows in parts for r in rows] + problems = reader.problems({"OneType": len(everything)}) + problems += problems_for_type(db, "OneType", everything, SCHEMA, "k", "after join") + store.reopen() + problems += problems_for_type( + store.db, "OneType", everything, SCHEMA, "k", "after reopen" + ) + assert not problems, "\n".join(problems) + + +def test_writer_threads_on_different_types_and_paths(store, small_chunks): + db = store.db + paths = [BY_ID["many"], BY_ID["cols"], BY_ID["arrow"], BY_ID["df"]] + names = [f"Thread{i}" for i in range(len(paths))] + sent = {name: make_rows(90, start=1000 * i) for i, name in enumerate(names)} + for name in names: + make_type(db, name) + + def job(fam, name): + return lambda: fam.ingest(None, db, name, sent[name]) + + with Reader(db, names) as reader: + errors = _run_threads([job(f, n) for f, n in zip(paths, names)]) + assert not errors, errors + problems = reader.problems({n: len(sent[n]) for n in names}) + for name in names: + problems += problems_for_type(db, name, sent[name], SCHEMA, "k", "after join") + store.reopen() + for name in names: + problems += problems_for_type( + store.db, name, sent[name], SCHEMA, "k", "after reopen" + ) + assert not problems, "\n".join(problems) + + +# -- a bad row in the middle of a batch --------------------------------------------------------------------------------------- + +BAD_ROWS = { + "duplicate_key": lambda rows, p: {**rows[p], "k": rows[0]["k"]}, + "wrong_declared_type": lambda rows, p: {**rows[p], "val": "not a number"}, + "refused_value": lambda rows, p: {**rows[p], "extra": 2**70}, +} +POSITIONS = [1, 4, 5, 7, 13] # 14 rows, commit_every 5: batches [0-4] [5-9] [10-13] + +FAIL_CALLS = { + "insert_many": lambda db, name, sent: db.insert_many( + name, sent, commit_every=COMMIT + ), + "insert_columns": lambda db, name, sent: db.insert_columns( + name, _list_columns(sent), commit_every=COMMIT + ), +} + + +def _bad_cases(): + for path in FAIL_CALLS: + for bad in BAD_ROWS: + if bad == "refused_value" and path != "insert_many": + continue # the row's extra property has no column to go into + for p in POSITIONS: + yield pytest.param(path, bad, p, id=f"{path}-{bad}-row{p}") + + +@pytest.mark.parametrize("path, bad, p", list(_bad_cases())) +def test_a_bad_row_keeps_the_committed_batches_and_nothing_else( + store, small_chunks, path, bad, p +): + """Documented: rows already committed stay committed, the batch that holds the bad row is rolled back, the error is raised.""" + db = store.db + make_type(db, "Bad", unique=True) + rows = make_rows(14) + sent = list(rows) + sent[p] = BAD_ROWS[bad](rows, p) + with pytest.raises((ArcadeDBError, OverflowError)): + FAIL_CALLS[path](db, "Bad", sent) + assert not db.is_transaction_active() + kept = rows[: (p // COMMIT) * COMMIT] + problems = problems_for_type(db, "Bad", kept, SCHEMA, "k", "after the failed call") + store.reopen() + problems += problems_for_type(store.db, "Bad", kept, SCHEMA, "k", "after reopen") + assert not problems, "\n".join(problems) + + +ARROW_CALLS = { + "insert_arrow": lambda db, name, rows: db.insert_arrow( + name, _arrow(rows), batch_size=COMMIT + ), + "insert_dataframe": lambda db, name, rows: db.insert_dataframe( + name, _frame(rows), batch_size=COMMIT + ), +} + + +@pytest.mark.parametrize("path", list(ARROW_CALLS)) +@pytest.mark.parametrize("p", POSITIONS) +def test_arrow_and_dataframe_batches_before_a_duplicate_stay_committed(store, path, p): + db = store.db + make_type(db, "ArrowDup", unique=True) + rows = make_rows(14) + sent = list(rows) + sent[p] = BAD_ROWS["duplicate_key"](rows, p) + with pytest.raises(ArcadeDBError) as err: + ARROW_CALLS[path](db, "ArrowDup", sent) + committed = (p // COMMIT) * COMMIT + if committed: + assert ( + f"the {committed} row(s) of the batches before this one were committed" + in str(err.value) + ), err.value + else: + assert "batches before this one" not in str(err.value), err.value + kept = rows[:committed] + problems = problems_for_type(db, "ArrowDup", kept, SCHEMA, "k", "after the call") + store.reopen() + problems += problems_for_type( + store.db, "ArrowDup", kept, SCHEMA, "k", "after reopen" + ) + assert not problems, "\n".join(problems) + + +@pytest.mark.parametrize("path", list(ARROW_CALLS)) +def test_a_column_the_declared_type_refuses_stores_nothing(store, path): + """Documented: a TypeError before any row is written.""" + db = store.db + make_type(db, "Refused") + rows = make_rows(12) + text_rows = [{**r, "val": str(r["val"])} for r in rows] + schema = pa.schema( + [ + ("k", pa.int64()), + ("val", pa.string()), + ("flag", pa.bool_()), + ("txt", pa.string()), + ] + ) + if path == "insert_arrow": + with pytest.raises(TypeError): + db.insert_arrow( + "Refused", pa.Table.from_pylist(text_rows, schema=schema), batch_size=3 + ) + else: + frame = _frame(rows) + frame["val"] = pd.array([r["val"] for r in text_rows], dtype="string") + with pytest.raises(TypeError): + db.insert_dataframe("Refused", frame, batch_size=3) + problems = problems_for_type(db, "Refused", [], SCHEMA, "k", "after the call") + store.reopen() + problems += problems_for_type(store.db, "Refused", [], SCHEMA, "k", "after reopen") + assert not problems, "\n".join(problems) + + +@pytest.mark.parametrize("p", POSITIONS) +def test_create_vertices_chunks_before_a_duplicate_stay_stored(store, small_chunks, p): + db = store.db + make_type(db, "BadV", kind="VERTEX", unique=True) + rows = make_rows(14) + sent = [dict(r) for r in rows] + sent[p] = BAD_ROWS["duplicate_key"](rows, p) + batch = db.graph_batch(commit_every=CHUNK, batch_size=CHUNK) + try: + with pytest.raises(ArcadeDBError): + batch.create_vertices("BadV", sent) + finally: + batch.close() + assert not db.is_transaction_active() + kept = rows[: (p // CHUNK) * CHUNK] + problems = problems_for_type(db, "BadV", kept, SCHEMA, "k", "after the call") + store.reopen() + problems += problems_for_type(store.db, "BadV", kept, SCHEMA, "k", "after reopen") + assert not problems, "\n".join(problems) + + +@pytest.mark.parametrize("path", ["insert_many", "insert_columns"]) +def test_a_parallel_duplicate_is_reported_and_nothing_is_stored_twice( + store, small_chunks, path +): + """Documented for parallel=True: the failure is raised after the load, records other than the failed ones may be stored.""" + db = store.db + make_type(db, "ParDup", buckets=4, unique=True) + db.async_executor().set_commit_every(COMMIT) + rows = make_rows(40) + sent = list(rows) + sent[30] = BAD_ROWS["duplicate_key"](rows, 30) + with pytest.raises(ArcadeDBError, match="failed record"): + if path == "insert_many": + db.insert_many("ParDup", sent, parallel=True) + else: + db.insert_columns("ParDup", _numpy_columns(sent), parallel=True) + + def keys(database): + stored = database.query("sql", "SELECT k FROM ParDup").to_list() + return [int(r["k"]) for r in stored] + + before = keys(db) + assert len(before) == len(set(before)), "a key was stored twice" + assert set(before) <= {r["k"] for r in rows}, "a key was stored that was never sent" + assert db.count_type("ParDup") == len(before) + store.reopen() + after = keys(store.db) + assert sorted(after) == sorted(before), "the stored keys changed across a reopen" + + +# -- async: a rejected row and a failing callback must reach the caller ------------------------------------------------------ + + +def test_async_rejected_rows_reach_their_error_callbacks(store): + db = store.db + make_type(db, "AsyncDup", unique=True) + rows = make_rows(12) + sent = list(rows) + sent[5] = {**rows[5], "k": rows[0]["k"]} + sent[9] = {**rows[9], "k": rows[2]["k"]} + rejected, reported = [], [] + ex = db.async_executor().set_parallel_level(1).set_commit_every(1) + ex.on_error(reported.append) + for i, row in enumerate(sent): + ex.create_record( + _doc_of(db, "AsyncDup", row), + error_callback=lambda _e, i=i: rejected.append(i), + ) + ex.wait_completion() + ex.close() + assert sorted(rejected) == [5, 9] + assert len(reported) == 2, reported + kept = [r for i, r in enumerate(sent) if i not in (5, 9)] + problems = problems_for_type(db, "AsyncDup", kept, SCHEMA, "k", "after wait") + store.reopen() + problems += problems_for_type( + store.db, "AsyncDup", kept, SCHEMA, "k", "after reopen" + ) + assert not problems, "\n".join(problems) + + +@pytest.mark.filterwarnings("ignore::pytest.PytestUnraisableExceptionWarning") +def test_a_callback_that_raises_loses_no_record_without_a_report(store): + """A record is stored, or its own error callback was told: nothing is dropped silently when a per-record callback raises.""" + db = store.db + make_type(db, "CbRaise") + rows = make_rows(20) + raised, reported = [], [] + + def callback(i): + def run(_record): + if i % 4 == 1: + raised.append(i) + raise RuntimeError(f"callback {i}") + + return run + + ex = db.async_executor().set_parallel_level(1).set_commit_every(3) + for i, row in enumerate(rows): + ex.create_record( + _doc_of(db, "CbRaise", row), + callback=callback(i), + error_callback=lambda _e, i=i: reported.append(i), + ) + ex.wait_completion(60_000) + ex.close() + assert sorted(raised) == [1, 5, 9, 13, 17], "the callbacks did not all run" + kept = [r for i, r in enumerate(rows) if i not in set(reported)] + problems = problems_for_type( + db, + "CbRaise", + kept, + SCHEMA, + "k", + f"after wait, errors reported for {sorted(reported)}", + ) + store.reopen() + problems += problems_for_type( + store.db, "CbRaise", kept, SCHEMA, "k", "after reopen" + ) + assert not problems, "\n".join(problems) + + +# -- close() straight after the last call ------------------------------------------------------------------------------------- + + +@pytest.mark.parametrize("how", ["executor_close", "database_close"]) +def test_close_straight_after_async_submissions_stores_all_or_raises(store, how): + """Either every record is stored or an error is raised: never a silent partial.""" + db = store.db + make_type(db, "Straight") + n = 200 + rows = make_rows(n) + ex = db.async_executor().set_parallel_level(1).set_commit_every(10) + for row in rows: + ex.create_record(_doc_of(db, "Straight", row)) + try: + if how == "executor_close": + ex.close() + db.close() + except Exception: # noqa: BLE001 - an error was raised: not a silent partial + return + store.db = arcadedb.open_database(store.path) + problems = problems_for_type( + store.db, "Straight", rows, SCHEMA, "k", "after reopen" + ) + assert not problems, "\n".join(problems) + + +def test_close_straight_after_the_last_edges_stores_them_all(store, small_chunks): + db = store.db + rids = _vertex_rids(db) + db.command("sql", "CREATE EDGE TYPE Late") + pairs = _endpoints(40) + weights = [i + 0.25 for i in range(40)] + batch = db.graph_batch(commit_every=CHUNK, batch_size=CHUNK) + batch.new_edges( + [rids[a] for a, _ in pairs], + "Late", + [rids[b] for _, b in pairs], + properties=[{"w": w} for w in weights], + ) + batch.close() + store.reopen() + want = [{"a": a, "b": b, "w": weights[i]} for i, (a, b) in enumerate(pairs)] + problems = problems_for_edges(store.db, "Late", want, "after reopen") + assert not problems, "\n".join(problems) diff --git a/tests/test_jvm_ingest_integrity_faults.py b/tests/test_jvm_ingest_integrity_faults.py new file mode 100644 index 00000000..78586ab8 --- /dev/null +++ b/tests/test_jvm_ingest_integrity_faults.py @@ -0,0 +1,54 @@ +"""The JVM wheel's column conversion and bulk vertex path with a rows-dropped fault in them (ArcadeData/arcadedb-python#336). + +``arcadedb.core._column_to_java`` converts every column of ``insert_columns`` (and of ``insert_arrow`` and ``insert_dataframe``, +which call it), and ``GraphBatch._create_vertices_json_bulk`` carries the rows of ``create_vertices``. Each test puts a fault into +one of them and runs the same checks as ``test_ingest_integrity.py``: they must report it. A file about the JPype path, so +``NATIVE_NOT_APPLICABLE`` lists it. +""" + +import pytest + +import arcadedb +from arcadedb import core +from tests.test_ingest_integrity import ( + BY_ID, + FOUND, + MUTATION_SIZE, + IntegrityError, + run_cases, +) + + +@pytest.mark.parametrize("family", ["cols", "arrow", "df"]) +def test_a_column_converted_one_row_short_is_reported( + store, small_chunks, monkeypatch, family +): + real = core._column_to_java + seen = [] + + def faulty(name, values): + seen.append(name) + return real(name, values[:-1]) + + monkeypatch.setattr(core, "_column_to_java", faulty) + with pytest.raises(IntegrityError) as found: + run_cases(store, BY_ID[family], [MUTATION_SIZE]) + assert FOUND in str(found.value), found.value + assert seen, "the fault was not in the path" + + +def test_a_bulk_vertex_chunk_one_row_short_is_reported( + store, small_chunks, monkeypatch +): + real = arcadedb.GraphBatch._create_vertices_json_bulk + seen = [] + + def faulty(self, type_name, rows): + seen.append(len(rows)) + return real(self, type_name, rows[:-1]) + + monkeypatch.setattr(arcadedb.GraphBatch, "_create_vertices_json_bulk", faulty) + with pytest.raises(IntegrityError) as found: + run_cases(store, BY_ID["vertices"], [MUTATION_SIZE]) + assert FOUND in str(found.value), found.value + assert seen, "the fault was not in the path" diff --git a/tests/test_native_ingest_integrity_faults.py b/tests/test_native_ingest_integrity_faults.py new file mode 100644 index 00000000..088d4612 --- /dev/null +++ b/tests/test_native_ingest_integrity_faults.py @@ -0,0 +1,39 @@ +"""The native columnar fast path with a rows-dropped fault in it (ArcadeData/arcadedb-python#336). + +``arcadedb._native.bulk.insert_columns_fast`` is the one entry that ``insert_columns``, ``insert_arrow`` and ``insert_dataframe`` +share on the native build. Each test puts a fault into it and runs the same checks as ``test_ingest_integrity.py``: they must +report it. Needs the native library, so conftest.py collects this file on the native backend only. +""" + +import pytest + +from arcadedb._native import bulk +from tests.test_ingest_integrity import ( + BY_ID, + FOUND, + MUTATION_SIZE, + IntegrityError, + run_cases, +) + + +@pytest.mark.parametrize("fault", ["drop_the_last_row", "skip_the_call"]) +@pytest.mark.parametrize("family", ["cols", "arrow", "df"]) +def test_a_fault_in_the_columnar_fast_path_is_reported( + store, small_chunks, monkeypatch, family, fault +): + real = bulk.insert_columns_fast + seen = [] + + def faulty(db, type_name, columns, *args, **kwargs): + seen.append(type_name) + if fault == "skip_the_call": + return len(next(iter(columns.values()))) + short = {name: column[:-1] for name, column in columns.items()} + return real(db, type_name, short, *args, **kwargs) + + monkeypatch.setattr(bulk, "insert_columns_fast", faulty) + with pytest.raises(IntegrityError) as found: + run_cases(store, BY_ID[family], [MUTATION_SIZE]) + assert FOUND in str(found.value), found.value + assert seen, "the fault was not in the path" From fac44ffcb93a08790144b6c27e08ad4cffdc4ba1 Mon Sep 17 00:00:00 2001 From: Taewoon Kim Date: Mon, 9 Nov 2026 21:19:51 +0900 Subject: [PATCH 17/49] create_vertices: Java float[] and float32 arrays cross as one matrix per property on both wheels (refs #337) JVM: a Java float[] was no vector candidate, so classify_vertex_rows gave up on the first row and the whole load went through _to_java_property_matrix (convert_python_to_java per value, one Object[] per row, one Object[][] per call): 59 to 61 percent of the wall of the e2 and l3d loads in the untimed ingest profile of 26.11.1. A float32 numpy array on an undeclared property took the same way. take_vector_columns (type_conversion) now picks the columns: declared ARRAY_OF_FLOATS properties as before plus Java float[] and double[], and any other property whose value in every row is a float32 array or a Java float[]; the stored value of those is a float[] on both paths. The rest rows are rebuilt from columns by C iterators when the rows share their keys. Native: the float32 arrays of a row went through json.dumps with a Python hook (_gb_default, _default) per row, after a _prep walk of every value, one ["B", offset, count] tag per row parsed back in Java, and np.concatenate of one small array per row (29 to 31 percent prep in the same profile). encode_vertex_args writes the other properties with one json.dumps (no hook when they are plain scalars) and the matrices are the front of the side buffer, a ctypes view with no copy, described by one "vecs" entry that NativeExtras.withVectors turns into one float[] per row. The old per-row tag stays for edge properties and for a column that does not qualify. Tests: both wheels pin the stored bits and the split; the JVM wheel counts the JPype crossings and the property-matrix fallbacks; the native wheel counts the ctypes crossings and the codec's per-row hooks. The existing native test that expected a tag per vertex row now expects the matrix. The Java is not compiled locally: CI builds the native image. Co-Authored-By: Claude Sonnet 5.5 --- docs/modules/ROOT/pages/how-to/vectors.adoc | 4 +- .../org/humemai/nativelib/NativeExtras.java | 44 ++- src/arcadedb/_native/codec.py | 60 +++ src/arcadedb/_native/extras.py | 58 ++- src/arcadedb/graph_batch.py | 18 +- src/arcadedb/type_conversion.py | 171 +++++++-- tests/conftest.py | 3 + .../test_graph_batch_vector_crossings_jvm.py | 241 ++++++++++++ .../test_native_graph_batch_binary_vectors.py | 9 +- tests/test_native_vertex_vector_crossings.py | 227 ++++++++++++ tests/test_vertex_vector_loads.py | 344 ++++++++++++++++++ 11 files changed, 1115 insertions(+), 64 deletions(-) create mode 100644 tests/test_graph_batch_vector_crossings_jvm.py create mode 100644 tests/test_native_vertex_vector_crossings.py create mode 100644 tests/test_vertex_vector_loads.py diff --git a/docs/modules/ROOT/pages/how-to/vectors.adoc b/docs/modules/ROOT/pages/how-to/vectors.adoc index 84beb2db..4396b194 100644 --- a/docs/modules/ROOT/pages/how-to/vectors.adoc +++ b/docs/modules/ROOT/pages/how-to/vectors.adoc @@ -110,9 +110,9 @@ print(db.count_type("Chunk")) 200 ---- -`GraphBatch.create_vertices` rows and the properties of `new_edge` and `new_edges` take the same arrays. On the `arcadedb` (native) package, a non-empty one-dimensional `float32` array in a row or an edge property crosses in a binary side buffer, and the stored values are bit-identical to the ones the text path gave. A list or a `float64` array still crosses as number text, so make the array with `np.asarray(v, dtype=np.float32)` before the load. A laptop A/B on the native wheel (100,000 vertices for each load, `ab.py --noise`, ABBA order, 12 clean pairs for each figure, clean A/A control) gave this wall time of the new path against the old one (95 % interval of the median pair ratio): 0.517 [0.515, 0.520] at 128 dimensions, 0.218 [0.217, 0.219] at 96, and 0.271 [0.270, 0.272] at 64. A load without vectors was unchanged (1.009 [1.003, 1.012]). +`GraphBatch.create_vertices` rows and the properties of `new_edge` and `new_edges` take the same arrays. On the `arcadedb` (native) package, a non-empty one-dimensional `float32` array in a row or an edge property crosses in a binary side buffer, and the stored values are bit-identical to the ones the text path gave. A list or a `float64` array still crosses as number text, so make the array with `np.asarray(v, dtype=np.float32)` before the load. When a property holds a `float32` array of one length in every row of a `create_vertices` call, the whole column crosses as one float32 matrix in that buffer, with no tag or Python call for each row. A laptop A/B on the native wheel (100,000 vertices for each load, `ab.py --noise`, ABBA order, 12 clean pairs for each figure, clean A/A control) gave this wall time of the new path against the old one (95 % interval of the median pair ratio): 0.517 [0.515, 0.520] at 128 dimensions, 0.218 [0.217, 0.219] at 96, and 0.271 [0.270, 0.272] at 64. A load without vectors was unchanged (1.009 [1.003, 1.012]). -On the `arcadedb-jvm` package, `create_vertices` already sends the values of a property declared `ARRAY_OF_FLOATS` as one binary float32 matrix when every row holds a list or array of numbers of one length. So `float32` arrays in a property declared `ARRAY_OF_FLOATS` take the binary path on both packages. +On the `arcadedb-jvm` package, `create_vertices` sends the values of a property declared `ARRAY_OF_FLOATS` as one binary float32 matrix when every row holds a list, a numpy array or a Java `float[]` or `double[]` of numbers of one length. A property that is not declared takes the same matrix when every row holds a `float32` numpy array or a Java `float[]` (what `to_java_float_array` returns on this package) of one length, and the stored value is a `float[]`, as before. A list or a `float64` array on an undeclared property keeps its doubles and the older path. So `float32` arrays and `to_java_float_array` values take the binary path on both packages, declared or not. == Search diff --git a/native/src/main/java/org/humemai/nativelib/NativeExtras.java b/native/src/main/java/org/humemai/nativelib/NativeExtras.java index a62574ad..f359fe80 100644 --- a/native/src/main/java/org/humemai/nativelib/NativeExtras.java +++ b/native/src/main/java/org/humemai/nativelib/NativeExtras.java @@ -73,6 +73,7 @@ import org.graalvm.word.WordFactory; import java.util.ArrayList; +import java.util.Arrays; import java.util.HashMap; import java.util.HashSet; import java.util.List; @@ -250,6 +251,45 @@ static Object[] varargs(final Object props) { return out; } + /** + * The float32 vector columns of create_vertices rows: vecs is {"names": [..], "dims": [..], "offs": [..]} and the side buffer holds, for + * column c, one vector after the other (rows * dims[c] floats) from float offset offs[c]. Row i gets the property names[c] as a float[] + * of its slice, appended to its property array: the float[] that a {"\u0001":["B",offset,count]} tag per row gave, from one matrix and + * no tag per row. A null or empty vecs leaves the rows as they are. + */ + private static void withVectors(final List flats, final Object vecs, final long buf, final long bufFloats) { + if (!(vecs instanceof Map v)) + return; + final List names = (List) v.get("names"), dims = (List) v.get("dims"), offs = (List) v.get("offs"); + final int columns = names.size(), rows = flats.size(); + if (columns == 0) + return; + if (buf == 0 || dims.size() != columns || offs.size() != columns) + throw new IllegalArgumentException("vector columns without a matching side buffer"); + final int[] dim = new int[columns]; + final long[] off = new long[columns]; + for (int c = 0; c < columns; c++) { + dim[c] = ((Number) dims.get(c)).intValue(); + off[c] = ((Number) offs.get(c)).longValue(); + if (dim[c] < 1 || off[c] < 0 || off[c] + (long) rows * dim[c] > bufFloats) + throw new IllegalArgumentException("vector column outside the side buffer"); + } + final CFloatPointer fp = WordFactory.pointer(buf); + for (int i = 0; i < rows; i++) { + final Object[] row = flats.get(i); + final Object[] grown = Arrays.copyOf(row, row.length + 2 * columns); + for (int c = 0; c < columns; c++) { + final float[] f = new float[dim[c]]; + final int base = (int) (off[c] + (long) i * dim[c]); + for (int k = 0; k < dim[c]; k++) + f[k] = fp.read(base + k); + grown[row.length + 2 * c] = names.get(c); + grown[row.length + 2 * c + 1] = f; + } + flats.set(i, grown); + } + } + static String ridList(final RID[] rids) { final StringBuilder sb = new StringBuilder("["); for (int i = 0; i < rids.length; i++) { @@ -488,8 +528,10 @@ else if (opName.equals("create_vertices")) final RID[] rids; if (!(a.get("rows") instanceof Integer)) rids = batch.createVertices((String) a.get("type"), intOf(a.get("count"), 0)); - else + else { + withVectors(flats, a.get("vecs"), buf, bufFloats); rids = batch.createVertices((String) a.get("type"), flats.toArray(new Object[0][])); + } return ArcadeNative.out("{\"ok\":true,\"rids\":" + ridList(rids) + "}"); } case "new_edge" -> batch.newEdge(new RID((String) a.get("src")), (String) a.get("type"), new RID((String) a.get("dst")), diff --git a/src/arcadedb/_native/codec.py b/src/arcadedb/_native/codec.py index cf6d3f15..51d5d74e 100644 --- a/src/arcadedb/_native/codec.py +++ b/src/arcadedb/_native/codec.py @@ -1,5 +1,7 @@ """Python <-> native JSON: query parameters out, typed rows in.""" +import ctypes +import itertools import json import os from collections.abc import Mapping @@ -276,6 +278,64 @@ def encode_gb_args(args): return text, buf, state[1] +_STR_TYPE = frozenset((str,)) + + +def _plain_rows(rows): + """True when every row is a dict whose keys are str and whose values are plain scalars: :func:`_prep` and the default hook would + change nothing, so ``json.dumps`` writes the rows as they are. The census runs in C (a set of types per level), not in Python per row. + """ + if not set(map(type, set(itertools.chain.from_iterable(rows)))) <= _STR_TYPE: + return False + return ( + set(map(type, itertools.chain.from_iterable(map(dict.values, rows)))) <= _PLAIN + ) + + +def _side_buffer(flat): + """The float32 array `flat` as the side-buffer argument of ``arcadedb_n_gb_bin``: a ctypes view of its memory (no copy), or its bytes + when the array is read-only or not contiguous.""" + try: + return (ctypes.c_char * flat.nbytes).from_buffer(flat) + except (TypeError, ValueError): + return flat.tobytes() + + +def encode_vertex_args(type_name, rows, names, matrices): + """The ``args`` of ``arcadedb_n_gb_bin`` for ``create_vertices`` whose vectors are float32 matrices: ``(text, side buffer, float count)``. + + `rows` hold the properties other than `names`; matrices[c] is the ``(len(rows), dim)`` float32 array of the property names[c], one + row of it per vertex. The matrices are the front of the side buffer, back to back, and the text carries their names, dimensions and + float offsets (``"vecs"``): no tag and no Python call per row. A value of `rows` that needs the hook (a datetime, a float32 array that + did not qualify, a list of floats) is written as in :func:`encode_gb_args`, after the matrices in the buffer. + """ + offs, total = [], 0 + for matrix in matrices: + offs.append(total) + total += matrix.size + args = { + "type": type_name, + "rows": rows, + "vecs": { + "names": names, + "dims": [int(m.shape[1]) for m in matrices], + "offs": offs, + }, + } + if _plain_rows(rows): + text, arrays = _dumps(args), list(matrices) + else: + state = [list(matrices), total] + text = json.dumps(_prep(args), default=lambda v: _gb_default(v, state)) + arrays = state[0] + flat = ( + arrays[0].reshape(-1) + if len(arrays) == 1 + else _np.concatenate([a.reshape(-1) for a in arrays]) + ) + return text, _side_buffer(flat), flat.size + + _INT64_MAX = 2**63 - 1 _NS_DATETIME = None if _np is None else _np.dtype("datetime64[ns]") diff --git a/src/arcadedb/_native/extras.py b/src/arcadedb/_native/extras.py index 82440e9a..a5ff9eca 100644 --- a/src/arcadedb/_native/extras.py +++ b/src/arcadedb/_native/extras.py @@ -19,8 +19,18 @@ from ..callbacks import check_event, check_function from ..exceptions import ArcadeDBError from ..graph_batch import GraphBatch as _BaseGraphBatch +from ..type_conversion import take_vector_columns from . import interrupts, lib, not_yet -from .codec import _PLAIN, _default, _np, _prep, encode_gb_args, loads, sample_array +from .codec import ( + _PLAIN, + _default, + _np, + _prep, + encode_gb_args, + encode_vertex_args, + loads, + sample_array, +) # -------------------------------------------------------------------------------------------------------- upcall @@ -181,6 +191,30 @@ def _op(self, prefix: str, op: str, **args) -> dict: return _call(prefix, "arcadedb_n_gb", self._gb, lib.b(op), lib.b(text)) return _call(prefix, "arcadedb_n_gb", self._gb, lib.b(op), lib.b(_enc(args))) + def _create_chunk(self, prefix: str, type_name: str, rows: list) -> list: + """One crossing and one transaction for `rows` (dicts): the RIDs of the new vertices. + + A property whose value in every row is a float32 numpy array of one length crosses as one float32 matrix in the side buffer + (``take_vector_columns``, ``encode_vertex_args``), with no tag, hook call or Python code per row; every other value is + written as :func:`encode_gb_args` writes it. + """ + taken = take_vector_columns(rows, (), undeclared=True) if lib.GB_BIN else None + if taken is None: + return self._op(prefix, "create_vertices", type=type_name, rows=rows)[ + "rids" + ] + names, matrices, rest = taken + text, buf, count = encode_vertex_args(type_name, rest, names, matrices) + return _call( + prefix, + "arcadedb_n_gb_bin", + self._gb, + lib.b("create_vertices"), + lib.b(text), + buf, + count, + )["rids"] + def new_vertex(self, type_name: str): from .records import Vertex @@ -234,8 +268,10 @@ def create_vertices(self, type_name: str, count_or_properties) -> list: """The RIDs of the new vertices. Rows go in chunks of the batch's ``commit_every`` (1000 unless graph_batch() got another value; 0: 10,000), one crossing and one transaction per chunk: chunks that already went in stay stored if a later chunk fails. - A non-empty one-dimensional ``float32`` numpy array in a row crosses in a binary side buffer; a list or a ``float64`` array - crosses as number text (``new_edge`` and ``new_edges`` props follow the same rule). + A property whose value in every row is a non-empty one-dimensional ``float32`` numpy array of one length crosses as one + float32 matrix in a binary side buffer, one buffer per property and chunk; any other ``float32`` array of a row crosses in the side + buffer one array at a time, and a list or a ``float64`` array as number text (``new_edge`` and ``new_edges`` props follow the + one-array rule). """ self._check_not_closed() db = self._db @@ -260,8 +296,9 @@ def create_vertices(self, type_name: str, count_or_properties) -> list: ] return rids rows = list(count_or_properties) + kinds = set(map(type, rows)) if ( - not set(map(type, rows)) <= _DICT_OR_NONE + not kinds <= _DICT_OR_NONE ): # exact types in C; only an odd row pays the isinstance loop for row in rows: if row is not None and not isinstance(row, dict): @@ -271,15 +308,12 @@ def create_vertices(self, type_name: str, count_or_properties) -> list: # chunked, as the JPype wrapper does: one crossing, one JSON text and one reply per chunk, so a 1M-vertex load # never holds the text, the parsed rows and the reply of all of it at once (the Serial GC copies what lives) rids = [] + has_none = type(None) in kinds for start in range(0, len(rows), chunk): - rids.extend( - self._op( - prefix, - "create_vertices", - type=type_name, - rows=[r or {} for r in rows[start : start + chunk]], - )["rids"] - ) + part = rows[start : start + chunk] + if has_none: + part = [r or {} for r in part] + rids.extend(self._create_chunk(prefix, type_name, part)) if not rows: rids = self._op(prefix, "create_vertices", type=type_name, rows=[])[ "rids" diff --git a/src/arcadedb/graph_batch.py b/src/arcadedb/graph_batch.py index efc1aff2..094e1d23 100644 --- a/src/arcadedb/graph_batch.py +++ b/src/arcadedb/graph_batch.py @@ -192,9 +192,12 @@ def create_vertices( count_or_properties: Either an integer vertex count or an iterable of per-vertex property dictionaries. Use `None` or `{}` for vertices without properties when passing an iterable. A property declared - ``ARRAY_OF_FLOATS`` whose value in every row is a list or a numpy - array of numbers of one length crosses as one binary float32 - matrix instead of JSON text. + ``ARRAY_OF_FLOATS`` whose value in every row is a list, a numpy + array or a Java ``float[]`` or ``double[]`` of numbers of one length + crosses as one binary float32 matrix instead of JSON text, and so + does any other property whose value in every row is a float32 + numpy array or a Java ``float[]`` (``to_java_float_array``) of one + length: one buffer per property and chunk, stored as a ``float[]``. A list (not one of only numbers), tuple, dict, date-time, date, Decimal or bytes value also stays on the bulk path: it crosses inside the JSON as a tagged value and is stored as the @@ -252,8 +255,9 @@ def create_vertices( def _create_vertices_json_bulk(self, type_name, rows): """Bulk path: rows cross as one JSON string, RIDs return as one joined string (two bulk copies instead of per-value marshaling, measured - ~2.4x). The float vectors of ARRAY_OF_FLOATS properties cross as one - binary matrix beside the JSON instead of as text (split_float_vectors). + ~2.4x). The float vectors of ARRAY_OF_FLOATS properties, and the float32 + arrays and Java float[] of any property, cross as one binary matrix + beside the JSON instead of as text (split_float_vectors). A list (not one of only numbers), tuple, dict, date-time, date, Decimal or bytes value crosses inside the JSON as the tagged value json_bulk_dumps_tagged writes and VertexBatcher decodes (the stored @@ -284,7 +288,9 @@ def _create_vertices_json_bulk(self, type_name, rows): rids = [] for start in range(0, len(rows), self._BULK_CHUNK): chunk = rows[start : start + self._BULK_CHUNK] - split = split_float_vectors(self._java_db, type_name, chunk, declared) + split = split_float_vectors( + self._java_db, type_name, chunk, declared, undeclared=True + ) rest = split[0] if split else chunk text = None if declared_scalars_ok(rest, declared): diff --git a/src/arcadedb/type_conversion.py b/src/arcadedb/type_conversion.py index 8f90133c..cc040da8 100644 --- a/src/arcadedb/type_conversion.py +++ b/src/arcadedb/type_conversion.py @@ -8,6 +8,7 @@ import itertools import json import math +import operator import re import sys from datetime import date, datetime, time, timedelta, timezone @@ -1077,11 +1078,10 @@ def classify_vertex_rows(rows: Any, declared: Any): odd = set(map(type, _column(rows, key))) - _BULK_SCALAR_TYPES if not odd: continue - if key in declared and all( - kind is list or (_np is not None and issubclass(kind, _np.ndarray)) - for kind in odd - ): + if key in declared and all(_is_vector_kind(kind) for kind in odd): continue # the vectors of a declared column + if all(_is_float_array_kind(kind) for kind in odd): + continue # a column of arrays: take_vector_columns takes the float32 ones; a chunk it leaves goes the matrix way for value in _column(rows, key): if type(value) in _BULK_SCALAR_TYPES or ( key in declared and is_vector_candidate(value) @@ -1142,41 +1142,103 @@ def declared_float_vectors(java_db, type_name): return [str(name) for name in vectors.declared(java_db, type_name)] +_JAVA_VECTOR_ARRAYS = None # (float[] class, double[] class) of JPype once the JVM runs; () where there is no JVM (the native wheel) + + +def _java_vector_arrays(): + """The JPype classes of a Java ``float[]`` and a ``double[]`` (what ``to_java_float_array`` returns on the JVM wheel), () without a JVM. + + Resolved once the JVM has started: JPype builds an array class from a Java class, so it cannot be done at import. + """ + global _JAVA_VECTOR_ARRAYS + if _JAVA_VECTOR_ARRAYS is None: + try: + if not jpype.isJVMStarted(): + return () + _JAVA_VECTOR_ARRAYS = ( + jpype.JArray(jpype.JFloat), + jpype.JArray(jpype.JDouble), + ) + except Exception: # the native wheel's stand-in for jpype has no such classes + _JAVA_VECTOR_ARRAYS = () + return _JAVA_VECTOR_ARRAYS + + +def _is_vector_kind(kind): + """The type of a value that a declared ARRAY_OF_FLOATS column may hold as a vector: a list, a numpy array, a Java float[] or double[].""" + return ( + kind is list + or (_np is not None and issubclass(kind, _np.ndarray)) + or issubclass(kind, _java_vector_arrays()) + ) + + +def _is_float_array_kind(kind): + """The type of a value that may be a float32 vector on any property: a numpy array or a Java float[].""" + arrays = _java_vector_arrays() + return (_np is not None and issubclass(kind, _np.ndarray)) or ( + bool(arrays) and issubclass(kind, arrays[0]) + ) + + def is_vector_candidate(value): - """A value split_float_vectors may take out of the JSON: a list or a numpy array.""" - return type(value) is list or (_np is not None and isinstance(value, _np.ndarray)) + """A value split_float_vectors may take out of the JSON: a list, a numpy array, or a Java float[] or double[].""" + return _is_vector_kind(type(value)) -def split_float_vectors(java_db, type_name, rows, declared=None): - """Take the float vectors of `rows` out of the JSON text of a bulk load. +def _is_float32_vector(value): + """A numpy float32 array or a Java float[]: the value the property-matrix path stores as a float[] on any property.""" + if _np is not None and isinstance(value, _np.ndarray): + return value.dtype == _np.float32 + arrays = _java_vector_arrays() + return bool(arrays) and isinstance(value, arrays[0]) - A list of floats costs Python about 450 ns a number to write as text and the - engine about as much again to parse it, so a vector load spent most of its - time on the text. For each property of `type_name` that is declared - ARRAY_OF_FLOATS, whose value in EVERY row is a list or a numpy array of - numbers of one length, the vectors cross as one float32 matrix instead - (``FloatVectors.Matrix``, one buffer copy). The engine converts the numbers of a list to float for - such a property, so the float32 matrix stores the same values. Every other - property, and a column that is missing in a row, ragged, not numeric, or - null in a row, stays in the JSON. - Returns ``(rest_rows, names, matrices)``: `rest_rows` are the rows without - those properties, `names` a ``String[]`` and `matrices` an ``Object[]`` of - ``FloatVectors.Matrix`` for the bridge; or None when no property qualifies - (the caller sends `rows` as they are). +def _without_keys(rows, gone): + """`rows` (each one holds every name of `gone`) as new dicts without those keys; the values are the same objects. + + Rows that all have the same keys in the same order are rebuilt from columns by C iterators (itemgetter, zip, dict), with no Python + code per row; any other rows take the dict comprehension. + """ + if len(set(map(tuple, rows))) == 1: + keep = [key for key in rows[0] if key not in gone] + if not keep: + return list(map(dict, itertools.repeat((), len(rows)))) + columns = [list(map(operator.itemgetter(key), rows)) for key in keep] + return list(map(dict, map(zip, itertools.repeat(keep), zip(*columns)))) + gone = set(gone) + return [{k: v for k, v in row.items() if k not in gone} for row in rows] + + +def take_vector_columns(rows, declared, undeclared=False): + """The float vector columns of `rows` as float32 matrices, and the rows without them (the one split of both wheels). + + A declared (ARRAY_OF_FLOATS) property qualifies when its value in EVERY row is a list, a numpy array or a Java float[] or double[] of + numbers of one length: the engine converts the numbers to float for such a property, so a float32 matrix stores the same values. + With `undeclared`, any other property qualifies when its value in every row is a float32 numpy array or a Java float[] of one length + (the first row decides which names are tried, the stacked column must come out as float32): the property-matrix path stores such a + value as a float[], and so does the matrix. A float64 array, a list, a Java double[] on an undeclared property, a missing value, a + null, a ragged or non-numeric column does not qualify and stays in the rows. + + Returns None when no column qualifies, else ``(names, matrices, rest)``: the matrices are C-contiguous float32 arrays of shape + ``(len(rows), dim)``, `rest` the rows without those properties. """ if _np is None or not rows: return None - vectors = _float_vectors_class() - if vectors is None: - return None - if declared is None: - declared = declared_float_vectors(java_db, type_name) + declared = set(declared) + names = list(declared) + first = rows[0] + if undeclared and isinstance(first, dict): + names.extend( + key + for key, value in first.items() + if key not in declared and _is_float32_vector(value) + ) taken = [] matrices = [] - for name in declared: + for name in names: try: - column = [row[name] for row in rows] + column = list(map(operator.itemgetter(name), rows)) except (KeyError, TypeError): continue if not is_vector_candidate(column[0]): @@ -1187,24 +1249,53 @@ def split_float_vectors(java_db, type_name, rows, declared=None): continue if array.ndim != 2 or array.shape[1] == 0 or array.dtype.kind not in "fiu": continue + if name not in declared and array.dtype != _np.float32: + continue with _np.errstate( all="ignore" ): # a double beyond float32 becomes Infinity, as in the engine - flat = _np.ascontiguousarray(array, dtype=_np.float32).reshape(-1) + matrix = _np.ascontiguousarray(array, dtype=_np.float32) taken.append(name) - matrices.append( - vectors.Matrix(jpype.JArray(jpype.JFloat)(flat), int(array.shape[1])) - ) + matrices.append(matrix) if not taken: return None - if len(taken) == 1: - only = taken[0] - rest = [{k: v for k, v in row.items() if k != only} for row in rows] - else: - gone = set(taken) - rest = [{k: v for k, v in row.items() if k not in gone} for row in rows] + return taken, matrices, _without_keys(rows, taken) + + +def split_float_vectors(java_db, type_name, rows, declared=None, undeclared=False): + """Take the float vectors of `rows` out of the JSON text of a bulk load (the JVM wheel). + + A list of floats costs Python about 450 ns a number to write as text and the + engine about as much again to parse it, so a vector load spent most of its + time on the text. The vectors of the columns that take_vector_columns picks + cross as one float32 matrix per property instead (``FloatVectors.Matrix``, + one buffer copy). Every other property, and a column that is missing in a + row, ragged, not numeric, or null in a row, stays in the JSON. + + Returns ``(rest_rows, names, matrices)``: `rest_rows` are the rows without + those properties, `names` a ``String[]`` and `matrices` an ``Object[]`` of + ``FloatVectors.Matrix`` for the bridge; or None when no property qualifies + (the caller sends `rows` as they are). + """ + if _np is None or not rows: + return None + vectors = _float_vectors_class() + if vectors is None: + return None + if declared is None: + declared = declared_float_vectors(java_db, type_name) + taken = take_vector_columns(rows, declared, undeclared) + if taken is None: + return None + names, matrices, rest = taken + float_array = jpype.JArray(jpype.JFloat) return ( rest, - jpype.JArray(jpype.JString)(taken), - jpype.JArray(jpype.JObject)(matrices), + jpype.JArray(jpype.JString)(names), + jpype.JArray(jpype.JObject)( + [ + vectors.Matrix(float_array(matrix.reshape(-1)), int(matrix.shape[1])) + for matrix in matrices + ] + ), ) diff --git a/tests/conftest.py b/tests/conftest.py index 3ed7133f..27d52e9e 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -83,6 +83,8 @@ collect_ignore.append("test_native_point_op_glue.py") # the address table of append_samples (arcadedb_n_async_bin), observed on the native library collect_ignore.append("test_native_timeseries_binary.py") + # the crossings of GraphBatch.create_vertices with float32 columns (lib.call, the codec's per-row hooks), observed on the native library + collect_ignore.append("test_native_vertex_vector_crossings.py") if not os.path.isfile( os.path.join(os.path.dirname(os.path.dirname(__file__)), "docs", "index.md") ): @@ -120,6 +122,7 @@ "test_insert_many_gc_pause.py": "the cyclic collector pause is the JVM build's (the native insert_many allocates nothing the collector tracks)", "test_graph_batch_tagged_shapes.py": "the JVM wheel's graph batch bulk path (bridge VertexBatcher and EdgeBatcher, the property-matrix fallback)", "test_graph_batch_single_pass.py": "the JVM wheel's graph batch bulk path (GraphBatch._create_vertices_json_bulk, the property-matrix fallback)", + "test_graph_batch_vector_crossings_jvm.py": "the JPype crossings of the JVM wheel's create_vertices vector path (the bridge classes, the property-matrix fallback)", "test_async_proxy_cache.py": "the JPype proxy classes of the JVM build's async executor", "test_read_crossings.py": "crossing counts of the JVM build's read path (JPype proxies of the Java objects and the bridge classes)", "test_reader_crossings.py": "crossing counts of the JVM build's record and row readers (JPype proxies of the Java objects and the bridge classes)", diff --git a/tests/test_graph_batch_vector_crossings_jvm.py b/tests/test_graph_batch_vector_crossings_jvm.py new file mode 100644 index 00000000..a16d36c3 --- /dev/null +++ b/tests/test_graph_batch_vector_crossings_jvm.py @@ -0,0 +1,241 @@ +"""GraphBatch.create_vertices (the JVM wheel): a Java float[] or float32 array crosses JPype as one matrix per property and chunk (#337). + +Before, a Java ``float[]`` (what ``to_java_float_array`` returns) was no vector candidate: ``classify_vertex_rows`` gave up on the first +row, and the whole load went through ``_to_java_property_matrix``, which converts every value of every row with ``convert_python_to_java``, +builds one ``Object[]`` per row and one ``Object[][]`` for the call (an unwrapped Python loop that was 59 to 61 percent of the wall of the +e2 and l3d loads in the untimed ingest profile of 26.11.1). A float32 numpy array on a property that is not declared ``ARRAY_OF_FLOATS`` +took the same way. Now ``take_vector_columns`` stacks the column once (numpy reads a Java array through the buffer protocol), one +``float[]`` copy and one ``FloatVectors.Matrix`` cross per property, and one ``VertexBatcher.createVerticesJson`` call per chunk. + +The JPype crossings are counted by wrapping the bridge classes and the Python-side conversion; the stored values are in +test_vertex_vector_loads.py. The JPype bridge does not exist on the native wheel (its counts are in test_native_vertex_vector_crossings.py), +so conftest.py does not collect this file there. +""" + +import jpype +import numpy as np +import pytest + +import arcadedb +from arcadedb import graph_batch as gbm +from arcadedb import type_conversion as tc +from arcadedb.graph_batch import GraphBatch + +ROWS, CHUNK, DIM = 250, 100, 16 # three chunks: 100, 100, 50 + + +class _Counting: + """Forwards every call of a Java class (or of the bridge's FloatVectors) and records its name.""" + + def __init__(self, target, log): + self._target, self._log = target, log + + def __getattr__(self, name): + attribute = getattr(self._target, name) + if not callable(attribute): + return attribute + + def call(*args): + self._log.append(name) + return attribute(*args) + + return call + + +@pytest.fixture +def crossings(monkeypatch): + """What the load does: property-matrix fallbacks, per-value conversions, bridge calls, Matrix constructions, per-row inserts.""" + log = {"fallback": [], "convert": [], "bridge": [], "matrix": [], "per_row": []} + + real_matrix_path = GraphBatch._to_java_property_matrix + + def fallback(self, rows): + log["fallback"].append(len(rows)) + return real_matrix_path(self, rows) + + monkeypatch.setattr(GraphBatch, "_to_java_property_matrix", fallback) + + real_convert = tc.convert_python_to_java + + def convert(value): + log["convert"].append(type(value).__name__) + return real_convert(value) + + monkeypatch.setattr(tc, "convert_python_to_java", convert) + monkeypatch.setattr(gbm, "convert_python_to_java", convert) + + real_jclass = jpype.JClass + + def jclass(name, *args, **kwargs): + cls = real_jclass(name, *args, **kwargs) + if name == "com.arcadedb.python.VertexBatcher": + return _Counting(cls, log["bridge"]) + return cls + + monkeypatch.setattr(jpype, "JClass", jclass) + + real_vectors = tc._float_vectors_class + + def vectors_class(): + vectors = real_vectors() + return None if vectors is None else _Counting(vectors, log["matrix"]) + + monkeypatch.setattr(tc, "_float_vectors_class", vectors_class) + + from arcadedb import core + + real_per_row = core.Database._insert_many_per_row + + def per_row(self, type_name, rows): + log["per_row"].append(len(rows)) + return real_per_row(self, type_name, rows) + + monkeypatch.setattr(core.Database, "_insert_many_per_row", per_row) + monkeypatch.setattr(GraphBatch, "_BULK_CHUNK", CHUNK) + return log + + +@pytest.fixture +def db(temp_db): + for name in ("Declared", "Free", "Doc"): + kind = "DOCUMENT" if name == "Doc" else "VERTEX" + temp_db.command("sql", f"CREATE {kind} TYPE {name}") + temp_db.command("sql", f"CREATE PROPERTY {name}.id LONG") + temp_db.command("sql", "CREATE PROPERTY Declared.embedding ARRAY_OF_FLOATS") + temp_db.command("sql", "CREATE PROPERTY Doc.embedding ARRAY_OF_FLOATS") + return temp_db + + +def _x(n=ROWS, dim=DIM): + return np.random.default_rng(7).standard_normal((n, dim)).astype(np.float32) + + +def _floats(x): + return [jpype.JArray(jpype.JFloat)(row) for row in x] + + +@pytest.mark.parametrize("type_name", ["Declared", "Free"]) +@pytest.mark.parametrize("shape", ["java_float_array", "float32_ndarray"]) +def test_a_float32_vector_crosses_as_one_matrix_per_chunk( + db, crossings, type_name, shape +): + x = _x() + vectors = _floats(x) if shape == "java_float_array" else list(x) + rows = [{"id": i, "embedding": vectors[i]} for i in range(ROWS)] + with db.graph_batch(parallel_flush=False) as gb: + assert len(gb.create_vertices(type_name, rows)) == ROWS + assert ( + crossings["fallback"] == [] + ), "the property-matrix path ran (a JPype conversion per value and per row)" + assert crossings["convert"] == [], "a value was converted one at a time" + assert ( + crossings["bridge"] == ["createVerticesJson"] * 3 + ) # one crossing per chunk of 100 + # one float32 matrix per chunk: `declared` is asked once per call, `Matrix` once per chunk + assert crossings["matrix"].count("Matrix") == 3 + sql = f"SELECT id, embedding FROM {type_name}" # nosec B608 - test-owned name + stored = {r["id"]: r["embedding"] for r in db.query("sql", sql).to_list()} + for i in range(ROWS): + assert np.array(stored[i], dtype=np.float32).tobytes() == x[i].tobytes() + + +def test_two_vector_properties_cross_as_two_matrices_per_chunk(db, crossings): + x, y = _x(), _x(dim=8) + rows = [ + {"id": i, "embedding": jpype.JArray(jpype.JFloat)(x[i]), "second": y[i]} + for i in range(ROWS) + ] + with db.graph_batch(parallel_flush=False) as gb: + gb.create_vertices("Declared", rows) + assert crossings["fallback"] == [] and crossings["convert"] == [] + assert crossings["bridge"] == ["createVerticesJson"] * 3 + assert crossings["matrix"].count("Matrix") == 6 + + +def test_a_ragged_column_still_takes_the_property_matrix_path_for_its_chunk( + db, crossings +): + x = _x() + vectors = _floats(x) + rows = [{"id": i, "embedding": vectors[i]} for i in range(ROWS)] + rows[120] = {"id": 120, "embedding": jpype.JArray(jpype.JFloat)(x[120][:5])} + with db.graph_batch(parallel_flush=False) as gb: + assert len(gb.create_vertices("Declared", rows)) == ROWS + assert crossings["fallback"] == [100], "only the chunk of row 120" + assert crossings["bridge"] == ["createVerticesJson"] * 2 + stored = { + r["id"]: r["embedding"] + for r in db.query("sql", "SELECT id, embedding FROM Declared").to_list() + } + assert np.array(stored[120], dtype=np.float32).tobytes() == x[120][:5].tobytes() + assert np.array(stored[0], dtype=np.float32).tobytes() == x[0].tobytes() + + +def test_a_java_double_array_is_a_vector_only_on_a_declared_property(db, crossings): + x64 = np.random.default_rng(8).random((ROWS, DIM)) * 1000.0 + 1e-3 + rows = [ + {"id": i, "embedding": jpype.JArray(jpype.JDouble)(x64[i])} for i in range(ROWS) + ] + with db.graph_batch(parallel_flush=False) as gb: + gb.create_vertices("Declared", rows) + assert ( + crossings["fallback"] == [] + and crossings["bridge"] == ["createVerticesJson"] * 3 + ) + declared = { + r["id"]: r["embedding"] + for r in db.query("sql", "SELECT id, embedding FROM Declared").to_list() + } + for i in ( + 0, + 99, + 100, + ROWS - 1, + ): # the engine stores floats for the declared property + want = x64[i].astype(np.float32).tobytes() + assert np.array(declared[i], dtype=np.float32).tobytes() == want + del crossings["fallback"][:], crossings["bridge"][:] + with db.graph_batch(parallel_flush=False) as gb: + gb.create_vertices("Free", rows) + assert crossings["fallback"] == [ + ROWS + ], "undeclared: the double[] keeps the matrix path" + free = { + r["id"]: r["embedding"] + for r in db.query("sql", "SELECT id, embedding FROM Free").to_list() + } + assert np.array(free[5], dtype=np.float64).tolist() == x64[5].tolist() + + +def test_insert_many_sends_a_java_float_array_column_through_the_matrix(db, crossings): + x = _x(50) + arrays = _floats(x) + rows = [{"id": i, "embedding": arrays[i]} for i in range(50)] + assert db.insert_many("Doc", rows) == 50 + assert crossings["per_row"] == [], "the column went row by row" + assert crossings["matrix"].count("Matrix") == 1 + stored = { + r["id"]: r["embedding"] + for r in db.query("sql", "SELECT id, embedding FROM Doc").to_list() + } + for i in range(50): + assert np.array(stored[i], dtype=np.float32).tobytes() == x[i].tobytes() + + +def test_take_vector_columns_reads_java_arrays_through_one_stack(db): + float_array, double_array = jpype.JArray(jpype.JFloat), jpype.JArray(jpype.JDouble) + x = _x(6, 5) + floats = [{"v": float_array(row), "k": i} for i, row in enumerate(x)] + names, matrices, rest = tc.take_vector_columns(floats, (), undeclared=True) + assert names == ["v"] and matrices[0].dtype == np.float32 + assert matrices[0].tobytes() == x.tobytes() and rest == [{"k": i} for i in range(6)] + doubles = [{"v": double_array(row.astype(np.float64))} for row in x] + assert tc.take_vector_columns(doubles, (), undeclared=True) is None + assert tc.take_vector_columns(doubles, ["v"])[1][0].tobytes() == x.tobytes() + ints = [{"v": jpype.JArray(jpype.JInt)([1, 2, 3])} for _ in range(3)] + assert tc.take_vector_columns(ints, ["v"], undeclared=True) is None + assert not tc.is_vector_candidate(ints[0]["v"]) + assert tc.is_vector_candidate(floats[0]["v"]) and tc.is_vector_candidate( + doubles[0]["v"] + ) + assert isinstance(arcadedb.to_java_float_array(x[0]), float_array) diff --git a/tests/test_native_graph_batch_binary_vectors.py b/tests/test_native_graph_batch_binary_vectors.py index 429adf49..c5543109 100644 --- a/tests/test_native_graph_batch_binary_vectors.py +++ b/tests/test_native_graph_batch_binary_vectors.py @@ -1,7 +1,8 @@ """GraphBatch float32 vectors cross the native library in a binary side buffer, not as number text. ``create_vertices`` rows and ``new_edges`` props that hold a float32 ``ndarray`` go through ``arcadedb_n_gb_bin`` (the float32 buffer is -read in Java through the ``["B", offset, count]`` tag, as ``arcadedb_n_exec_bin`` reads query parameters); everything else keeps the text +read in Java through the ``["B", offset, count]`` tag, as ``arcadedb_n_exec_bin`` reads query parameters; a ``create_vertices`` column of +float32 arrays of one length crosses as one matrix described by ``"vecs"`` instead, see test_native_vertex_vector_crossings.py); everything else keeps the text of ``arcadedb_n_gb``: a vector given as a list, a float64 array, a row with no vector. The stored values do not change: the same float32 as the ``"F"``-tagged text gave, bit for bit. A library without the entry (``lib.GB_BIN`` false) keeps the text path. @@ -95,9 +96,11 @@ def test_float32_vectors_take_the_binary_entry_and_are_stored_bit_equal(db, call assert calls.entries("create_vertices") == ["arcadedb_n_gb_bin"] _, _, text, count = next(c for c in calls.calls if c[1] == "create_vertices") assert count == N * dim - assert '"B"' in text and '"F"' not in text + assert ( + '"vecs"' in text and '"B"' not in text and '"F"' not in text + ) # one matrix, no tag per row (#337) assert str(float(x[2, 0])) not in text # the numbers are not in the text - assert len(text) < N * 80 # a vector is a tag, not dim numbers + assert len(text) < N * 80 # a row is its other properties, not dim numbers _assert_bit_equal(_stored(db, "Vx"), x) diff --git a/tests/test_native_vertex_vector_crossings.py b/tests/test_native_vertex_vector_crossings.py new file mode 100644 index 00000000..8523df95 --- /dev/null +++ b/tests/test_native_vertex_vector_crossings.py @@ -0,0 +1,227 @@ +"""Native GraphBatch.create_vertices: float32 vectors are split and encoded as columns, not row by row (#337). + +Before, every float32 array of a row was found by ``json.dumps`` through a Python hook (``codec._gb_default``, then ``_default``: about a +dozen ``isinstance`` tests and one array copy per row), the rows were walked by ``codec._prep`` first (one Python call per value), a tag +``{"\\u0001":["B",offset,count]}`` was written and parsed back in Java per row, and ``np.concatenate`` joined one small array per row +(the 29 to 31 percent prep share of the e2 and l3d loads in the untimed ingest profile of 26.11.1). Now ``take_vector_columns`` stacks each +float32 column once, ``codec.encode_vertex_args`` writes the other properties with one ``json.dumps`` (no hook when they are plain scalars) +and the matrices cross as the front of the side buffer, described by one ``"vecs"`` entry, which the library +(``NativeExtras.withVectors``) turns into one ``float[]`` per row. One ``arcadedb_n_gb_bin`` crossing per chunk, as before. + +Needs the native library, so conftest.py collects this file on the native backend only. The stored values are in test_vertex_vector_loads.py +(both wheels) and test_native_graph_batch_binary_vectors.py. +""" + +import ctypes +import json +from datetime import datetime + +import numpy as np +import pytest + +from arcadedb._native import codec, lib + +CHUNK, ROWS, DIM = 100, 250, 16 # three chunks: 100, 100, 50 + + +class _Spy: + """The create_vertices crossings (entry, JSON text, side buffer, float count) and the per-row Python calls of the codec.""" + + def __init__(self, monkeypatch): + self.crossings, self.hooks = [], {"gb_default": 0, "default": 0, "prep": 0} + real_call, real_gb_default = lib.call, codec._gb_default + real_default, real_prep = codec._default, codec._prep + + def call(name, *args): + if ( + name in ("arcadedb_n_gb", "arcadedb_n_gb_bin") + and args[1] == b"create_vertices" + ): + buf = args[3] if name == "arcadedb_n_gb_bin" else None + count = args[4] if name == "arcadedb_n_gb_bin" else 0 + self.crossings.append((name, args[2].decode(), buf, count)) + return real_call(name, *args) + + def counting(key, real): + def wrapper(*args, **kwargs): + self.hooks[key] += 1 + return real(*args, **kwargs) + + return wrapper + + monkeypatch.setattr(lib, "call", call) + monkeypatch.setattr( + codec, "_gb_default", counting("gb_default", real_gb_default) + ) + monkeypatch.setattr(codec, "_default", counting("default", real_default)) + monkeypatch.setattr(codec, "_prep", counting("prep", real_prep)) + + def entries(self): + return [c[0] for c in self.crossings] + + +@pytest.fixture +def spy(monkeypatch): + return _Spy(monkeypatch) + + +@pytest.fixture +def db(temp_db): + for name in ("Declared", "Free"): + temp_db.command("sql", f"CREATE VERTEX TYPE {name}") + temp_db.command("sql", f"CREATE PROPERTY {name}.id LONG") + temp_db.command("sql", "CREATE PROPERTY Declared.embedding ARRAY_OF_FLOATS") + temp_db.command("sql", "CREATE PROPERTY Free.born DATETIME") + return temp_db + + +def _x(n=ROWS, dim=DIM, seed=7): + return np.random.default_rng(seed).standard_normal((n, dim)).astype(np.float32) + + +def _stored(db, type_name, prop="embedding"): + sql = f"SELECT id, {prop} AS v FROM {type_name}" # nosec B608 - test-owned names + return {r["id"]: r["v"] for r in db.query("sql", sql).to_list()} + + +def _assert_bit_equal(stored, x): + assert sorted(stored) == list(range(len(x))) + for i in range(len(x)): + assert np.array(stored[i], dtype=np.float32).tobytes() == x[i].tobytes() + + +@pytest.mark.parametrize("type_name", ["Declared", "Free"]) +def test_a_float32_column_is_one_matrix_with_no_hook_call_per_row(db, spy, type_name): + x = _x() + rows = [{"id": i, "embedding": x[i]} for i in range(ROWS)] + with db.graph_batch(commit_every=CHUNK) as gb: + assert len(gb.create_vertices(type_name, rows)) == ROWS + assert spy.entries() == ["arcadedb_n_gb_bin"] * 3 # one crossing per chunk of 100 + assert spy.hooks == {"gb_default": 0, "default": 0, "prep": 0}, spy.hooks + assert [c[3] for c in spy.crossings] == [CHUNK * DIM, CHUNK * DIM, 50 * DIM] + for _, text, buf, count in spy.crossings: + args = json.loads(text) + assert args["vecs"] == {"names": ["embedding"], "dims": [DIM], "offs": [0]} + assert '"B"' not in text and '"F"' not in text # no tag and no number text + assert all(set(row) == {"id"} for row in args["rows"]) + assert isinstance( + buf, ctypes.Array + ) # a view of the matrix, not a copy of its bytes + _assert_bit_equal(_stored(db, type_name), x) + + +def test_two_columns_and_a_date_time_keep_the_buffer_layout(db, spy): + x, y = _x(), _x(dim=8, seed=8) + when = datetime(2020, 1, 2, 3, 4, 5) + rows = [ + {"id": i, "embedding": x[i], "second": y[i], "born": when} for i in range(ROWS) + ] + with db.graph_batch(commit_every=CHUNK) as gb: + gb.create_vertices("Free", rows) + assert spy.entries() == ["arcadedb_n_gb_bin"] * 3 + first = json.loads(spy.crossings[0][1]) + assert first["vecs"] == { + "names": ["embedding", "second"], + "dims": [DIM, 8], + "offs": [0, CHUNK * DIM], + } + assert spy.crossings[0][3] == CHUNK * DIM + CHUNK * 8 + _assert_bit_equal(_stored(db, "Free"), x) + _assert_bit_equal(_stored(db, "Free", "second"), y) + sql = "SELECT born FROM Free WHERE id = 7" + assert db.query("sql", sql).first().get("born") == when + + +def test_a_float32_array_that_does_not_qualify_follows_the_matrices_in_the_buffer( + db, spy +): + x = _x(60) + rows = [{"id": i, "embedding": x[i], "short": x[i][: 2 + i % 2]} for i in range(60)] + with db.graph_batch(commit_every=CHUNK) as gb: + gb.create_vertices("Free", rows) + assert spy.entries() == ["arcadedb_n_gb_bin"] + text, count = spy.crossings[0][1], spy.crossings[0][3] + assert json.loads(text)["vecs"]["offs"] == [0] + assert json.loads(text)["rows"][0]["short"] == {"\u0001": ["B", 60 * DIM, 2]} + assert count == 60 * DIM + sum(2 + i % 2 for i in range(60)) + _assert_bit_equal(_stored(db, "Free"), x) + short = _stored(db, "Free", "short") + for i in range(60): + assert ( + np.array(short[i], dtype=np.float32).tobytes() + == x[i][: 2 + i % 2].tobytes() + ) + + +def test_a_float64_or_list_column_keeps_the_text_entry(db, spy): + x = _x(30) + rows = [{"id": i, "embedding": x[i].astype(np.float64)} for i in range(30)] + with db.graph_batch(commit_every=CHUNK) as gb: + gb.create_vertices("Free", rows) + gb.create_vertices( + "Declared", [{"id": i, "embedding": x[i].tolist()} for i in range(30)] + ) + assert spy.entries() == ["arcadedb_n_gb"] * 2 + assert all('"vecs"' not in c[1] for c in spy.crossings) + _assert_bit_equal(_stored(db, "Declared"), x) + + +def test_a_library_without_the_entry_keeps_the_text_path(db, spy, monkeypatch): + monkeypatch.setattr(lib, "GB_BIN", False) + x = _x(30) + with db.graph_batch(commit_every=CHUNK) as gb: + gb.create_vertices("Free", [{"id": i, "embedding": x[i]} for i in range(30)]) + assert spy.entries() == ["arcadedb_n_gb"] + assert '"F"' in spy.crossings[0][1] and '"vecs"' not in spy.crossings[0][1] + _assert_bit_equal(_stored(db, "Free"), x) + + +def test_a_vector_column_outside_the_side_buffer_is_refused(db): + x = _x(4) + with db.graph_batch() as gb: + text = json.dumps( + { + "type": "Free", + "rows": [{"id": i} for i in range(4)], + "vecs": {"names": ["embedding"], "dims": [DIM], "offs": [8]}, + } + ) + buf = x.tobytes() # 4 vectors of 16 floats: offset 8 leaves room for 56, not 64 + reply = json.loads( + lib.call( + "arcadedb_n_gb_bin", + gb._gb, + b"create_vertices", + text.encode(), + buf, + x.size, + ) + ) + assert not reply.get("ok") + assert "outside the side buffer" in json.dumps(reply) + assert db.query("sql", "SELECT count(*) AS c FROM Free").first().get("c") == 0 + + +def test_the_encoder_in_isolation(): + x = _x(5, 4) + rows = [{"id": i} for i in range(5)] + text, buf, count = codec.encode_vertex_args("T", rows, ["v"], [x]) + assert json.loads(text) == { + "type": "T", + "rows": rows, + "vecs": {"names": ["v"], "dims": [4], "offs": [0]}, + } + assert count == 20 and bytes(buf) == x.tobytes() + frozen = x.copy() + frozen.setflags( + write=False + ) # a read-only matrix cannot be viewed writable: its bytes cross instead + assert codec.encode_vertex_args("T", rows, ["v"], [frozen])[1] == x.tobytes() + assert ( + codec._plain_rows(rows) + and codec._plain_rows([{}, {}]) + and codec._plain_rows([]) + ) + assert not codec._plain_rows([{"a": np.float64(1)}]) + assert not codec._plain_rows([{"a": [1]}]) + assert not codec._plain_rows([{1: 2}]) diff --git a/tests/test_vertex_vector_loads.py b/tests/test_vertex_vector_loads.py new file mode 100644 index 00000000..60718b1e --- /dev/null +++ b/tests/test_vertex_vector_loads.py @@ -0,0 +1,344 @@ +"""GraphBatch.create_vertices: float32 vectors of any property cross as one matrix, on both wheels (#337). + +A row value that is a Java ``float[]`` (what ``to_java_float_array`` returns on the JVM wheel) or a numpy float32 array (what it returns on +the native wheel) is a vector candidate for every property, declared ``ARRAY_OF_FLOATS`` or not. Before, the JVM wheel refused a Java +``float[]`` as a candidate and sent the whole load through the property-matrix path (a JPype conversion of every value of every row), and +both wheels wrote one tag per row for a float32 array. Now one float32 matrix per property and chunk crosses (``take_vector_columns``). + +This file pins what is stored (bit for bit, for both wheels, with the one call both wheels share) and the shared split itself. The crossing +counts are in test_graph_batch_vector_crossings_jvm.py (JPype) and test_native_vertex_vector_crossings.py (ctypes). +""" + +import numpy as np +import pytest + +import arcadedb +from arcadedb import type_conversion as tc + +N = 40 +TYPES = ["Declared", "Free"] # embedding declared ARRAY_OF_FLOATS, and not declared +SHAPES = ["to_java_float_array", "float32_ndarray"] + + +def _vectors(dim, n=N, seed=5): + """float32 values a decimal text of 6 digits would not hold: -0.0, the largest, the smallest normal and the smallest subnormal.""" + x = np.random.default_rng(seed).standard_normal((n, dim)).astype(np.float32) + x[0, 0] = np.float32(-0.0) + x[1, 0] = np.finfo(np.float32).max + x[1, 1] = np.finfo(np.float32).tiny + x[1, 2] = np.finfo(np.float32).smallest_subnormal + return x + + +def _make(shape): + if shape == "to_java_float_array": + return ( + arcadedb.to_java_float_array + ) # a float[] on the JVM wheel, a float32 ndarray on the native wheel + return lambda vector: vector + + +def _bits(stored): + return ( + np.array(stored, dtype=np.float64).astype(np.float32).view(np.uint32).tolist() + ) + + +@pytest.fixture +def db(temp_db): + for name in ("Declared", "Bulk", "One"): + temp_db.command("sql", f"CREATE VERTEX TYPE {name}") + temp_db.command("sql", f"CREATE PROPERTY {name}.id LONG") + temp_db.command("sql", "CREATE PROPERTY Declared.embedding ARRAY_OF_FLOATS") + temp_db.command("sql", "CREATE PROPERTY Declared.born DATETIME") + temp_db.command("sql", "CREATE VERTEX TYPE Free") + temp_db.command("sql", "CREATE PROPERTY Free.id LONG") + temp_db.command("sql", "CREATE PROPERTY Free.born DATETIME") + temp_db.command("sql", "CREATE PROPERTY Bulk.embedding ARRAY_OF_FLOATS") + temp_db.command("sql", "CREATE PROPERTY One.embedding ARRAY_OF_FLOATS") + return temp_db + + +def _stored(db, type_name, prop="embedding"): + sql = f"SELECT id, {prop} AS v FROM {type_name}" # nosec B608 - test-owned names + return {r["id"]: r["v"] for r in db.query("sql", sql).to_list()} + + +def _count(db, type_name): + sql = f"SELECT count(*) AS c FROM {type_name}" # nosec B608 - test-owned name + return db.query("sql", sql).first().get("c") + + +@pytest.mark.parametrize("type_name", TYPES) +@pytest.mark.parametrize("dim", [64, 96]) +@pytest.mark.parametrize("shape", SHAPES) +def test_float32_vectors_are_stored_bit_equal(db, type_name, dim, shape): + x = _vectors(dim) + make = _make(shape) + with db.graph_batch() as gb: + rids = gb.create_vertices( + type_name, [{"id": i, "embedding": make(x[i])} for i in range(N)] + ) + assert len(rids) == N and len(set(rids)) == N + stored = _stored(db, type_name) + assert sorted(stored) == list(range(N)) + for i in range(N): + assert _bits(stored[i]) == x[i].view(np.uint32).tolist() + for i in (0, 17, N - 1): # the RIDs come back in row order + sql = f"SELECT id FROM {rids[i]}" # nosec B608 - a RID from this test + assert db.query("sql", sql).first().get("id") == i + + +@pytest.mark.parametrize("shape", SHAPES) +def test_the_matrix_stores_what_the_per_vertex_path_stores(db, shape): + x = _vectors(32, 12) + rows = [{"id": i, "embedding": _make(shape)(x[i])} for i in range(12)] + with db.graph_batch() as gb: + gb.create_vertices("Bulk", rows) + with db.graph_batch() as gb: + for row in rows: + gb.create_vertex("One", **row) + assert _stored(db, "Bulk") == _stored(db, "One") + assert _count(db, "Bulk") == 12 + + +def test_float64_numpy_rows_keep_their_documented_values(db): + x64 = np.random.default_rng(3).random((N, 64)) * 1000.0 + 1e-3 + rows = [{"id": i, "embedding": x64[i]} for i in range(N)] + with db.graph_batch() as gb: + gb.create_vertices("Declared", rows) + gb.create_vertices("Free", rows) + declared, free = _stored(db, "Declared"), _stored(db, "Free") + for i in range(N): + # declared ARRAY_OF_FLOATS: the engine stores floats, so the doubles round to float32 + assert _bits(declared[i]) == x64[i].astype(np.float32).view(np.uint32).tolist() + # undeclared: the list keeps its doubles, exactly + assert np.array(free[i], dtype=np.float64).tolist() == x64[i].tolist() + + +def test_lists_of_floats_keep_their_documented_values(db): + x64 = np.random.default_rng(4).random((N, 40)) * 1000.0 + 1e-3 + rows = [{"id": i, "embedding": x64[i].tolist()} for i in range(N)] + with db.graph_batch() as gb: + gb.create_vertices("Declared", rows) + gb.create_vertices("Free", rows) + declared, free = _stored(db, "Declared"), _stored(db, "Free") + for i in range(N): + assert _bits(declared[i]) == x64[i].astype(np.float32).view(np.uint32).tolist() + assert np.array(free[i], dtype=np.float64).tolist() == x64[i].tolist() + + +@pytest.mark.parametrize("type_name", TYPES) +@pytest.mark.parametrize("shape", SHAPES) +def test_ragged_null_and_missing_vectors_take_the_fallback(db, type_name, shape): + x = _vectors(16, 6) + make = _make(shape) + rows = [ + {"id": 0, "embedding": make(x[0])}, + {"id": 1, "embedding": make(x[1][:9])}, # another length + {"id": 2, "embedding": None}, + {"id": 3}, # no vector + {"id": 4, "embedding": make(x[4])}, + None, # a vertex without properties + ] + with db.graph_batch() as gb: + assert len(gb.create_vertices(type_name, rows)) == 6 + stored = _stored(db, type_name) + assert ( + _count(db, type_name) == 6 and None in stored + ) # the vertex of the None row has no id + assert _bits(stored[0]) == x[0].view(np.uint32).tolist() + assert _bits(stored[1]) == x[1][:9].view(np.uint32).tolist() + assert stored[2] is None and stored[3] is None + assert _bits(stored[4]) == x[4].view(np.uint32).tolist() + + +@pytest.mark.parametrize("type_name", TYPES) +@pytest.mark.parametrize("shape", SHAPES) +def test_a_float64_row_among_float32_rows_loses_nothing(db, type_name, shape): + x = _vectors(16, 8) + make = _make(shape) + rows = [{"id": i, "embedding": make(x[i]), "tag": f"t{i}"} for i in range(8)] + rows[5] = {"id": 5, "tag": "t5", "embedding": make(x[5].astype(np.float64))} + with db.graph_batch() as gb: + gb.create_vertices(type_name, rows) + stored, tags = _stored(db, type_name), _stored(db, type_name, "tag") + assert [tags[i] for i in range(8)] == [f"t{i}" for i in range(8)] + for i in range(8): + if type_name == "Free" and i == 5 and shape == "float32_ndarray": + # a float64 array on an undeclared property keeps its doubles (it is no float32 vector) + assert ( + np.array(stored[i], dtype=np.float64).tolist() + == x[5].astype(np.float64).tolist() + ) + else: + assert _bits(stored[i]) == x[i].view(np.uint32).tolist() + + +@pytest.mark.parametrize("type_name", TYPES) +@pytest.mark.parametrize("shape", SHAPES) +def test_the_other_properties_stay_with_their_rows(db, type_name, shape): + x = _vectors(16, 12) + make = _make(shape) + rows = [] + for i in range(12): + row = { + "id": i, + "name": f"n{i}", + "score": i * 0.5, + "flag": i % 2 == 0, + "note": None, + "embedding": make(x[i]), + } + if i % 4 == 3: # another key order: the rows no longer share one + row = dict(reversed(list(row.items()))) + rows.append(row) + with db.graph_batch() as gb: + gb.create_vertices(type_name, rows) + sql = f"SELECT id, name, score, flag, embedding FROM {type_name} ORDER BY id" # nosec B608 - test-owned name + got = db.query("sql", sql).to_list() + assert [r["id"] for r in got] == list(range(12)) + for i, r in enumerate(got): + assert ( + r["name"] == f"n{i}" and r["score"] == i * 0.5 and r["flag"] is (i % 2 == 0) + ) + assert _bits(r["embedding"]) == x[i].view(np.uint32).tolist() + + +@pytest.mark.parametrize("type_name", TYPES) +@pytest.mark.parametrize("shape", SHAPES) +def test_two_vector_properties_and_a_date_time_in_one_load(db, type_name, shape): + from datetime import datetime + + x, y = _vectors(8, 20), _vectors(8, 20, seed=9) + make = _make(shape) + rows = [ + { + "id": i, + "embedding": make(x[i]), + "second": make(y[i]), + "born": datetime(2020, 1, 2, 3, 4, 5), + } + for i in range(20) + ] + with db.graph_batch() as gb: + gb.create_vertices(type_name, rows) + first, second = _stored(db, type_name), _stored(db, type_name, "second") + for i in range(20): + assert _bits(first[i]) == x[i].view(np.uint32).tolist() + assert _bits(second[i]) == y[i].view(np.uint32).tolist() + sql = f"SELECT born FROM {type_name} WHERE id = 7" # nosec B608 - test-owned name + assert db.query("sql", sql).first().get("born") == datetime(2020, 1, 2, 3, 4, 5) + + +@pytest.mark.parametrize("type_name", TYPES) +@pytest.mark.parametrize("shape", SHAPES) +def test_a_load_over_the_chunk_boundary(db, type_name, shape): + make = _make(shape) + with db.graph_batch() as gb: + # the rows per crossing: the JVM wheel's 100,000, the native wheel's commit_every (1000 unless graph_batch() got another value) + chunk = getattr(gb, "_vertex_chunk", None) or gb._BULK_CHUNK + n = chunk + 3 + x = _vectors(4, n) + rids = gb.create_vertices( + type_name, [{"id": i, "embedding": make(x[i])} for i in range(n)] + ) + assert len(rids) == n and len(set(rids)) == n + assert _count(db, type_name) == n + stored = _stored(db, type_name) + got = np.array([stored[i] for i in range(n)], dtype=np.float64).astype(np.float32) + assert (got.view(np.uint32) == x.view(np.uint32)).all() + for i in ( + 0, + chunk - 1, + chunk, + n - 1, + ): # the RIDs come back in row order across the boundary + sql = f"SELECT id FROM {rids[i]}" # nosec B608 - a RID from this test + assert db.query("sql", sql).first().get("id") == i + + +@pytest.mark.parametrize("shape", SHAPES) +def test_insert_many_takes_the_same_vectors(db, shape): + db.command("sql", "CREATE DOCUMENT TYPE Doc") + db.command("sql", "CREATE PROPERTY Doc.id LONG") + db.command("sql", "CREATE PROPERTY Doc.embedding ARRAY_OF_FLOATS") + x = _vectors(32, N) + make = _make(shape) + assert ( + db.insert_many("Doc", [{"id": i, "embedding": make(x[i])} for i in range(N)]) + == N + ) + stored = _stored(db, "Doc") + for i in range(N): + assert _bits(stored[i]) == x[i].view(np.uint32).tolist() + + +# ---- the split itself (no database): the one function both wheels call + + +def _rows(x, **extra): + return [{"a": i, "b": f"s{i}", "v": x[i], **extra} for i in range(len(x))] + + +def test_take_vector_columns_takes_a_float32_column_of_any_property(): + x = _vectors(5, 7) + names, matrices, rest = tc.take_vector_columns(_rows(x), (), undeclared=True) + assert names == ["v"] and matrices[0].dtype == np.float32 + assert matrices[0].shape == (7, 5) and matrices[0].flags.c_contiguous + assert matrices[0].tobytes() == x.tobytes() # bit for bit + assert rest == [{"a": i, "b": f"s{i}"} for i in range(7)] + assert all(list(row) == ["a", "b"] for row in rest) + + +def test_take_vector_columns_leaves_an_undeclared_column_that_is_no_float32_vector(): + x = _vectors(5, 7) + as_lists = [{"v": row.tolist()} for row in x] + as_doubles = [{"v": row.astype(np.float64)} for row in x] + assert tc.take_vector_columns(as_lists, (), undeclared=True) is None + assert tc.take_vector_columns(as_doubles, (), undeclared=True) is None + assert tc.take_vector_columns(as_lists, ["v"])[1][0].shape == (7, 5) + assert tc.take_vector_columns(as_doubles, ["v"])[1][0].shape == (7, 5) + assert ( + tc.take_vector_columns(_rows(x), ()) is None + ) # undeclared names are tried only when asked + + +def test_take_vector_columns_declines_a_ragged_null_or_missing_column(): + x = _vectors(5, 4) + for rows in ( + [{"v": x[0]}, {"v": x[1][:3]}], + [{"v": x[0]}, {"v": None}], + [{"v": x[0]}, {"w": 1}], + [{"v": x[0]}, None], + [{"v": np.zeros(0, dtype=np.float32)}], + [{"v": x}], # a 2-D array per row + ): + assert tc.take_vector_columns(rows, ["v"], undeclared=True) is None + + +def test_take_vector_columns_rebuilds_the_rest_rows_in_each_rows_own_key_order(): + x = _vectors(3, 2) + rows = [{"a": 1, "v": x[0], "b": 2}, {"b": 3, "v": x[1], "a": 4}] + rest = tc.take_vector_columns(rows, (), undeclared=True)[2] + assert rest == [{"a": 1, "b": 2}, {"b": 3, "a": 4}] + assert [list(row) for row in rest] == [["a", "b"], ["b", "a"]] + only = tc.take_vector_columns([{"v": x[0]}, {"v": x[1]}], (), undeclared=True)[2] + assert only == [{}, {}] and only[0] is not only[1] + two = [{"x": r, "y": r * 2, "k": i} for i, r in enumerate(x)] + names, _, rest = tc.take_vector_columns(two, (), undeclared=True) + assert sorted(names) == ["x", "y"] and rest == [{"k": 0}, {"k": 1}] + + +def test_take_vector_columns_takes_a_strided_or_read_only_row(): + wide = _vectors(10, 6) + strided = tc.take_vector_columns( + [{"v": wide[i, ::2]} for i in range(6)], (), undeclared=True + )[1][0] + assert strided.tobytes() == np.ascontiguousarray(wide[:, ::2]).tobytes() + frozen = wide.copy() + frozen.setflags(write=False) + matrix = tc.take_vector_columns( + [{"v": frozen[i]} for i in range(6)], (), undeclared=True + )[1][0] + assert matrix.tobytes() == wide.tobytes() and matrix.flags.writeable From c24f154cfb3619dec5e130fb670ea160626082e1 Mon Sep 17 00:00:00 2001 From: Taewoon Kim Date: Mon, 9 Nov 2026 21:21:37 +0900 Subject: [PATCH 18/49] Native parallel columnar load: interrupt-safe, token-safe, no silent loss (review of #328) The failure token is now made by the caller (arcadedb_n_async "token") and its id travels in the colp header, so the caller holds the id whatever happens to the crossing. An interrupt in it (or between the chunks of the row path) waits for the writers and reads the tokens before it propagates; a wait that raises still reads every token (the entries of an unread token lived forever); the error of a failed handover says how many rows were handed over and that some may have been stored. insert_table(parallel=True) no longer says the earlier batches were written in the caller's transaction (the writers commit them, a rollback does not remove them). Tests: duplicate keys (stored + reported >= n, stored == distinct), the writers' batch boundary, nothing lands after the call, an interrupt in the crossing and in the row path, a failing wait, the in-transaction message. Co-Authored-By: Claude Sonnet 5.5 --- .../java/org/humemai/nativelib/BulkOps.java | 32 +-- .../org/humemai/nativelib/NativeExtras.java | 50 +++-- src/arcadedb/_native/bulk.py | 25 ++- src/arcadedb/_native/columnar.py | 10 +- src/arcadedb/_native/database.py | 80 ++++--- src/arcadedb/_native/extras.py | 13 ++ src/arcadedb/dataframe.py | 25 ++- tests/conftest.py | 2 + tests/test_native_columnar_pack.py | 10 +- ...t_native_columnar_parallel_review_fixes.py | 200 ++++++++++++++++++ 10 files changed, 368 insertions(+), 79 deletions(-) create mode 100644 tests/test_native_columnar_parallel_review_fixes.py diff --git a/native/src/main/java/org/humemai/nativelib/BulkOps.java b/native/src/main/java/org/humemai/nativelib/BulkOps.java index 40f789e0..cdf3cd98 100644 --- a/native/src/main/java/org/humemai/nativelib/BulkOps.java +++ b/native/src/main/java/org/humemai/nativelib/BulkOps.java @@ -63,22 +63,20 @@ static long insertColumns(final Database db, final String type, final String hea /** * The pointer form (arcadedb_n_insert_colp): the table holds SLOTS longs per column, see {@link #insertColumns(Database, String, Map, long[], ErrorCallback)}. - * Returns the reply text: {"ok":true,"n":rows}, and for a header with "parallel":true also "fid", the token that the "failures" operation of - * arcadedb_n_async reads (see there). In that mode the records are handed to the database's async executor, the call does not wait for - * the writers, and it neither opens, commits nor rolls back a transaction ("commit" does not apply), exactly as insert_parallel does for rows. + * Returns the reply text {"ok":true,"n":rows}. A header with "parallel":true must also carry "fid", a failure token that the caller made + * with the "token" operation of arcadedb_n_async: the records go to the database's async executor, the call does not wait for the writers + * and neither opens, commits nor rolls back a transaction ("commit" does not apply), exactly as insert_parallel does for rows. The + * caller waits for the writers and reads (and so removes) the token with the "failures" operation, also when this call throws or never + * returns to it: the token is the caller's, so this call never drops it. */ @SuppressWarnings("unchecked") static String insertColumnsReply(final Database db, final String type, final String header, final long[] spec) { final Map h = (Map) Json.parse(header); if (!Boolean.TRUE.equals(h.get("parallel"))) return "{\"ok\":true,\"n\":" + insertColumns(db, type, h, spec, null) + "}"; - final NativeExtras.FailureToken token = new NativeExtras.FailureToken(); - try { - return "{\"ok\":true,\"n\":" + insertColumns(db, type, h, spec, token.onError) + ",\"fid\":" + token.fid + "}"; - } catch (final Throwable t) { - token.drop(); // the caller gets the error, not a token to read failures from - throw t; - } + if (!(h.get("fid") instanceof Number fid)) + throw new IllegalArgumentException("a parallel columnar insert needs a failure token (header \"fid\", from the \"token\" operation)"); + return "{\"ok\":true,\"n\":" + insertColumns(db, type, h, spec, NativeExtras.FailureToken.of(fid.longValue()).onError) + "}"; } /** @@ -165,10 +163,16 @@ private static long insertColumns(final Database db, final String type, final Ma throw new IllegalArgumentException("Cannot bulk-insert into '" + type + "' because it is an edge type; load the vertices, then create the edges (graph_batch().new_edges() or CREATE EDGE)"); final DatabaseAsyncExecutor async = db.async(); - for (int i = 0; i < n; i++) { - final MutableDocument doc = vertex ? db.newVertex(type) : db.newDocument(type); - setRow(doc, i, names, kinds, dims, data, nulls, blobs, scratch); - async.createRecord(doc, null, onError); + int handed = 0; + try { + for (; handed < n; handed++) { + final MutableDocument doc = vertex ? db.newVertex(type) : db.newDocument(type); + setRow(doc, handed, names, kinds, dims, data, nulls, blobs, scratch); + async.createRecord(doc, null, onError); + } + } catch (final Throwable t) { + // the records handed over before the failure are the writers' now, whatever this call does + throw new IllegalStateException("after handing " + handed + " of " + n + " rows to the async writers (those may have been stored): " + t, t); } return n; } diff --git a/native/src/main/java/org/humemai/nativelib/NativeExtras.java b/native/src/main/java/org/humemai/nativelib/NativeExtras.java index c0e87e87..404181b5 100644 --- a/native/src/main/java/org/humemai/nativelib/NativeExtras.java +++ b/native/src/main/java/org/humemai/nativelib/NativeExtras.java @@ -527,19 +527,42 @@ else if (opName.equals("create_vertices")) private static final Map ASYNC_FAILS = new ConcurrentHashMap<>(); private static final Map> ASYNC_FIRST = new ConcurrentHashMap<>(); - /** A failure token for one parallel load: the id the caller reads with the "failures" operation, and the callback that counts. */ + /** + * A failure token for one parallel load: the id the caller reads (and so removes) with the "failures" operation, and the callback that + * counts. The caller creates it before the load ("token" operation) and passes the id in the load's header, so that it holds the id + * whatever happens to the call, an interrupt included, and always has the means to wait for the writers and remove the entries. + */ static final class FailureToken { - final long fid = ArcadeNative.IDS.getAndIncrement(); - final AtomicLong count = new AtomicLong(); - final AtomicReference first = new AtomicReference<>(); - final ErrorCallback onError = t -> { - count.incrementAndGet(); - first.compareAndSet(null, t); - }; + final long fid; + final AtomicLong count; + final AtomicReference first; + final ErrorCallback onError; + + private FailureToken(final long fid, final AtomicLong count, final AtomicReference first) { + this.fid = fid; + this.count = count; + this.first = first; + this.onError = t -> { + count.incrementAndGet(); + first.compareAndSet(null, t); + }; + } - FailureToken() { - ASYNC_FAILS.put(fid, count); - ASYNC_FIRST.put(fid, first); + /** A new token, registered for the "failures" operation. */ + static FailureToken create() { + final FailureToken token = new FailureToken(ArcadeNative.IDS.getAndIncrement(), new AtomicLong(), new AtomicReference<>()); + ASYNC_FAILS.put(token.fid, token.count); + ASYNC_FIRST.put(token.fid, token.first); + return token; + } + + /** The token that "token" created, for the load that its id was passed to. */ + static FailureToken of(final long fid) { + final AtomicLong count = ASYNC_FAILS.get(fid); + final AtomicReference first = ASYNC_FIRST.get(fid); + if (count == null || first == null) + throw new IllegalArgumentException("unknown failure token " + fid + " (already read, or never created)"); + return new FailureToken(fid, count, first); } /** The load failed before it returned: the caller gets the error, not a token to read failures from. */ @@ -846,7 +869,7 @@ else if (a.get("slot") instanceof Number slot) ex.appendSamples((String) a.get("type"), stamps, columns); } case "insert_parallel" -> { - final FailureToken token = new FailureToken(); + final FailureToken token = FailureToken.create(); final long fid = token.fid; final ErrorCallback onError = token.onError; final String type = (String) a.get("type"); @@ -867,6 +890,9 @@ else if (a.get("slot") instanceof Number slot) } return ArcadeNative.out("{\"ok\":true,\"fid\":" + fid + "}"); } + case "token" -> { + return ArcadeNative.out("{\"ok\":true,\"fid\":" + FailureToken.create().fid + "}"); + } case "failures" -> { final long fid = ((Number) a.get("fid")).longValue(); final AtomicLong count = ASYNC_FAILS.remove(fid); diff --git a/src/arcadedb/_native/bulk.py b/src/arcadedb/_native/bulk.py index dcaca257..9851848f 100644 --- a/src/arcadedb/_native/bulk.py +++ b/src/arcadedb/_native/bulk.py @@ -8,6 +8,7 @@ import os +from ..exceptions import ArcadeDBError from . import columnar, lib _ENABLED = os.environ.get("ARCADEDB_NATIVE_BULK", "1") != "0" @@ -22,7 +23,7 @@ def _usable(): ) -def _send(db, type_name, packed, rows, commit_every, prefix, parallel=False): +def _send(db, type_name, packed, rows, commit_every, prefix, fid=None): from .database import _call spec = columnar.spec_of(packed) @@ -31,7 +32,7 @@ def _send(db, type_name, packed, rows, commit_every, prefix, parallel=False): "arcadedb_n_insert_colp", db._h, lib.b(type_name), - lib.b(columnar.header_of(rows, commit_every, packed, parallel)), + lib.b(columnar.header_of(rows, commit_every, packed, fid)), spec, len(spec), ) # `packed` holds the arrays the addresses point into until the call returns @@ -55,9 +56,10 @@ def insert_columns_fast( def insert_columns_parallel(db, type_name, columns, declared=None): - """Hand ``columns`` (name -> column) to the async writers in one crossing: ``(rows, failure token)``, or None when a column cannot be - sent this way. The call does not wait for the writers: the caller waits for them and reads the failures for the token - (``Database._finish_parallel``), as for ``insert_many(parallel=True)``. + """Hand ``columns`` (name -> column) to the async writers in one crossing, wait for them and return the row count, or None when a + column cannot be sent this way (nothing was handed over then). A record that a writer rejects raises + (``Database._finish_parallel``), as for ``insert_many(parallel=True)``, and so does an interrupt, after the writers have finished: + the failure token is made before the crossing, so the call always has the id to wait for and to read. """ if not _usable(): return None @@ -65,8 +67,17 @@ def insert_columns_parallel(db, type_name, columns, declared=None): if packed is None: return None rows, cols = packed - reply = _send(db, type_name, cols, rows, 0, "Failed to bulk-insert", parallel=True) - return int(reply["n"]), reply["fid"] + from . import extras + + fid = extras.failure_token(db) + try: + _send(db, type_name, cols, rows, 0, "Failed to bulk-insert", fid=fid) + except ArcadeDBError as e: + raise db._parallel_load_failed(type_name, e, [fid]) from e + except BaseException: + db._abandon_parallel([fid]) # a ^C: the writers still run, so wait for them + raise + return db._finish_parallel(type_name, [fid], rows) def insert_rows_columnar(db, type_name, chunk): diff --git a/src/arcadedb/_native/columnar.py b/src/arcadedb/_native/columnar.py index fe62837a..27d261a4 100644 --- a/src/arcadedb/_native/columnar.py +++ b/src/arcadedb/_native/columnar.py @@ -343,9 +343,10 @@ def spec_of(packed): return (ctypes.c_int64 * len(flat))(*flat) -def header_of(rows, commit_every, packed, parallel=False): - """The JSON header text: row count, commit interval and ``[name, kind, dim]`` per column; ``"parallel": true`` hands the records to the - async executor (the image then ignores the commit interval and the reply carries the failure token). +def header_of(rows, commit_every, packed, fid=None): + """The JSON header text: row count, commit interval and ``[name, kind, dim]`` per column; with a failure token ``fid`` it also says + ``"parallel": true``, which hands the records to the async executor (the image then ignores the commit interval and counts the + failed records on that token). """ import json @@ -354,8 +355,9 @@ def header_of(rows, commit_every, packed, parallel=False): "commit": min(int(commit_every), 2**31 - 1), "cols": [[c.name, c.kind, c.dim] for c in packed], } - if parallel: + if fid is not None: header["parallel"] = True + header["fid"] = fid return json.dumps(header) diff --git a/src/arcadedb/_native/database.py b/src/arcadedb/_native/database.py index b0c07932..6fab6c58 100644 --- a/src/arcadedb/_native/database.py +++ b/src/arcadedb/_native/database.py @@ -995,20 +995,50 @@ def _command_many(self, language: str, command: str, rows) -> int: raise _error("Command failed", reply) return reply["n"] - def _parallel_load_failed(self, type_name, error) -> ArcadeDBError: - """The error for a parallel load that failed while handing rows to the writers: waits for the queue to drain first, so - the caller sees a final state, and says that the rows handed over before the failure may have been stored. + def _drain_parallel(self, tokens): + """Waits for the async writers, then reads, and so removes, the failure token of every parallel load handed in. Returns + ``(problem, failed, first)``: the error of the wait or of a read (or None), the failed records the writers reported on all the + tokens and the first failure's text. The tokens are read whatever the wait did: an unread token keeps its entries for good. """ + from . import extras + + problem = None try: self._async().wait_completion() - except Exception: # nosec B110 - the original error is the one to report - pass + except Exception as e: + problem = e + failed, first = 0, None + for token in tokens: + try: + result = extras.failures(self, token) + except Exception as e: + problem = problem or e + continue + if result["count"] and first is None: + first = result["first"] + failed += result["count"] + return problem, failed, first + + def _abandon_parallel(self, tokens): + """A parallel load that is being interrupted: waits for the writers (they run on) and removes the tokens; the error that is on its + way out is the one to report. + """ + self._drain_parallel(tokens) + + def _parallel_load_failed(self, type_name, error, tokens=()) -> ArcadeDBError: + """The error for a parallel load that failed while handing rows to the writers: waits for the queue to drain first, so + the caller sees a final state, and says that the rows handed over before the failure may have been stored. + """ + _, failed, _ = self._drain_parallel(tokens) text = str(error) prefix = "Failed to bulk-insert: " if text.startswith(prefix): text = text[len(prefix) :] + reported = ( + f"; the writers had reported {failed} failed record(s)" if failed else "" + ) return ArcadeDBError( - f"Failed to bulk-insert into '{type_name}': {text}; the rows handed to the " + f"Failed to bulk-insert into '{type_name}': {text}{reported}; the rows handed to the " f"parallel writers before the failure may have been stored (the call waited " f"for them before raising)" ) @@ -1030,32 +1060,28 @@ def _insert_many_parallel(self, type_name, rows) -> int: tokens.append(extras.insert_parallel(self, type_name, chunk)["fid"]) except ArcadeDBError as e: # as the JPype path: the rows handed to the writers before the failure are written whatever this call does - raise self._parallel_load_failed(type_name, e) from e + raise self._parallel_load_failed(type_name, e, tokens) from e n += len(chunk) except ArcadeDBError: raise except BaseException: - try: - self._async().wait_completion() - except Exception: # nosec B110 - the original error is the one to report - pass + self._abandon_parallel(tokens) raise return self._finish_parallel(type_name, tokens, n) def _finish_parallel(self, type_name, tokens, n) -> int: """The end of a parallel load: wait for the async writers, then read the failure count of every token and raise when a record failed.""" - from . import extras - try: - self._async().wait_completion() - except Exception as e: - raise ArcadeDBError(f"Failed to bulk-insert into '{type_name}': {e}") from e - n_failed, first = 0, None - for token in tokens: - result = extras.failures(self, token) - if result["count"] and first is None: - first = result["first"] - n_failed += result["count"] + problem, n_failed, first = self._drain_parallel(tokens) + except BaseException: + self._abandon_parallel( + tokens + ) # a ^C in the wait: finish it, and remove the tokens + raise + if problem is not None: + raise ArcadeDBError( + f"Failed to bulk-insert into '{type_name}': {problem}" + ) from problem if n_failed: raise ArcadeDBError( f"Failed to bulk-insert into '{type_name}': the parallel writers reported {n_failed} failed record(s) " @@ -1343,13 +1369,9 @@ def insert_columns( if parallel: from .bulk import insert_columns_parallel - try: - sent = insert_columns_parallel(self, type_name, columns) - except ArcadeDBError as e: - # as insert_many(parallel=True): rows handed to the writers before the failure may be stored - raise self._parallel_load_failed(type_name, e) from e - if sent is not None: - return self._finish_parallel(type_name, [sent[1]], sent[0]) + done = insert_columns_parallel(self, type_name, columns) + if done is not None: + return done else: from .bulk import insert_columns_fast diff --git a/src/arcadedb/_native/extras.py b/src/arcadedb/_native/extras.py index 5f8a4702..439fb9ef 100644 --- a/src/arcadedb/_native/extras.py +++ b/src/arcadedb/_native/extras.py @@ -915,6 +915,19 @@ def insert_parallel(database, type_name: str, rows) -> dict: ) +def failure_token(database) -> int: + """A new failure token for one parallel load: the caller makes it before the load, hands its id to the load and reads it with + :func:`failures` after the final wait, so it holds the id whatever happens to the load (an interrupt included). + """ + return _call( + "Failed to make a failure token", + "arcadedb_n_async", + database._h, + b"token", + lib.b(_enc({})), + )["fid"] + + def failures(database, token: int) -> dict: return _call( "Failed to read the parallel writers", diff --git a/src/arcadedb/dataframe.py b/src/arcadedb/dataframe.py index d65e51b8..46362e53 100644 --- a/src/arcadedb/dataframe.py +++ b/src/arcadedb/dataframe.py @@ -421,8 +421,12 @@ def insert_table( native = _is_native(db) written = 0 # a document load inside the caller's own transaction joins it (insert_columns commits - # nothing then); a vertex load refuses to run inside one - in_caller_transaction = (not is_vertex) and bool(db.is_transaction_active()) + # nothing then); a vertex load refuses to run inside one. parallel=True never joins it: the + # async writers commit the records themselves, so a rollback of the caller's transaction does + # not remove them + in_caller_transaction = ( + (not is_vertex) and (not parallel) and bool(db.is_transaction_active()) + ) graph_batch = db.graph_batch() if is_vertex else None try: for batch in table.to_batches(max_chunksize=batch_size): @@ -459,6 +463,13 @@ def insert_table( f"{exc}; the {written} row(s) of the batches before this one " f"were written in your transaction, not committed" ) from exc + if written and parallel: + # the async writers commit on their own, inside the caller's transaction or not + raise ArcadeDBError( + f"{exc}; the {written} row(s) of the batches before this one " + f"were handed to the async writers, which commit on their own: " + f"a rollback of your transaction does not remove them" + ) from exc if written: raise ArcadeDBError( f"{exc}; the {written} row(s) of the batches before this one " @@ -503,14 +514,12 @@ def _insert_documents_native( if parallel: from ._native.bulk import insert_columns_parallel - try: - sent = insert_columns_parallel( + if ( + insert_columns_parallel( db, type_name, columns, {} if declared is None else declared ) - except ArcadeDBError as e: - raise db._parallel_load_failed(type_name, e) from e - if sent is not None: - db._finish_parallel(type_name, [sent[1]], sent[0]) + is not None + ): return else: from ._native.bulk import insert_columns_fast diff --git a/tests/conftest.py b/tests/conftest.py index be5fa583..707833d1 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -49,6 +49,8 @@ collect_ignore.append("test_native_columnar_insert.py") # insert_columns(parallel=True) takes the columnar entry and stores what the row path stores (arcadedb_n_insert_colp with "parallel") collect_ignore.append("test_native_columnar_parallel.py") + # the review of the native parallel loads: no silent loss, an interrupt waits for the writers and reads the failure token + collect_ignore.append("test_native_columnar_parallel_review_fixes.py") # the rows of an insert_many chunk with date-times, Decimals, bytes and numpy scalars are told apart by the entry points they cross collect_ignore.append("test_native_mixed_chunk.py") # the native build's ingest defaults (insert_many and graph_batch commit size), observed on the native library diff --git a/tests/test_native_columnar_pack.py b/tests/test_native_columnar_pack.py index 56b283e1..af26f211 100644 --- a/tests/test_native_columnar_pack.py +++ b/tests/test_native_columnar_pack.py @@ -233,11 +233,11 @@ def test_the_address_table_and_header_describe_the_columns(): "cols": [["id", "q", 0], ["s", "s", 0]], } assert json.loads(columnar.header_of(rows, 2**40, packed))["commit"] == 2**31 - 1 - # "parallel" is in the header only when asked for (the image hands the records to the async writers then) - assert json.loads(columnar.header_of(rows, 10_000, packed, parallel=True))[ - "parallel" - ] - assert "parallel" not in json.loads(columnar.header_of(rows, 10_000, packed)) + # "parallel" and the failure token are in the header only when a token is given (the image hands the records to the async writers then) + parallel = json.loads(columnar.header_of(rows, 10_000, packed, fid=41)) + assert parallel["parallel"] is True and parallel["fid"] == 41 + plain = json.loads(columnar.header_of(rows, 10_000, packed)) + assert "parallel" not in plain and "fid" not in plain def test_rows_become_columns_only_when_every_row_has_the_same_keys(): diff --git a/tests/test_native_columnar_parallel_review_fixes.py b/tests/test_native_columnar_parallel_review_fixes.py new file mode 100644 index 00000000..72313ee6 --- /dev/null +++ b/tests/test_native_columnar_parallel_review_fixes.py @@ -0,0 +1,200 @@ +"""Review of the native parallel loads (#328): nothing is lost silently, an interrupt leaves no writer running and no token behind. + +``insert_columns(parallel=True)`` hands the records to the async writers in one crossing and waits for them before it returns. These +tests pin what a caller relies on: a record that a writer rejects is counted (``stored + reported failed >= rows``) and the stored +count is exactly the distinct keys; a load at the writers' batch boundary stores every row and nothing lands after the call returns; +a ^C during the crossing, or between the chunks of the row path, waits for the writers and reads (so removes) the failure token; a wait +that fails still reads the tokens; and the message of a failed parallel batch inside a caller's transaction does not claim that a +rollback removes rows the writers committed. Needs the native library: conftest.py collects this file on the native backend only. +""" + +import re +import time + +import numpy as np +import pytest + +from arcadedb._native import database as native_database +from arcadedb._native import extras +from arcadedb.exceptions import ArcadeDBError + +COLP = "arcadedb_n_insert_colp" + + +def _count(db, name): + sql = f"SELECT count(*) AS n FROM {name}" # nosec B608 - test-owned type name + return db.query("sql", sql).to_list()[0]["n"] + + +def _type(db, name="Col", unique=False): + db.command("sql", f"CREATE DOCUMENT TYPE {name} BUCKETS 3") + db.command("sql", f"CREATE PROPERTY {name}.id INTEGER") + db.command("sql", f"CREATE PROPERTY {name}.txt STRING") + if unique: + db.command("sql", f"CREATE INDEX ON {name} (id) UNIQUE") + + +class TokenSpy: + """The failure tokens that were made and the ones that were read (a read removes the token's entries in the image).""" + + def __init__(self, monkeypatch): + self.made, self.read = [], [] + real_make, real_read = extras.failure_token, extras.failures + real_rows = extras.insert_parallel + + def make(database): + fid = real_make(database) + self.made.append(fid) + return fid + + def failures(database, token): + self.read.append(token) + return real_read(database, token) + + def rows(database, type_name, chunk): + fid = real_rows(database, type_name, chunk) + self.made.append(fid["fid"]) + return fid + + monkeypatch.setattr(extras, "failure_token", make) + monkeypatch.setattr(extras, "failures", failures) + monkeypatch.setattr(extras, "insert_parallel", rows) + + +@pytest.fixture +def tokens(monkeypatch): + return TokenSpy(monkeypatch) + + +def _failed_count(error): + return int(re.search(r"reported (\d+) failed record", str(error)).group(1)) + + +def test_a_duplicate_key_is_stored_once_and_the_rest_are_reported(temp_db): + _type(temp_db, unique=True) + n, dupes = 3_000, 50 + ids = np.arange(n, dtype=np.int64) + ids[n - dupes :] = np.arange(dupes) # the last 50 repeat the first 50 keys + with pytest.raises(ArcadeDBError, match="failed record") as err: + temp_db.insert_columns("Col", {"id": ids, "txt": ["t"] * n}, parallel=True) + stored = _count(temp_db, "Col") + assert ( + stored + _failed_count(err.value) >= n + ) # every row is a stored row or a reported one + assert stored == len(set(ids.tolist())) == n - dupes + keys = {r["id"] for r in temp_db.query("sql", "SELECT id FROM Col").to_list()} + assert keys == set(ids.tolist()) # each key once, no key lost + + +def test_a_load_at_the_writers_batch_boundary_stores_every_row_and_nothing_later( + temp_db, +): + _type(temp_db) + threads = temp_db.async_executor().get_parallel_level() + n = threads * 10_240 + 1 # one row past a full batch for every writer + ids = np.arange(n, dtype=np.int64) + assert ( + temp_db.insert_columns("Col", {"id": ids, "txt": ["t"] * n}, parallel=True) == n + ) + stored = _count(temp_db, "Col") + assert stored == n + time.sleep(0.3) + temp_db.async_executor().wait_completion() + assert _count(temp_db, "Col") == n # nothing landed after the call returned + distinct = temp_db.query("sql", "SELECT count(DISTINCT id) AS c FROM Col") + assert distinct.to_list()[0]["c"] == n + + +def test_an_interrupt_in_the_crossing_waits_for_the_writers_and_reads_the_token( + temp_db, tokens, monkeypatch +): + _type(temp_db) + real_call = native_database._call + + def interrupted(prefix, name, *args): + reply = real_call(prefix, name, *args) + if name == COLP: + raise KeyboardInterrupt # the records are handed over; the reply never reaches the caller + return reply + + monkeypatch.setattr(native_database, "_call", interrupted) + n = 30_000 + with pytest.raises(KeyboardInterrupt): + temp_db.insert_columns( + "Col", {"id": np.arange(n, dtype=np.int64), "txt": ["t"] * n}, parallel=True + ) + assert ( + temp_db.async_executor().is_processing() is False + ) # waited: no writer runs on + assert len(tokens.made) == 1 and tokens.read == tokens.made # and the token is gone + assert ( + _count(temp_db, "Col") == n + ) # the rows handed over were all written, none half + + +def test_an_interrupt_between_the_chunks_of_the_row_path_waits_and_reads_the_tokens( + temp_db, tokens, monkeypatch +): + _type(temp_db) + real_rows = extras.insert_parallel + seen = [] + + def second_chunk_interrupted(database, type_name, chunk): + if seen: + raise KeyboardInterrupt + seen.append(1) + return real_rows(database, type_name, chunk) + + monkeypatch.setattr(extras, "insert_parallel", second_chunk_interrupted) + rows = [{"id": i, "txt": "t"} for i in range(25_000)] + with pytest.raises(KeyboardInterrupt): + temp_db.insert_many("Col", rows, parallel=True) + assert temp_db.async_executor().is_processing() is False + assert _count(temp_db, "Col") == 10_000 # the first chunk, whole + assert len(tokens.made) == 1 and tokens.read == tokens.made + + +def test_a_wait_that_fails_still_reads_the_tokens(temp_db, tokens, monkeypatch): + _type(temp_db) + real_wait = extras.NativeAsyncExecutor.wait_completion + state = {"raised": False} + + def failing_wait(self, timeout_ms=None): + if not state["raised"]: + state["raised"] = True + raise RuntimeError("the wait failed") + return real_wait(self, timeout_ms) + + monkeypatch.setattr(extras.NativeAsyncExecutor, "wait_completion", failing_wait) + n = 2_000 + with pytest.raises(ArcadeDBError, match="the wait failed"): + temp_db.insert_columns( + "Col", {"id": np.arange(n, dtype=np.int64), "txt": ["t"] * n}, parallel=True + ) + assert len(tokens.made) == 1 and tokens.read == tokens.made + temp_db.async_executor().wait_completion() + assert _count(temp_db, "Col") == n + + +def test_a_failed_parallel_batch_in_a_caller_transaction_does_not_promise_a_rollback( + temp_db, +): + import pyarrow as pa + + _type(temp_db, unique=True) + ids = list(range(300)) + ids[250] = 7 # a duplicate key in the third batch of 100 + table = pa.table({"id": pa.array(ids, pa.int64()), "txt": ["t"] * 300}) + temp_db.begin() + try: + with pytest.raises(ArcadeDBError) as err: + temp_db.insert_arrow("Col", table, batch_size=100, parallel=True) + message = str(err.value) + assert "not committed" not in message and "your transaction" in message + assert "async writers" in message and "does not remove them" in message + finally: + temp_db.rollback() + temp_db.async_executor().wait_completion() + assert ( + _count(temp_db, "Col") >= 200 + ) # the writers committed the first two batches: the rollback did not remove them From f2dbc7b12b213f7f2990704143a586a955cb60d2 Mon Sep 17 00:00:00 2001 From: Taewoon Kim Date: Mon, 9 Nov 2026 21:22:57 +0900 Subject: [PATCH 19/49] Test: vetoes and first-row errors through the eager form of save() (#317) A before_create veto on a new record, a before_update veto on both UPDATE forms, and an error met while the first row is computed (rerr), each compared with the lazy form of the same statement. Co-Authored-By: Claude Sonnet 5.5 --- tests/test_native_one_row_crossings.py | 140 ++++++++++++++++++++++++- 1 file changed, 139 insertions(+), 1 deletion(-) diff --git a/tests/test_native_one_row_crossings.py b/tests/test_native_one_row_crossings.py index f694445b..8f768565 100644 --- a/tests/test_native_one_row_crossings.py +++ b/tests/test_native_one_row_crossings.py @@ -11,6 +11,7 @@ deterministic. The oracle for the values and the errors is the lazy form itself (``db.command(...).first()``), kept for every other caller. """ +import contextlib import datetime as dt from decimal import Decimal @@ -18,7 +19,7 @@ import pytest from arcadedb import ArcadeDBError -from arcadedb._native import lib +from arcadedb._native import extras, lib, records class _Crossings: @@ -211,3 +212,140 @@ def test_an_update_outside_a_transaction_still_raises_before_reading(db, crossin with pytest.raises(ArcadeDBError, match="Transaction not begun"): record.save() assert crossings.names == ["arcadedb_n_tx"] + + +# -- vetoes and errors of the first row, through the form whose reply carries the row + + +def test_a_before_create_veto_on_a_new_record_writes_nothing_and_raises_nothing( + db, crossings +): + veto = db.on("before_create", lambda r: r.get("name") != "blocked", "D") + vetoes = extras.veto_count() + with db.transaction(): + doc = db.new_document("D") + doc.set("name", "blocked") + crossings.reset() + assert doc.save() is doc # no error: a veto is not a failure + assert crossings.names == ["exec_query"] # the veto came in the one crossing + assert extras.veto_count() == vetoes + 1 + assert doc.get_rid() == records._UNSAVED # the object was not given a record id + assert db.count_type("D") == 0 # nothing stored + veto.remove() + with db.transaction(): + doc.save() # the same object saves once the veto is lifted + assert db.count_type("D") == 1 + assert db.lookup_by_rid(str(doc.get_identity())).get("name") == "blocked" + + +def test_a_before_update_veto_writes_nothing_and_the_next_save_is_no_conflict( + db, crossings +): + rid = _saved(db, name="a") + veto = db.on("before_update", lambda r: r.get("name") != "frozen", "D") + vetoes = extras.veto_count() + mutable = db.lookup_by_rid(rid).modify() + mutable.set("name", "frozen") # a changed value of the kind stored as sent + with db.transaction(): + crossings.reset() + assert mutable.save() is mutable + assert crossings.names == ["arcadedb_n_tx", "exec_query", "exec_query"] + assert extras.veto_count() == vetoes + 1 + assert db.lookup_by_rid(rid).get("name") == "a" # nothing written + veto.remove() + mutable.set("name", "thawed") + with db.transaction(): + mutable.save() # the copy is as it was: the earlier veto left no conflict behind + assert db.lookup_by_rid(rid).get("name") == "thawed" + + +def test_a_before_update_veto_on_an_update_that_reads_its_row_back(db, crossings): + rid = _saved(db, name="a") + veto = db.on("before_update", lambda r: r.get("extra") != "x", "D") + vetoes = extras.veto_count() + mutable = db.lookup_by_rid(rid).modify() + mutable.set("extra", "x") # a new property: the statement is the RETURN AFTER one + with db.transaction(): + crossings.reset() + assert mutable.save() is mutable + assert crossings.names == ["arcadedb_n_tx", "exec_query", "exec_query"] + assert extras.veto_count() == vetoes + 1 + stored = db.lookup_by_rid(rid) + assert stored.get("extra") is None and stored.get("name") == "a" + veto.remove() + with db.transaction(): + mutable.save() + stored = db.lookup_by_rid(rid) + assert stored.get("extra") == "x" and stored.get("name") == "a" + + +@contextlib.contextmanager +def _rolled_back(db): + """A transaction that is always rolled back: a statement that failed leaves nothing to commit.""" + db.begin() + try: + yield + finally: + db.rollback() + + +def _raises_at(call): + """``("call", error)`` when the statement call raises, ``("read", error)`` when reading its first row does, ``(None, None)``.""" + try: + result = call() + except ArcadeDBError as error: + return "call", error + try: + result.first() + except ArcadeDBError as error: + return "read", error + return None, None + + +FIRST_ROW_ERRORS = { + "insert": ("INSERT INTO D SET v = 1 / (1 - 1)", False), + "update": ("UPDATE :rid SET v = 1 / (1 - 1)", True), +} + + +@pytest.mark.parametrize("case", list(FIRST_ROW_ERRORS)) +def test_an_error_met_while_the_first_row_is_computed_is_raised_when_it_is_read( + db, case +): + sql, with_rid = FIRST_ROW_ERRORS[case] + rid = _saved(db, v=1) + params = ({"rid": records.RID(rid)},) if with_rid else () + with _rolled_back(db): + lazy_phase, lazy = _raises_at(lambda: db.command("sql", sql, *params)) + with _rolled_back(db): + eager_phase, eager = _raises_at( + lambda: db._command_first_row("sql", sql, *params) + ) + assert ( + eager_phase == lazy_phase == "read" + ) # the engine computes the row when it is pulled: the reply carries it as rerr + assert str(eager).startswith("Reading the result set failed: ") + assert str(eager) == str(lazy) and type(eager) is type(lazy) + assert eager.native.get("cls") == lazy.native.get("cls") + + +def test_a_before_create_callback_that_raises_fails_save_as_the_lazy_form_does(db): + def explode(record): + raise RuntimeError("no creating today") + + handle = db.on("before_create", explode, "D") + with _rolled_back(db): + lazy_phase, lazy = _raises_at( + lambda: db.command("sql", "INSERT INTO D CONTENT :row", {"row": {"a": 1}}) + ) + with pytest.raises(ArcadeDBError) as eager: + with db.transaction(): + doc = db.new_document("D") + doc.set("a", 1) + doc.save() + handle.remove() + assert lazy_phase is not None + assert str(eager.value) == str(lazy) + assert "before_create" in str(eager.value) + assert "RuntimeError: no creating today" in str(eager.value) + assert db.count_type("D") == 0 From 4b317ba2552033a79abd954eb3641c337486b8ae Mon Sep 17 00:00:00 2001 From: Taewoon Kim Date: Mon, 9 Nov 2026 21:23:30 +0900 Subject: [PATCH 20/49] bulk dumps: raise the first bad row's error outside the except block (no json.dumps __context__); test the stored type across chunks (refs #318) Co-Authored-By: Claude Sonnet 5.5 --- src/arcadedb/type_conversion.py | 14 +++++++-- tests/test_bulk_single_pass_validation.py | 15 +++++++++ tests/test_graph_batch_single_pass.py | 37 +++++++++++++++++++++++ 3 files changed, 63 insertions(+), 3 deletions(-) diff --git a/src/arcadedb/type_conversion.py b/src/arcadedb/type_conversion.py index 8f90133c..2ec8f424 100644 --- a/src/arcadedb/type_conversion.py +++ b/src/arcadedb/type_conversion.py @@ -917,9 +917,17 @@ def _dumps_once(rows: Any, tagged: bool) -> str: if tagged else _dumps_checked(rows) ) - except (TypeError, ValueError): - _check_rows(rows, tagged) - raise + except (TypeError, ValueError) as error: + refused = error + else: + refused = None + if refused is not None: + # outside the except block, so the walk's error does not carry json.dumps's as __context__ (the old loop ran first) + try: + _check_rows(rows, tagged) + raise refused + finally: + del refused if ( _rows_need_walk(rows, tagged) or _SURROGATE_ESCAPE.search(text) diff --git a/tests/test_bulk_single_pass_validation.py b/tests/test_bulk_single_pass_validation.py index 5d9afe72..7fa56ed6 100644 --- a/tests/test_bulk_single_pass_validation.py +++ b/tests/test_bulk_single_pass_validation.py @@ -231,6 +231,21 @@ def test_the_first_bad_row_of_a_large_batch_gives_the_old_error(kind, tagged): assert (_outcome(new, only_later) == expected) == (kind in ("nan", "subnormal")) +@pytest.mark.parametrize("tagged", [False, True], ids=["untagged", "tagged"]) +def test_the_first_bad_rows_error_carries_no_context(tagged): + # json.dumps refuses the NaN of the second row first; the error is still the walk's for the first row, as it was when the + # loop ran before json.dumps, and it does not show the NaN error as the context it was raised in + rows = [{"a": 2**70}, {"b": math.nan}] + new = tc.json_bulk_dumps_tagged if tagged else tc.json_bulk_dumps + with pytest.raises(ValueError) as caught: + new(rows) + assert str(caught.value) == _outcome(lambda r: _old_dumps(r, tagged), rows)[1] + assert caught.value.__context__ is None + # with nothing for the walk to find, json.dumps's own error is the one raised + with pytest.raises(ValueError, match="Out of range float"): + new([{"b": math.nan}]) + + @pytest.mark.parametrize("tagged", [False, True], ids=["untagged", "tagged"]) def test_a_clean_large_batch_is_the_plain_json_text(tagged): rows = _batch(20_000, {}) diff --git a/tests/test_graph_batch_single_pass.py b/tests/test_graph_batch_single_pass.py index 2415c012..3b21bfdd 100644 --- a/tests/test_graph_batch_single_pass.py +++ b/tests/test_graph_batch_single_pass.py @@ -82,3 +82,40 @@ def spy(self, chunk): assert len(rids) == 250 stored = db.query("sql", "SELECT x FROM V WHERE id = 150").first().get("x") assert stored == 5e-324 and type(stored) is float + + +def test_the_stored_type_does_not_depend_on_which_chunk_held_a_bad_value( + temp_db_path, monkeypatch +): + """An undeclared int property is stored as the same type in the chunks that crossed as JSON text and in the chunk that took the + property-matrix path because a NaN sat in another property of one of its rows (the 100,000-row chunking of the real class). + """ + n = 250_000 + rows = [{"id": i, "k": i % 1000} for i in range(n)] + rows[150_000] = {"id": 150_000, "k": 1, "x": float("nan")} # chunk 2 of 3 only + probes = {5: "chunk 1", 150_001: "chunk 2", 249_999: "chunk 3"} + calls = [] + real = GraphBatch._to_java_property_matrix + + def spy(self, chunk): + calls.append(len(chunk)) + return real(self, chunk) + + monkeypatch.setattr(GraphBatch, "_to_java_property_matrix", spy) + with arcadedb.create_database(temp_db_path) as db: + db.command("sql", "CREATE VERTEX TYPE V") + with db.graph_batch(parallel_flush=False) as batch: + assert len(batch.create_vertices("V", rows)) == n + assert calls == [100_000], "only the chunk with the NaN takes the matrix path" + ids = ", ".join(str(i) for i in probes) + query = ( + f"SELECT id, k, k.javaType() AS t FROM V WHERE id IN [{ids}]" # nosec B608 + ) + got = db.query("sql", query).to_list() + by_id = {r["id"]: (r["k"], str(r["t"])) for r in got} + assert set(by_id) == set(probes) + assert {i: by_id[i][0] for i in probes} == {i: i % 1000 for i in probes} + types = {probes[i]: by_id[i][1] for i in probes} + assert ( + len(set(types.values())) == 1 + ), f"the stored type depends on the chunk: {types}" From d606ccee42bacd57f0bde43fbc399879a023158c Mon Sep 17 00:00:00 2001 From: Taewoon Kim Date: Mon, 9 Nov 2026 21:31:08 +0900 Subject: [PATCH 21/49] Native frames, second round: no replay of the first reply, types that belong to the whole result, bounded frames The first reply of a query stays what the caller already holds as dicts and is written as one more frame by Python (colframes.py) by the rules of the library's encoder, so the cursor keeps no rows (the replay kept up to 256 live Result objects per open cursor and read them after the snapshot the JSON reply had taken). The JSON reply names the properties that were float[] or double[] (fv, dv): a vector column is a declared array, not a list that happens to fit float32, at any result size. columns.py, with its size-dependent guess, is gone. Frames: the row limit is clamped to a C int (a batch_size of 2**31 or more wrapped to zero and looped), a frame ends by the larger of the values read and rows x columns x 9 bytes (a schemaless result with a property per row allocated gigabytes against a 1 MiB estimate), an empty frame from an open cursor is an error, a NULL reply and a library older than the wheel are errors with a message. Types: float32 next to float64 or another dimension unify to a list of float (never text), a frame of only nulls takes the type of the others, an integer beyond 2**53 next to a float stays exact in a json column on both wheels (ColumnBatcher too), an empty result with columns= returns those columns on both wheels, to_columns arrays are writable, to_dataframe vectors are numpy float32 cells and datetimes datetime64[us] on both wheels (documented). Co-Authored-By: Claude Sonnet 5.5 --- .../org/humemai/nativelib/ArcadeNative.java | 55 ++-- .../org/humemai/nativelib/ColumnFrames.java | 55 ++-- .../org/humemai/nativelib/NativeCodec.java | 47 ++- src/arcadedb/_columnar.py | 82 +++++- src/arcadedb/_native/colframes.py | 269 ++++++++++++------ src/arcadedb/_native/columns.py | 167 ----------- src/arcadedb/_native/database.py | 77 +++-- src/arcadedb/_native/lib.py | 18 +- src/arcadedb/results.py | 49 ++-- .../com/arcadedb/python/ColumnBatcher.java | 15 +- tests/test_column_frames.py | 190 +++++++++++++ tests/test_native_column_frames_reference.py | 159 ++++++++--- 12 files changed, 751 insertions(+), 432 deletions(-) delete mode 100644 src/arcadedb/_native/columns.py diff --git a/native/src/main/java/org/humemai/nativelib/ArcadeNative.java b/native/src/main/java/org/humemai/nativelib/ArcadeNative.java index 836e40a7..13ce33a2 100644 --- a/native/src/main/java/org/humemai/nativelib/ArcadeNative.java +++ b/native/src/main/java/org/humemai/nativelib/ArcadeNative.java @@ -20,7 +20,6 @@ import com.arcadedb.engine.ComponentFile; import com.arcadedb.exception.DuplicatedKeyException; import com.arcadedb.exception.NeedRetryException; -import com.arcadedb.query.sql.executor.Result; import com.arcadedb.query.sql.executor.ResultSet; import com.arcadedb.schema.VertexType; import com.arcadedb.server.ArcadeDBServer; @@ -37,7 +36,6 @@ import java.nio.ByteBuffer; import java.nio.ByteOrder; import java.nio.charset.StandardCharsets; -import java.util.ArrayList; import java.util.List; import java.util.Map; import java.util.concurrent.ConcurrentHashMap; @@ -47,17 +45,6 @@ public final class ArcadeNative { static final Map DBS = new ConcurrentHashMap<>(); private static final Map RESULT_SETS = new ConcurrentHashMap<>(); - /** - * The rows of the first reply of an open cursor, kept until the cursor is read again or closed: that reply is JSON for the row APIs, - * and a columnar read (arcadedb_n_next_col) that starts from it replays these rows into its first frame instead of losing them. - * At most the first reply (256 rows) of a cursor that has more rows. - */ - private static final Map> HEADS = new ConcurrentHashMap<>(); - - private static ResultSet removeCursor(final long id) { - HEADS.remove(id); - return RESULT_SETS.remove(id); - } private static final Map SERVERS = new ConcurrentHashMap<>(); static final AtomicLong IDS = new AtomicLong(1); @@ -301,10 +288,9 @@ public static CCharPointer next(final IsolateThread thread, final long rsId, fin rs = RESULT_SETS.get(rsId); if (rs == null) throw new IllegalArgumentException("unknown result set " + rsId); - HEADS.remove(rsId); - return out(batchOf(rsId, rs, max, null)); + return out(batchOf(rsId, rs, max)); } catch (final Throwable t) { - removeCursor(rsId); + RESULT_SETS.remove(rsId); closeQuietly(rs); return err(t); } @@ -323,10 +309,9 @@ public static CCharPointer nextBin(final IsolateThread thread, final long rsId, if (rs == null) throw new IllegalArgumentException("unknown result set " + rsId); final ResultSet cursor = rs; - HEADS.remove(rsId); - return withFloatTail(() -> batchOf(rsId, cursor, max, null)); + return withFloatTail(() -> batchOf(rsId, cursor, max)); } catch (final Throwable t) { - removeCursor(rsId); + RESULT_SETS.remove(rsId); closeQuietly(rs); return err(t); } @@ -336,19 +321,18 @@ public static CCharPointer nextBin(final IsolateThread thread, final long rsId, * The next rows of an open cursor as one column frame (see ColumnFrames), written as an int64 LE byte count and the frame; a failure is * the negated byte count of the error JSON and that JSON. max: rows at most (a frame also ends at about 1 MiB of values). columns: a * JSON array of the column names to read, or "" for the union of the names of the rows. flags bit 0: datetimes as microseconds (the - * to_dataframe form). skip: of the rows that the first reply of this cursor already held (HEADS), the number the caller has taken; the - * first frame starts after them, and a frame that ends before those rows do leaves the rest in HEADS for the next call (skip 0). The frame that ends the rows closes the cursor. The reply is freed with arcadedb_free. + * to_dataframe form). The rows the cursor already gave out in a JSON reply are not in the frame. The frame that ends the rows closes the + * cursor. The reply is freed with arcadedb_free. */ @CEntryPoint(name = "arcadedb_n_next_col") @SuppressWarnings("unchecked") public static CCharPointer nextCol(final IsolateThread thread, final long rsId, final int max, final CCharPointer columns, - final int flags, final int skip) { + final int flags) { ResultSet rs = null; try { rs = RESULT_SETS.get(rsId); if (rs == null) throw new IllegalArgumentException("unknown result set " + rsId); - final List head = HEADS.remove(rsId); String[] pinned = null; final String spec = str(columns); if (spec != null && !spec.isEmpty()) { @@ -357,17 +341,14 @@ public static CCharPointer nextCol(final IsolateThread thread, final long rsId, for (int i = 0; i < pinned.length; i++) pinned[i] = (String) names.get(i); } - final ColumnFrames.Frame frame = ColumnFrames.next(rs, head, skip, max, pinned, (flags & 1) != 0); + final ColumnFrames.Frame frame = ColumnFrames.next(rs, max, pinned, (flags & 1) != 0); if (frame.done) { - removeCursor(rsId); + RESULT_SETS.remove(rsId); closeQuietly(rs); - } else if (head != null && frame.headUsed < head.size()) { - // the frame ended (batch_size rows or about 1 MiB) before the replayed rows did: the next call starts with the rest - HEADS.put(rsId, new ArrayList<>(head.subList(frame.headUsed, head.size()))); } return frameOut(frame.bytes.length, frame.bytes); } catch (final Throwable t) { - removeCursor(rsId); + RESULT_SETS.remove(rsId); closeQuietly(rs); final byte[] e = errorJson(t).getBytes(StandardCharsets.UTF_8); return frameOut(-e.length, e); @@ -384,7 +365,7 @@ private static CCharPointer frameOut(final int length, final byte[] b) { @CEntryPoint(name = "arcadedb_n_rs_close") public static CCharPointer rsClose(final IsolateThread thread, final long rsId) { - closeQuietly(removeCursor(rsId)); + closeQuietly(RESULT_SETS.remove(rsId)); return OK(); } @@ -393,20 +374,16 @@ private static String batch(final ResultSet rs, final int max) { return "{\"ok\":true,\"rs\":0,\"rows\":[],\"meta\":[],\"drained\":true}"; final long id = IDS.getAndIncrement(); RESULT_SETS.put(id, rs); - final List keep = new ArrayList<>(); - final String reply = batchOf(id, rs, max, keep); - if (RESULT_SETS.containsKey(id)) - HEADS.put(id, keep); - return reply; + return batchOf(id, rs, max); } - private static String batchOf(final long id, final ResultSet rs, final int max, final List keep) { + private static String batchOf(final long id, final ResultSet rs, final int max) { final StringBuilder sb = new StringBuilder("{\"ok\":true,\"rs\":"); final int mark = sb.length(); sb.append(id).append(','); - final boolean drained = NativeCodec.appendBatch(sb, rs, max, keep); + final boolean drained = NativeCodec.appendBatch(sb, rs, max); if (drained) { - removeCursor(id); + RESULT_SETS.remove(id); closeQuietly(rs); sb.replace(mark, mark + String.valueOf(id).length(), "0"); } @@ -719,7 +696,7 @@ public static CCharPointer lastError(final IsolateThread thread) { public static void rsCloseMany(final IsolateThread thread, final CLongPointer ids, final int n) { try { for (int i = 0; i < n; i++) - closeQuietly(removeCursor(ids.read(i))); + closeQuietly(RESULT_SETS.remove(ids.read(i))); } catch (final Throwable ignored) { // closing is best effort, as in arcadedb_n_rs_close } diff --git a/native/src/main/java/org/humemai/nativelib/ColumnFrames.java b/native/src/main/java/org/humemai/nativelib/ColumnFrames.java index a45aad38..02d9106c 100644 --- a/native/src/main/java/org/humemai/nativelib/ColumnFrames.java +++ b/native/src/main/java/org/humemai/nativelib/ColumnFrames.java @@ -57,13 +57,10 @@ private ColumnFrames() { static final class Frame { final byte[] bytes; final boolean done; - /** The index in {@code head} of the first row this frame did not take (a frame may end before the head does). */ - final int headUsed; - Frame(final byte[] bytes, final boolean done, final int headUsed) { + Frame(final byte[] bytes, final boolean done) { this.bytes = bytes; this.done = done; - this.headUsed = headUsed; } } @@ -126,38 +123,33 @@ else if (v instanceof Map m) } /** - * The next frame. {@code head} holds rows that were already read from the cursor when its first reply was encoded as JSON (the - * frame starts at {@code head[skip]}, then goes on with the cursor), or is null. {@code pinned} is the column set, or null to take - * the union of the names of the rows of this frame, in order of first appearance. With {@code micros} a naive datetime column is - * epoch microseconds ("dtu") and a column with a DATE, a time-zone-aware temporal or a year Python cannot hold is "json". + * The next frame of the cursor. {@code pinned} is the column set, or null to take the union of the names of the rows of this frame, in + * order of first appearance. With {@code micros} a naive datetime column is epoch microseconds ("dtu") and a column with a DATE, a + * time-zone-aware temporal or a year Python cannot hold is "json". + * + * The frame ends at {@code max} rows or when what it will hold reaches FRAME_BYTES: the larger of the sizes of the values read and of the + * fixed-width buffers (rows times columns times 9 bytes, which a schemaless result with a different property in every row makes much + * bigger than its values), so the allocation of a frame stays near its budget whatever the shape of the rows. A frame has one row at least. */ - static Frame next(final ResultSet rs, final List head, final int skip, final int max, final String[] pinned, - final boolean micros) { - final List results = new ArrayList<>(Math.min(Math.max(max, 1), 1 << 14)); - int h = Math.max(skip, 0); + static Frame next(final ResultSet rs, final int max, final String[] pinned, final boolean micros) { + final int cap = Math.max(max, 1); + final List results = new ArrayList<>(Math.min(cap, 1 << 14)); + final Set names = new LinkedHashSet<>(); + String[] previous = null; long budget = 0; boolean done = false; - while (results.size() < max && budget < FRAME_BYTES) { - final Result row; - if (head != null && h < head.size()) - row = head.get(h++); - else if (rs.hasNext()) - row = rs.next(); - else { + while (results.size() < cap + && Math.max(budget, (long) results.size() * (pinned != null ? pinned.length : names.size()) * 9L) < FRAME_BYTES) { + if (!rs.hasNext()) { done = true; break; } + final Result row = rs.next(); results.add(row); budget += estimate(row); - } - if (!done) - done = (head == null || h >= head.size()) && !rs.hasNext(); - - String[] columns = pinned; - if (columns == null) { - final Set names = new LinkedHashSet<>(); - String[] previous = null; - for (final Result row : results) { + if (pinned == null) { + // rows of one type almost always list the same names in the same order: a row is compared with the one before it name by name + // and only a row that differs is hashed into the set final Set rowNames = row.getPropertyNames(); if (previous != null && rowNames.size() == previous.length) { int i = 0; @@ -172,8 +164,11 @@ else if (rs.hasNext()) names.addAll(rowNames); previous = rowNames.toArray(new String[0]); } - columns = names.toArray(new String[0]); } + if (!done) + done = !rs.hasNext(); + + final String[] columns = pinned != null ? pinned : names.toArray(new String[0]); final int n = columns.length; final int count = results.size(); final List rows = new ArrayList<>(count); @@ -380,6 +375,6 @@ else if (type.equals("dtu")) out.putInt(headerBytes.length); out.put(headerBytes); out.put(bodyBytes); - return new Frame(out.array(), done, h); + return new Frame(out.array(), done); } } diff --git a/native/src/main/java/org/humemai/nativelib/NativeCodec.java b/native/src/main/java/org/humemai/nativelib/NativeCodec.java index 5d037a50..b3bbab06 100644 --- a/native/src/main/java/org/humemai/nativelib/NativeCodec.java +++ b/native/src/main/java/org/humemai/nativelib/NativeCodec.java @@ -51,13 +51,38 @@ private NativeCodec() { * Appends {@code "rows":[..],"meta":[..],"drained":b} (no braces) with up to max rows. meta has one entry per row: 0 * for a projection, or [rid, typeName, kind] for a record row (kind v, e or d). */ - static boolean appendBatch(final StringBuilder sb, final ResultSet rs, final int max) { - return appendBatch(sb, rs, max, null); + /** + * The names of the properties whose value was a float[] (f) or a double[] (d) in a row of a batch. A JSON reply writes both as a list of + * numbers, so the reader cannot tell a vector from a list of doubles: the reply names the vectors, {@code "fv":[..]} and {@code "dv":[..]} + * after "drained" (absent when there is none), and a columnar read types those columns as vectors, as a column frame does. + */ + static final class VecHints { + final java.util.LinkedHashSet f = new java.util.LinkedHashSet<>(); + final java.util.LinkedHashSet d = new java.util.LinkedHashSet<>(); + + void appendTo(final StringBuilder sb) { + appendNames(sb, ",\"fv\":", f); + appendNames(sb, ",\"dv\":", d); + } + + private static void appendNames(final StringBuilder sb, final String key, final java.util.Set names) { + if (names.isEmpty()) + return; + sb.append(key).append('['); + boolean first = true; + for (final String name : names) { + if (!first) + sb.append(','); + first = false; + quote(sb, name); + } + sb.append(']'); + } } - /** As above; {@code keep}, when not null, receives each row that was encoded (a columnar read replays them, see ArcadeNative.HEADS). */ - static boolean appendBatch(final StringBuilder sb, final ResultSet rs, final int max, final java.util.List keep) { + static boolean appendBatch(final StringBuilder sb, final ResultSet rs, final int max) { final StringBuilder meta = new StringBuilder("["); + final VecHints hints = new VecHints(); sb.append("\"rows\":["); int n = 0; boolean drained = false; @@ -76,15 +101,13 @@ static boolean appendBatch(final StringBuilder sb, final ResultSet rs, final int sb.append(','); meta.append(','); } - appendRow(sb, meta, row); + appendRow(sb, meta, row, hints); } catch (final Throwable t) { sb.setLength(rowStart); meta.setLength(metaStart); throw t; } n++; - if (keep != null) - keep.add(row); if (sb.length() >= MAX_BATCH_CHARS) break; } @@ -97,10 +120,11 @@ static boolean appendBatch(final StringBuilder sb, final ResultSet rs, final int sb.append("],\"meta\":").append(meta).append("],\"drained\":").append(drained); if (readError != null) sb.append(",\"rerr\":").append(readError); + hints.appendTo(sb); return drained; } - private static void appendRow(final StringBuilder sb, final StringBuilder meta, final Result row) { + private static void appendRow(final StringBuilder sb, final StringBuilder meta, final Result row, final VecHints hints) { sb.append('{'); boolean first = true; for (final String name : row.getPropertyNames()) { @@ -109,7 +133,12 @@ private static void appendRow(final StringBuilder sb, final StringBuilder meta, first = false; quote(sb, name); sb.append(':'); - append(sb, row.getProperty(name)); + final Object value = row.getProperty(name); + if (value instanceof float[]) + hints.f.add(name); + else if (value instanceof double[]) + hints.d.add(name); + append(sb, value); } sb.append('}'); if (row.isElement()) { diff --git a/src/arcadedb/_columnar.py b/src/arcadedb/_columnar.py index 7cb62e36..16c66d81 100644 --- a/src/arcadedb/_columnar.py +++ b/src/arcadedb/_columnar.py @@ -165,6 +165,25 @@ def _unify_arrow_chunk_types(arrs: list, pa) -> list: except pa.ArrowException: pass + if all(pa.types.is_list(t) or pa.types.is_fixed_size_list(t) for t in types): + # vectors and lists of numbers: a dimension that changes between frames makes a list, float32 next to float64 is float64 + # (the widening is exact), never text + values = {t.value_type for t in types} + if values <= {pa.float32(), pa.float64()}: + value = pa.float64() if pa.float64() in values else pa.float32() + sizes = { + t.list_size if pa.types.is_fixed_size_list(t) else None for t in types + } + target = ( + pa.list_(value, next(iter(sizes))) + if len(sizes) == 1 and None not in sizes + else pa.list_(value) + ) + try: + return [pa.array(a.to_pylist(), type=target) for a in arrs] + except pa.ArrowException: + pass + if all(pa.types.is_decimal(t) for t in types): common = _common_decimal_type(list(types), pa) if common is not None: @@ -173,6 +192,31 @@ def _unify_arrow_chunk_types(arrs: list, pa) -> list: return _cast_all_to_string(arrs, pa) +MAX_FRAME_ROWS = 2**31 - 1 # a frame's row limit crosses to the library as a C int + + +def frame_rows(batch_size) -> int: + """The row limit of a frame for ``batch_size``: at least 1 (a ValueError otherwise), at most what a C int holds, so a huge + ``batch_size`` means "as many as one frame takes" instead of wrapping around to zero or a negative number. + """ + rows = int(batch_size) + if rows < 1: + raise ValueError(f"batch_size must be at least 1, got {batch_size}") + return min(rows, MAX_FRAME_ROWS) + + +def empty_columns(columns) -> Dict[str, Any]: + """The ``to_columns`` dict of a result with no rows: no column, or the ones the caller asked for, empty.""" + return {} if columns is None else {name: [] for name in columns} + + +def empty_arrow(columns): + """The ``to_arrow`` table of a result with no rows: no column, or the ones the caller asked for, empty and of the null type.""" + import pyarrow as pa + + return pa.table({} if columns is None else {name: pa.nulls(0) for name in columns}) + + def _null_mask(buf, pos: int, nulls_len: int, count: int): """``(null_bits, mask)`` of a column: ``mask`` is a bool array (True = null) or None when no row is null.""" import numpy as np @@ -213,14 +257,15 @@ def decode_columns_frame( pos += col["bytes"] if ctype == "i8": - arr = np.frombuffer(data, dtype=" bool: + return ( + isinstance(part, list) and len(part) == count and all(v is None for v in part) + ) + + def merge_columns( - names: List[str], batches: List[Tuple[int, Dict[str, Any]]] + names: List[str], batches: List[Tuple[int, Dict[str, Any]]], columns=None ) -> Dict[str, Any]: - """The ``to_columns`` dict of the parts of every frame: a column a frame lacks gets nulls (NaN, NaT or None) there.""" + """The ``to_columns`` dict of the parts of every frame. The type of a column belongs to the whole result, not to where the frames + end: a frame that lacks the column, or holds only nulls for it, takes the type of the frames that have values (NaN, NaT or None + in its rows). With no rows at all the result is :func:`empty_columns`.""" import numpy as np + if not batches or not names: + return empty_columns(columns) out: Dict[str, Any] = {} for name in names: - template = next(p[name] for _, p in batches if name in p) + typed = [ + p[name] for c, p in batches if name in p and not _is_all_none(p[name], c) + ] + template = typed[0] if typed else next(p[name] for _, p in batches if name in p) parts = [ - p[name] if name in p else _missing_column_part(c, template) + ( + p[name] + if name in p and not (typed and _is_all_none(p[name], c)) + else _missing_column_part(c, template) + ) for c, p in batches ] np_parts = [p for p in parts if not isinstance(p, list)] @@ -374,13 +436,15 @@ def decode_arrow_frame( return count, chunks, header -def merge_arrow(names: List[str], batches: List[Tuple[int, Dict[str, Any]]]): +def merge_arrow( + names: List[str], batches: List[Tuple[int, Dict[str, Any]]], columns=None +): """The ``pyarrow.Table`` of the chunks of every frame: one chunked column per name, one type per column (see :func:`_unify_arrow_chunk_types`), a frame that lacks a column adds nulls.""" import pyarrow as pa if not batches or not names: - return pa.table({}) + return empty_arrow(columns) return pa.table( { n: pa.chunked_array( diff --git a/src/arcadedb/_native/colframes.py b/src/arcadedb/_native/colframes.py index e15dcb2f..31e47cc0 100644 --- a/src/arcadedb/_native/colframes.py +++ b/src/arcadedb/_native/colframes.py @@ -1,29 +1,186 @@ """Column frames on the native build: the glue between ``arcadedb_n_next_col`` and the decoder both wheels share (``arcadedb._columnar``). -A result of the native build reaches Python in two forms. The rows after the first reply are column frames, read by the shared decoder. -The first reply of a query (up to ``_FIRST_BATCH`` rows) is JSON text that the row APIs read, and it is the only part of a result that a -columnar read holds as dicts: the rows that are still buffered when the columnar read starts go through the rules of :mod:`.columns` -and are merged with the frames as one more chunk. When the cursor still holds those rows (``arcadedb_n_next_col`` replays them), they -are not taken from the dicts at all; a result that the first reply held whole has no cursor, so its rows are the only form it has. -That chunk goes away with the JSON reply. - -Two things in a chunk of dicts cannot be known from the values: whether a list of floats is a ``float[]`` (a frame says so with its -column type) and nothing else. A column of lists of floats that float32 holds exactly is a vector column in the chunk when the frames -say so, or when there are no frames; when the frames carry it as a list column, the chunk keeps the list. +A result of the native build reaches Python in two forms. The rows after the first reply are column frames, which the library writes and +the shared decoder reads. The first reply of a query (up to ``_FIRST_BATCH`` rows) is JSON text that the row APIs read, and a columnar read +finds those rows (and any the caller read by row) as dicts in the result's buffer. They cannot be read again from the cursor, so they are +written here as one more frame, in the layout the library writes, by the rules the library's encoder follows, and read by the same +decoder: a column has one type however a result splits into the first reply and frames. The JSON reply names the properties that were +``float[]`` or ``double[]`` (``fv`` and ``dv``): a list of numbers in a row cannot say that by itself, and a vector column is a column of +declared arrays, not a column of lists that happen to fit float32. This frame goes away with the JSON reply. """ -from datetime import datetime +import json +import struct +from datetime import date, datetime, timedelta, timezone +from decimal import Decimal from .. import _columnar -from . import columns as _rows from .codec import loads_tagged +_EPOCH = datetime(1970, 1, 1) +_MS = timedelta(milliseconds=1) +_US = timedelta(microseconds=1) +_INT64 = (-(1 << 63), (1 << 63) - 1) +_TWO53 = 1 << 53 + def json_values(data, side, count): - """The values of a ``json`` column of a frame: the native library's tagged JSON, restored as a row's values are.""" + """The values of a ``json`` column of a frame from the library: its tagged JSON, restored as a row's values are.""" return loads_tagged(bytes(data).decode("utf-8"))[0] +def _head_json_values(data, side, count): + """The values of a ``json`` column of the head frame: the buffered values themselves, by the index the column carries.""" + return side[int.from_bytes(bytes(data), "little")] + + +def _naive_utc(v): + if isinstance(v, datetime): + if v.tzinfo is not None: + v = v.astimezone(timezone.utc).replace(tzinfo=None) + return v + return datetime(v.year, v.month, v.day) + + +def _kind(values, micros): + """The column type of a frame that the values would get from the library's encoder: ``i8``, ``f8``, ``b1``, ``dt``, ``dtu``, + ``str``, ``dec`` or ``json``; ``null`` for a column of nothing but nulls.""" + kinds = set() + big_int = False + for v in values: + if v is None: + continue + t = type(v) + if t is bool: + k = "b1" + elif t is int: + if not _INT64[0] <= v <= _INT64[1]: + k = "json" # a BigInteger + else: + k = "i8" + if not -_TWO53 <= v <= _TWO53: + big_int = True + elif t is float: + k = "f8" + elif t is datetime or t is date: + k = "dt" + elif t is str: + k = "str" + elif t is Decimal: + k = "dec" + else: + k = "json" + kinds.add(k) + if k == "json": + break + if not kinds: + return "null" + if kinds == {"i8", "f8"}: + # an integer beyond 2**53 next to a floating value has no exact double: the values stay what they are + return "json" if big_int else "f8" + if len(kinds) > 1: + return "json" + kind = kinds.pop() + if kind == "dt": + if not micros: + return "dt" + if all(v is None or (type(v) is datetime and v.tzinfo is None) for v in values): + return "dtu" + return "json" + return kind + + +def _vector_dim(values): + """The one length of the lists of a column of float lists, or 0 when it is not one (a non-list, an empty or a ragged list).""" + dim = 0 + for v in values: + if v is None: + continue + if type(v) is not list or not v or any(type(x) is not float for x in v): + return 0 + if dim == 0: + dim = len(v) + elif len(v) != dim: + return 0 + return dim + + +def _bitmap(values): + out = bytearray((len(values) + 7) // 8) + for i, v in enumerate(values): + if v is None: + out[i >> 3] |= 1 << (i & 7) + return bytes(out) + + +def _column(values, vector, micros, side): + """``(type, data, dim)`` of one column: the buffer the library's encoder writes for it. ``vector`` is ``"f"`` or ``"d"`` for a + property that was a ``float[]`` or a ``double[]``, else None.""" + import numpy as np + + kind = _kind(values, micros) + nan = float("nan") + if kind == "i8": + return ( + kind, + np.array([0 if v is None else v for v in values], dtype="`` array of a column of float lists that float32 holds, or None.""" - import pyarrow as pa - - vec = _rows._float_vectors(values) - if vec is None: - return None - arr = pa.FixedSizeListArray.from_arrays(pa.array(vec.reshape(-1)), vec.shape[1]) - if any(v is None for v in values): - arr = pa.array(arr.to_pylist(), type=arr.type, mask=[v is None for v in values]) - return arr - - -def arrow_result(head_rows, columns, batches): +def arrow_result(head_rows, columns, batches, hints): """The ``pyarrow.Table`` of the buffered rows (dicts) followed by the decoded frames ``batches`` (a list of ``(count, chunks)``).""" - import pyarrow as pa - names = [] chunks_list = [] if head_rows: - table = _rows.to_arrow(head_rows, columns) - chunks = {} - for name in table.column_names: - column = table.column(name) - arr = ( - column.chunk(0) - if column.num_chunks == 1 - else pa.concat_arrays(column.chunks) - ) - if pa.types.is_null(arr.type): - # a frame types an all-null column as text - arr = pa.nulls(len(arr), pa.string()) - elif pa.types.is_list(arr.type) and arr.type.value_type == pa.float64(): - other = _first_part(batches, name) - if other is None or pa.types.is_fixed_size_list(other.type): - vector = _vector_arrow([row.get(name) for row in head_rows]) - if vector is not None: - arr = vector - chunks[name] = arr + frame, side = _head_frame(head_rows, columns, hints, False) + count, chunks, _ = _columnar.decode_arrow_frame(frame, side, _head_json_values) names.extend(chunks) - chunks_list.append((len(head_rows), chunks)) + chunks_list.append((count, chunks)) for count, chunks in batches: for name in chunks: if name not in names: names.append(name) chunks_list.append((count, chunks)) - return _columnar.merge_arrow(names, chunks_list) + return _columnar.merge_arrow(names, chunks_list, columns) diff --git a/src/arcadedb/_native/columns.py b/src/arcadedb/_native/columns.py deleted file mode 100644 index 90f59df6..00000000 --- a/src/arcadedb/_native/columns.py +++ /dev/null @@ -1,167 +0,0 @@ -"""Rows to columns for the native backend (the columnar path is Python over the rows; the JPype path decodes a Java buffer).""" - -from datetime import date, datetime, timezone -from decimal import Decimal - - -def _names(rows, columns): - if columns is not None: - return list(columns) - seen = {} - for row in rows: - for k in row: - seen.setdefault(k, None) - return list(seen) - - -def _kind(values): - """'b', 'i', 'f', 'dt', 'dec', 'obj' for the non-null values of a column.""" - kinds = set() - for v in values: - if v is None: - continue - t = type(v) - if t is bool: - kinds.add("b") - elif t is int: - kinds.add("i") - elif t is float: - kinds.add("f") - elif t is datetime or t is date: - kinds.add("dt") - elif t is Decimal: - kinds.add("dec") - else: - kinds.add("obj") - if not kinds: - return "obj" - if len(kinds) == 1: - return kinds.pop() - if kinds == {"i", "f"}: - return "f" - return "obj" - - -def _ms(v): - if isinstance(v, datetime): - if v.tzinfo is not None: - v = v.astimezone(timezone.utc).replace(tzinfo=None) - return v - return datetime(v.year, v.month, v.day) - - -def _float_vectors(values): - """A column of equal-length float lists (None allowed) that float32 holds exactly, as a (count, dim) float32 array with - NaN rows for the nulls; None for any other column. Rows cannot say float[] from List, so only a column whose every - value survives float32 unchanged is read as a vector column (the JPype path decides on the Java element type). - """ - import numpy as np - - dim = 0 - for v in values: - if v is None: - continue - if type(v) is not list or not v or not all(type(x) is float for x in v): - return None - if dim == 0: - dim = len(v) - elif len(v) != dim: - return None - if dim == 0: - return None - arr = np.full((len(values), dim), np.nan, dtype=np.float32) - for i, v in enumerate(values): - if v is not None: - arr[i] = v - if not np.array_equal(arr[i].astype(np.float64), np.array(v)): - return None - return arr - - -def to_columns(rows, columns=None): - import numpy as np - - out = {} - for name in _names(rows, columns): - values = [row.get(name) for row in rows] - if _kind(values) == "obj": - vectors = _float_vectors(values) - if vectors is not None: - out[name] = vectors - continue - has_null = any(v is None for v in values) - kind = _kind(values) - if kind == "i": - out[name] = ( - np.array([np.nan if v is None else v for v in values], dtype=np.float64) - if has_null - else np.array(values, dtype=np.int64) - ) - elif kind == "f": - out[name] = np.array( - [np.nan if v is None else v for v in values], dtype=np.float64 - ) - elif kind == "b": - out[name] = values if has_null else np.array(values, dtype=bool) - elif kind == "dt": - out[name] = np.array( - [ - ( - np.datetime64("NaT", "ms") - if v is None - else np.datetime64(_ms(v), "ms") - ) - for v in values - ], - dtype="datetime64[ms]", - ) - elif kind == "dec": - arr = np.empty(len(values), dtype=object) - arr[:] = values - out[name] = arr - else: - out[name] = values - return out - - -def to_arrow(rows, columns=None): - import pyarrow as pa - - arrays = {} - for name in _names(rows, columns): - values = [row.get(name) for row in rows] - kind = _kind(values) - if kind == "i": - arrays[name] = pa.array(values, type=pa.int64()) - elif kind == "f": - try: - arrays[name] = pa.array(values, type=pa.float64()) - except ( - pa.ArrowInvalid - ): # an int beyond +-2**53 next to floats: the column degrades to exact text - arrays[name] = pa.array( - [None if v is None else str(v) for v in values], type=pa.string() - ) - elif kind == "b": - arrays[name] = pa.array(values, type=pa.bool_()) - elif kind == "dt": - arrays[name] = pa.array( - [None if v is None else _ms(v) for v in values], type=pa.timestamp("ms") - ) - elif kind == "dec": - try: - arrays[name] = pa.array(values) - except ( - pa.ArrowInvalid - ): # wider than decimal256 (76 digits): an exact string - arrays[name] = pa.array( - [None if v is None else str(v) for v in values], type=pa.string() - ) - else: - try: - arrays[name] = pa.array([None if v is None else v for v in values]) - except (pa.ArrowInvalid, pa.ArrowTypeError): - arrays[name] = pa.array( - [None if v is None else str(v) for v in values], type=pa.string() - ) - return pa.table(arrays) diff --git a/src/arcadedb/_native/database.py b/src/arcadedb/_native/database.py index 71d178eb..792b82ce 100644 --- a/src/arcadedb/_native/database.py +++ b/src/arcadedb/_native/database.py @@ -9,6 +9,7 @@ from datetime import date, datetime from typing import Any, Dict, Iterator, List, Optional +from .. import _columnar from .._params import refuse_surplus_positionals from ..exceptions import ArcadeDBError from ..transactions import TransactionContext @@ -210,9 +211,10 @@ def __init__(self, database, reply: dict, tagged: bool = True): self._rs = reply["rs"] self._buf = deque(zip(reply["rows"], reply["meta"])) self._more = not reply["drained"] - # the rows of this reply that the cursor still holds, for a columnar read to replay (arcadedb_n_next_col): the first reply of - # a cursor with more rows. Any later read of the cursor by row (arcadedb_n_next) lets the library drop them - self._head_n = len(reply["rows"]) if self._more else 0 + # the properties that were float[] ("f") or double[] ("d") in the rows read so far: a JSON row cannot tell them from a list of + # doubles, and a columnar read types them as vector columns (colframes.py) + self._vectors: Dict[str, str] = {} + self._note_vectors(reply) self._closed = False self._exhausted = False # the engine computes lazily: an error met while the first rows were read is raised when the rows before it @@ -227,7 +229,7 @@ def _lazy(cls, database, rs_id: int) -> "NativeResultSet": self._rs = rs_id self._buf = deque() self._tagged = False - self._head_n = 0 + self._vectors = {} self._more = rs_id > 0 self._closed = False self._exhausted = False @@ -262,7 +264,6 @@ def _fill(self) -> bool: raise _read_error(err) if not self._more: return False - self._head_n = 0 try: if lib.NEXT_BIN and _BINARY_REPLY: reply, tagged = loads_tagged( @@ -287,8 +288,15 @@ def _fill(self) -> bool: self._buf.extend(zip(reply["rows"], reply["meta"])) self._more = not reply["drained"] self._pending_error = reply.get("rerr") + self._note_vectors(reply) return bool(self._buf) or self._pending_error is not None + def _note_vectors(self, reply: dict) -> None: + for name in reply.get("fv", ()): + self._vectors[name] = "f" + for name in reply.get("dv", ()): + self._vectors[name] = "d" + def _finish(self) -> None: self._exhausted = True self.close() @@ -415,31 +423,24 @@ def iter_json_batches(self, batch_size: int = 10_000, **kwargs): def _read_frames(self, batch_size, columns, decode, flags=0): """The rows still to be read, in the two forms a result has: ``(head_rows, batches)``. - ``head_rows`` are rows that this object holds as dicts (the first reply, when the cursor no longer has them); - ``batches`` is the list of ``(count, parts)`` that ``decode(frame_bytes)`` makes of each column frame the cursor - yields (about 1 MiB of values each, ``batch_size`` rows at most; ``flags`` 1 asks for datetimes as microseconds). The result - is finished when this returns. + ``head_rows`` are the rows this object holds as dicts (the first reply, and any reply the caller read by row): they are + not in the cursor any more. ``batches`` is the list of ``(count, parts)`` that ``decode(frame_bytes)`` makes of each column + frame the cursor yields (about 1 MiB of values each, ``batch_size`` rows at most; ``flags`` 1 asks for datetimes as + microseconds). The result is finished when this returns. """ - if int(batch_size) < 1: - raise ValueError("batch_size must be at least 1") + size = _columnar.frame_rows(batch_size) head_rows: list = [] batches: list = [] if not self._readable(): return head_rows, batches spec = json.dumps(list(columns)).encode("utf-8") if columns is not None else b"" - skip = 0 - if self._head_n and self._more and self._rs: - # the cursor still holds the rows of the first reply: it replays the ones this object has not handed out - skip = self._head_n - len(self._buf) - self._buf.clear() - else: - head_rows = [props for props, _ in self._buf] - self._buf.clear() - if self._pending_error is not None: - self._fill() # the rows before the error are consumed: raise it + head_rows = [props for props, _ in self._buf] + self._buf.clear() + if self._pending_error is not None: + self._fill() # the rows before the error are consumed: raise it while self._more and self._rs: try: - raw = lib.call_frame(self._rs, int(batch_size), spec, flags, skip) + raw = lib.call_frame(self._rs, size, spec, flags) except lib.ColumnFrameError as exc: # the library closed the cursor self._more = False @@ -447,20 +448,29 @@ def _read_frames(self, batch_size, columns, decode, flags=0): self._closed = True self._exhausted = True raise _read_error(exc.reply) from None - skip = 0 - self._head_n = 0 count, parts, header = decode(raw) if header.get("done"): self._more = False # the library closed the cursor with the frame that ended the rows self._rs = 0 + elif count == 0: + # an open cursor must give a row per frame: never loop on a library that does not + self._more = False + self.close() + raise ArcadeDBError( + "The native library returned an empty column frame for a cursor with rows left (frame row limit " + f"{size}). This is a bug in the library or in the wheel." + ) if count: batches.append((count, parts)) self._finish() return head_rows, batches def to_dataframe(self, convert_types: bool = True): - """The rows as a ``pandas.DataFrame``: the values ``to_list()`` returns (a DATETIME keeps its microseconds, a DATE is a - ``datetime.date`` in an object column, a ``Decimal`` stays one), read as column frames, not as one dict per row. + """The rows as a ``pandas.DataFrame``, read as column frames (no dict per row). The values are the ones ``to_list()`` returns, + with two exceptions that are the same on both builds. A DATETIME keeps its microseconds (a ``datetime64[us]`` column, where + ``to_columns()`` and ``to_arrow()`` carry milliseconds), a DATE is a ``datetime.date`` in an object column, and a ``Decimal`` + stays one. A column of ``float[]`` vectors holds one numpy ``float32`` array per cell (a row of the matrix ``to_columns()`` + returns, not a Python list per row); ``to_list()`` gives lists of floats. """ import pandas as pd @@ -480,12 +490,17 @@ def _columns(self, batch_size, columns, micros): lambda raw: colframes.decode_columns(raw, micros), 1 if micros else 0, ) - return colframes.columns_result(head_rows, columns, micros, batches) + return colframes.columns_result( + head_rows, columns, micros, batches, self._vectors + ) def to_columns(self, batch_size: int = 25_000, columns=None): - """Columns as the JPype path returns them (numpy int64/float64/bool/datetime64[ms] arrays, lists for text and - JSON-typed values, an object array of Decimal), decoded from column frames by the decoder both builds share. None when - numpy is missing.""" + """Columns as the JVM build returns them (numpy int64/float64/bool/datetime64[ms] arrays, lists for text and JSON-typed + values, an object array of Decimal), decoded from column frames by the decoder both builds share. The arrays are writable, + and a column's type belongs to the whole result, not to where a batch ends. A column is a float32 matrix when the property is + a ``float[]`` (ARRAY_OF_FLOATS), and a list of lists when it is a LIST of numbers whatever the values. An empty result is + ``{}``, or the ``columns`` asked for as empty lists. None when numpy is missing. + """ try: __import__("numpy") # availability check except ImportError: @@ -505,7 +520,7 @@ def to_arrow(self, batch_size: int = 25_000, columns=None): head_rows, batches = self._read_frames( batch_size, columns, colframes.decode_arrow ) - return colframes.arrow_result(head_rows, columns, batches) + return colframes.arrow_result(head_rows, columns, batches, self._vectors) def to_polars(self, batch_size: int = 25_000, columns=None): try: diff --git a/src/arcadedb/_native/lib.py b/src/arcadedb/_native/lib.py index dc3f1e98..778305b5 100644 --- a/src/arcadedb/_native/lib.py +++ b/src/arcadedb/_native/lib.py @@ -232,7 +232,7 @@ class _Params(ctypes.Structure): "arcadedb_n_insert_colp": [P, L, S, S, P, I], "arcadedb_n_next": [P, L, I], "arcadedb_n_next_bin": [P, L, I], - "arcadedb_n_next_col": [P, L, I, S, I, I], + "arcadedb_n_next_col": [P, L, I, S, I], "arcadedb_n_rs_close": [P, L], "arcadedb_n_cmd": [P, L, I, S, S, S, S, L, I], "arcadedb_n_last_error": [P], @@ -366,15 +366,25 @@ def call_bin(name: str, *args): return raw.decode("utf-8"), floats -def call_frame(rs_id: int, max_rows: int, spec: bytes, flags: int, skip: int) -> bytes: +def call_frame(rs_id: int, max_rows: int, spec: bytes, flags: int) -> bytes: """One column frame of an open cursor (``arcadedb_n_next_col``): its bytes, in the layout ``arcadedb._columnar`` reads. The reply is an int64 byte count and the frame. A negative count is the negated length of an error JSON: it is raised as - :class:`ColumnFrameError` with the decoded reply (the cursor is already closed on the Java side then). + :class:`ColumnFrameError` with the decoded reply (the cursor is already closed on the Java side then). ``max_rows`` is a C int: + callers clamp it (``arcadedb._columnar.frame_rows``). """ lib = _lib or _load() + if getattr(lib, "arcadedb_n_next_col", None) is None: + raise ArcadeDBError( + "The native library is older than this wheel: it has no arcadedb_n_next_col entry point, which to_arrow, to_columns, " + "to_dataframe and to_polars need. Reinstall the arcadedb wheel (a library and its Python files ship together)." + ) th = _thread_handle() - ptr = lib.arcadedb_n_next_col(th, rs_id, max_rows, spec, flags, skip) + ptr = lib.arcadedb_n_next_col(th, rs_id, max_rows, spec, flags) + if not ptr: + raise ArcadeDBError( + "The native library returned no reply to a column frame request" + ) try: n = int.from_bytes(ctypes.string_at(ptr, 8), "little", signed=True) if n < 0: diff --git a/src/arcadedb/results.py b/src/arcadedb/results.py index 945ab21e..2abc95dc 100644 --- a/src/arcadedb/results.py +++ b/src/arcadedb/results.py @@ -15,6 +15,9 @@ from ._columnar import ( decode_arrow_frame, decode_columns_frame, + empty_arrow, + empty_columns, + frame_rows, merge_arrow, merge_columns, ) @@ -546,10 +549,14 @@ def to_dataframe(self, convert_types: bool = True): Requires pandas to be installed. - Values are the ones ``to_list()`` returns: a DATETIME keeps its - microseconds (a ``datetime64[us]`` column, where ``to_columns()`` and - ``to_arrow()`` carry milliseconds), a DATE is a ``datetime.date`` in an - object column, and a ``Decimal`` stays one. + Values are the ones ``to_list()`` returns, with two exceptions that are + the same on both builds. A DATETIME keeps its microseconds (a + ``datetime64[us]`` column, where ``to_columns()`` and ``to_arrow()`` + carry milliseconds), a DATE is a ``datetime.date`` in an object column, + and a ``Decimal`` stays one. A column of ``float[]`` vectors holds one + numpy ``float32`` array per cell (a row of the matrix ``to_columns()`` + returns, no copy into a Python list per row); ``to_list()`` gives + lists of floats. Args: convert_types: Convert Java types to Python (default: True) @@ -599,7 +606,9 @@ def to_polars( Reads :meth:`to_arrow` and hands the table to Polars, so nulls keep their type (a nullable integer stays an integer column) and a DECIMAL - column is a Polars ``Decimal``. Requires polars and pyarrow + column is a Polars ``Decimal``. A column of ``float[]`` vectors is a + Polars ``Array(Float32, n)`` (``to_arrow()`` types it as a fixed-size + list of float32). Requires polars and pyarrow (``uv add 'arcadedb-jvm[polars]'``). Args: @@ -645,7 +654,11 @@ def to_columns( Null handling follows pandas conventions: int/datetime columns with nulls are promoted to float64 with NaN / datetime64 NaT; string and JSON columns use None. A DECIMAL column is an object array of exact - ``Decimal`` values (#115). + ``Decimal`` values (#115). The type of a column belongs to the whole + result, not to where a batch ends: a batch that lacks the column or + holds only nulls for it takes the type of the batches that have + values. The arrays are writable. An empty result is ``{}``, or, when + ``columns`` is given, those columns as empty lists. The columns are the union of every row's property names, in order of first appearance, because a document is schemaless: a property the @@ -690,13 +703,13 @@ def _read_columns(self, batch_size, columns, micros): # ";".join: a name may legally contain a semicolon, a quote or a # backslash. spec = json.dumps(list(columns)) if columns is not None else "" + size = frame_rows(batch_size) if not self._readable(): - return {} - total = 0 + return empty_columns(columns) while True: try: java_batch = column_batcher.nextColumnBatch( - self._java_result_set, int(batch_size), spec, micros + self._java_result_set, size, spec, micros ) except JException as exc: raise _read_error(exc) from exc @@ -711,11 +724,8 @@ def _read_columns(self, batch_size, columns, micros): if name not in names: names.append(name) batches.append((count, parts)) - total += count - if total == 0: - return {} - return merge_columns(names, batches) + return merge_columns(names, batches, columns) def to_arrow( self, batch_size: int = 25_000, columns: Optional[Sequence[str]] = None @@ -748,7 +758,7 @@ def to_arrow( """ try: __import__("numpy") # availability check - import pyarrow as pa + __import__("pyarrow") # availability check except ImportError: return None @@ -763,13 +773,13 @@ def to_arrow( batches: List[Tuple[int, Dict[str, Any]]] = [] spec = json.dumps(list(columns)) if columns is not None else "" + size = frame_rows(batch_size) if not self._readable(): - return pa.table({}) - total = 0 + return empty_arrow(columns) while True: try: java_batch = column_batcher.nextColumnBatch( - self._java_result_set, int(batch_size), spec, False + self._java_result_set, size, spec, False ) except JException as exc: raise _read_error(exc) from exc @@ -784,11 +794,8 @@ def to_arrow( if name not in names: names.append(name) batches.append((count, chunks)) - total += count - if total == 0 or not names: - return pa.table({}) - return merge_arrow(names, batches) + return merge_arrow(names, batches, columns) def iter_chunks( self, size: int = 1000, convert_types: bool = True diff --git a/src/java/com/arcadedb/python/ColumnBatcher.java b/src/java/com/arcadedb/python/ColumnBatcher.java index cf971a20..168b2851 100644 --- a/src/java/com/arcadedb/python/ColumnBatcher.java +++ b/src/java/com/arcadedb/python/ColumnBatcher.java @@ -151,16 +151,22 @@ public static Object[] nextColumnBatch(final ResultSet rs, final int max, final final byte[] nulls = new byte[(count + 7) / 8]; // detect the column type across ALL values; mixed/unsupported -> json String type = null; + boolean sawInt = false, sawFloating = false, bigInt = false; for (int r = 0; r < count; r++) { final Object v = rows.get(r)[c]; if (v == null) continue; final String t; - if (v instanceof Long || v instanceof Integer || v instanceof Short || v instanceof Byte) + if (v instanceof Long || v instanceof Integer || v instanceof Short || v instanceof Byte) { t = "i8"; - else if (v instanceof Double || v instanceof Float) + sawInt = true; + final long l = ((Number) v).longValue(); + if (l > (1L << 53) || l < -(1L << 53)) + bigInt = true; + } else if (v instanceof Double || v instanceof Float) { t = "f8"; - else if (v instanceof Boolean) + sawFloating = true; + } else if (v instanceof Boolean) t = "b1"; else if (v instanceof Date || v instanceof Calendar || v instanceof LocalDateTime || v instanceof LocalDate || v instanceof Instant @@ -198,6 +204,9 @@ else if (!type.equals(t)) { } if (type == null) type = "str"; // all-null column + // an integer beyond 2**53 next to a floating value has no exact double: the values stay what they are, in a json column + if (type.equals("f8") && sawInt && sawFloating && bigInt) + type = "json"; int vdim = -1; if (type.equals("f4v") || type.equals("f8v")) { diff --git a/tests/test_column_frames.py b/tests/test_column_frames.py index 7956b15a..dc06f830 100644 --- a/tests/test_column_frames.py +++ b/tests/test_column_frames.py @@ -336,8 +336,198 @@ def test_to_dataframe_and_to_polars(table): ] assert [list(v) for v in df["v3"]] == [r["v3"] for r in rows] assert df["ts"].iloc[1] == datetime(2026, 1, 2, 3, 4, 5, 1000) + assert str(df["ts"].dtype).startswith( + "datetime64[us" + ) # microseconds on both wheels, whatever pandas infers from datetimes + # a vector cell is a numpy float32 array, not a Python list (documented on to_dataframe) + assert all(isinstance(v, np.ndarray) and v.dtype == np.float32 for v in df["v3"]) import polars as pl p = db.query("sql", _sel()).to_polars() assert isinstance(p, pl.DataFrame) and p.height == 5000 + assert p.schema["v3"] == pl.Array(pl.Float32, 3) assert p["big"].to_list() == [r["big"] for r in rows] + + +# --- what a column's type depends on: the whole result, not the size of the result or where a batch ends --------------------- + + +@pytest.mark.parametrize("batch_size", [2**31 - 1, 2**31, 2**32, 10**10]) +def test_a_huge_batch_size_means_as_many_as_one_batch_takes(table, batch_size): + db, rows = table + t = db.query("sql", _sel()).to_arrow(batch_size=batch_size) + assert t.num_rows == 5000 + c = db.query("sql", _sel()).to_columns(batch_size=batch_size) + assert len(c["big"]) == 5000 + + +@pytest.mark.parametrize("batch_size", [0, -1]) +def test_a_batch_size_below_one_is_refused(table, batch_size): + db, _ = table + with pytest.raises(ValueError): + db.query("sql", _sel()).to_arrow(batch_size=batch_size) + with pytest.raises(ValueError): + db.query("sql", _sel()).to_columns(batch_size=batch_size) + + +def test_no_rows_with_columns_given_returns_those_columns_empty(table): + db, _ = table + sql = "SELECT n, s FROM T WHERE big < 0" + t = db.query("sql", sql).to_arrow(columns=["s", "n"]) + assert t.column_names == ["s", "n"] and t.num_rows == 0 + assert db.query("sql", sql).to_columns(columns=["s", "n"]) == {"s": [], "n": []} + assert db.query("sql", sql).to_columns() == {} + assert db.query("sql", sql).to_arrow().num_columns == 0 + + +@pytest.mark.parametrize("limit", [1, 10, 300]) +def test_to_columns_arrays_are_writable(table, limit): + db, _ = table + c = db.query( + "sql", + f"SELECT big, d, ts, v3 FROM T ORDER BY big LIMIT {limit}", # nosec B608 - an integer of the test + ).to_columns() + for name in ("big", "d", "ts", "v3"): + assert c[name].flags.writeable, (name, limit) + c[name][0] = c[name][0] + + +@pytest.mark.parametrize("rows", [255, 256, 257, 511, 512, 513]) +def test_sizes_around_the_first_reply_and_two_of_them(table, rows): + db, stored = table + t = db.query("sql", _sel() + f" LIMIT {rows}").to_arrow() + assert t.num_rows == rows + assert t.column("big").to_pylist() == [r["big"] for r in stored[:rows]] + assert t.column("v9").to_pylist() == [r["v9"] for r in stored[:rows]] + assert t.schema.field("v9").type == pa.list_(pa.float32(), 9) + assert t.schema.field("s").type == pa.string() + + +def test_two_cursors_open_at_once_and_a_cursor_closed_early(table): + db, rows = table + one = db.query("sql", _sel()) + two = db.query("sql", _sel()) + assert next(one).get("big") == rows[0]["big"] + assert next(two).get("big") == rows[0]["big"] + t_two = two.to_arrow() + t_one = one.to_arrow() + assert t_one.column("big").to_pylist() == [r["big"] for r in rows[1:]] + assert t_two.column("big").to_pylist() == [r["big"] for r in rows[1:]] + partly = db.query("sql", _sel()) + for _ in range(300): + next(partly) + partly.close() + fresh = db.query("sql", _sel() + " LIMIT 400").to_arrow() + assert fresh.num_rows == 400 + + +def test_a_cursor_with_nothing_read_closed_many_times_over(table): + db, _ = table + cursors = [db.query("sql", _sel()) for _ in range(300)] + for cursor in cursors: + cursor.close() + assert db.query("sql", _sel() + " LIMIT 3").to_arrow().num_rows == 3 + + +@pytest.fixture +def declared(temp_db): + db = temp_db + db.command("sql", "CREATE DOCUMENT TYPE Dec") + db.command("sql", "CREATE PROPERTY Dec.i INTEGER") + db.command("sql", "CREATE PROPERTY Dec.arr ARRAY_OF_FLOATS") + db.command("sql", "CREATE PROPERTY Dec.dbl ARRAY_OF_DOUBLES") + db.command("sql", "CREATE PROPERTY Dec.lst LIST") + rows = [ + { + "i": i, + "arr": [0.5 * i, 1.0, 2.0], + "dbl": [0.5 * i, 1.5, 2.5], + "lst": [0.5 * i, 1.0, 2.0], + } + for i in range(3000) + ] + db.insert_many("Dec", rows) + return db, rows + + +@pytest.mark.parametrize("rows", [10, 300, 3000]) +def test_a_vector_column_is_a_declared_array_whatever_the_result_size(declared, rows): + db, stored = declared + sql = f"SELECT i, arr, dbl, lst FROM Dec ORDER BY i LIMIT {rows}" # nosec B608 - an integer of the test + c = db.query("sql", sql).to_columns() + assert isinstance(c["arr"], np.ndarray) and c["arr"].dtype == np.float32 + assert isinstance(c["dbl"], np.ndarray) and c["dbl"].dtype == np.float64 + assert isinstance(c["lst"], list), type( + c["lst"] + ) # a LIST of doubles is lists, not a float32 matrix that happens to fit + assert c["lst"] == [r["lst"] for r in stored[:rows]] + t = db.query("sql", sql).to_arrow() + assert t.schema.field("arr").type == pa.list_(pa.float32(), 3) + assert t.schema.field("dbl").type == pa.list_(pa.float64(), 3) + assert t.schema.field("lst").type == pa.list_(pa.float64()) + assert t.column("dbl").to_pylist() == [r["dbl"] for r in stored[:rows]] + + +def test_a_double_array_stays_double_after_rows_read_past_the_first_reply(declared): + db, stored = declared + rs = db.query("sql", "SELECT i, dbl, arr FROM Dec ORDER BY i") + for _ in range(300): + next(rs) + t = rs.to_arrow() + assert t.schema.field("dbl").type == pa.list_(pa.float64(), 3) + assert t.schema.field("arr").type == pa.list_(pa.float32(), 3) + assert t.column("dbl").to_pylist() == [r["dbl"] for r in stored[300:]] + + +def test_a_property_that_is_numeric_from_row_30000_on_has_one_type(temp_db): + db = temp_db + db.command("sql", "CREATE DOCUMENT TYPE Late") + db.command("sql", "CREATE PROPERTY Late.i INTEGER") + rows = [{"i": i} for i in range(30000)] + [ + {"i": i, "late": i * 2} for i in range(30000, 30100) + ] + db.insert_many("Late", rows) + sql = "SELECT i, late FROM Late ORDER BY i" + t = db.query("sql", sql).to_arrow(batch_size=25_000, columns=["i", "late"]) + assert t.schema.field("late").type == pa.int64() + late = t.column("late").to_pylist() + assert late[:30000] == [None] * 30000 and late[30000:] == [ + i * 2 for i in range(30000, 30100) + ] + c = db.query("sql", sql).to_columns(batch_size=25_000, columns=["i", "late"]) + assert isinstance(c["late"], np.ndarray) and c["late"].dtype == np.float64 + assert np.isnan(c["late"][:30000]).all() + assert list(c["late"][30000:]) == [i * 2 for i in range(30000, 30100)] + + +@pytest.mark.parametrize("rows", [10, 300]) +def test_an_integer_beyond_2_53_next_to_a_float_stays_exact(temp_db, rows): + db = temp_db + db.command("sql", "CREATE DOCUMENT TYPE Mix") + db.command("sql", "CREATE PROPERTY Mix.i INTEGER") + big = 2**60 + 1 + db.insert_many( + "Mix", [{"i": i, "x": big if i % 2 == 0 else 1.5} for i in range(rows)] + ) + c = db.query("sql", "SELECT i, x FROM Mix ORDER BY i").to_columns() + want = [big if i % 2 == 0 else 1.5 for i in range(rows)] + assert list(c["x"]) == want + assert all(type(a) is type(b) for a, b in zip(c["x"], want)) + t = db.query("sql", "SELECT i, x FROM Mix ORDER BY i").to_arrow() + assert t.column("x").to_pylist() == [ + str(v) for v in want + ] # no common numeric type: exact text, as across batches + + +def test_vectors_whose_dimension_changes_between_frames_are_one_list_column(temp_db): + db = temp_db + db.command("sql", "CREATE DOCUMENT TYPE Dim") + db.command("sql", "CREATE PROPERTY Dim.i INTEGER") + db.command("sql", "CREATE PROPERTY Dim.v ARRAY_OF_FLOATS") + db.insert_many( + "Dim", + [{"i": i, "v": [0.5, 1.0, 2.0] if i < 300 else [0.5, 1.0]} for i in range(600)], + ) + t = db.query("sql", "SELECT i, v FROM Dim ORDER BY i").to_arrow(batch_size=100) + assert t.schema.field("v").type == pa.list_(pa.float32()) + assert t.column("v").to_pylist() == [[0.5, 1.0, 2.0]] * 300 + [[0.5, 1.0]] * 300 diff --git a/tests/test_native_column_frames_reference.py b/tests/test_native_column_frames_reference.py index 630a4390..2fc38c32 100644 --- a/tests/test_native_column_frames_reference.py +++ b/tests/test_native_column_frames_reference.py @@ -262,23 +262,20 @@ def ref_db(tmp_path_factory): db.close() -def _norm(v): - """A value as a plain Python object, NaN, NaT and None alike (pandas turns a None into NaN in some columns).""" - if isinstance(v, np.ndarray): - return [_norm(x) for x in v.tolist()] - if isinstance(v, (list, tuple)): - return [_norm(x) for x in v] - if v is None or v is pd.NaT: - return None - if isinstance(v, float) and math.isnan(v): - return None - if isinstance(v, np.datetime64): - return None if np.isnat(v) else v.astype("datetime64[us]").item() - if isinstance(v, np.generic): - return v.item() - if isinstance(v, pd.Timestamp): - return v.to_pydatetime() - return v +def _eq(have, want): + """Equal values, NaN equal to NaN, None equal only to None: a missing value is not a NaN.""" + if isinstance(have, np.ndarray) or isinstance(want, np.ndarray): + return np.array_equal(np.asarray(have), np.asarray(want), equal_nan=True) + if isinstance(have, (list, tuple)) and isinstance(want, (list, tuple)): + return len(have) == len(want) and all(_eq(x, y) for x, y in zip(have, want)) + if ( + isinstance(have, float) + and isinstance(want, float) + and math.isnan(have) + and math.isnan(want) + ): + return True + return have == want def _old_rows(db, sql, *args): @@ -292,16 +289,19 @@ def _assert_arrow_equal(new, old): have, want = new.column(name), old.column(name) if pa.types.is_null(want.type): assert have.type == pa.string() and have.null_count == len(have), name - elif pa.types.is_list(want.type) and want.type.value_type == pa.float64(): - assert pa.types.is_fixed_size_list(have.type) or pa.types.is_list( - have.type + elif ( + pa.types.is_list(want.type) + and want.type.value_type == pa.float64() + and name.startswith("v") + ): + # a float[] column: the same values as a fixed-size list of float32 (a float32 widened to float64 is exact) + assert have.type == pa.list_( + pa.float32(), + len(want[0].as_py() if want[0].is_valid else want[1].as_py()), ), name - assert have.type.value_type in (pa.float32(), pa.float64()), name else: assert have.type == want.type, (name, have.type, want.type) - assert [_norm(x) for x in have.to_pylist()] == [ - _norm(x) for x in want.to_pylist() - ], name + assert _eq(have.to_pylist(), want.to_pylist()), name def _assert_columns_equal(new, old): @@ -312,9 +312,22 @@ def _assert_columns_equal(new, old): assert isinstance(have, np.ndarray), (name, type(have)) assert have.dtype == want.dtype, (name, have.dtype, want.dtype) assert have.shape == want.shape, name + np.testing.assert_array_equal(have, want, err_msg=name) else: assert isinstance(have, list), (name, type(have)) - assert _norm(have) == _norm(want), name + assert _eq(have, want), name + + +def _missing_as_none(values): + """pandas decides how a missing value is shown (None, nan or NaT, by dtype and version): all of them are one here.""" + return [ + ( + None + if v is None or v is pd.NaT or (isinstance(v, float) and math.isnan(v)) + else v + ) + for v in values + ] def _assert_frame_equal(new, old_rows, columns=None): @@ -323,7 +336,31 @@ def _assert_frame_equal(new, old_rows, columns=None): old = old.reindex(columns=list(columns)) assert list(new.columns) == list(old.columns) for name in old.columns: - assert _norm(new[name].tolist()) == _norm(old[name].tolist()), name + if name.startswith("v") and name != "vv": + # a vector cell is a numpy float32 array (documented), the values are the stored ones + for cell, want in zip(new[name].tolist(), old[name].tolist()): + if want is None: + assert cell is None or ( + isinstance(cell, float) and math.isnan(cell) + ), name + else: + assert isinstance(cell, np.ndarray) and cell.dtype == np.float32, ( + name, + type(cell), + ) + assert _eq(cell.tolist(), want), name + continue + have = _missing_as_none(new[name].tolist()) + want = _missing_as_none(old[name].tolist()) + assert [ + x.to_pydatetime() if isinstance(x, pd.Timestamp) else x for x in have + ] == [ + x.to_pydatetime() if isinstance(x, pd.Timestamp) else x for x in want + ], name + if "ts" in old.columns: + assert str(new["ts"].dtype).startswith("datetime64[us"), new[ + "ts" + ].dtype # microseconds, whatever pandas infers from datetimes QUERIES = [ @@ -425,10 +462,68 @@ def test_to_list_still_reads_the_json_rows(ref_db): assert len(rows) == 300 and rows[299]["big"] == (1 << 40) + 299 -def test_the_cursor_keeps_no_rows_after_a_row_read(ref_db): +def test_a_frame_stays_near_its_budget_when_every_row_has_other_properties(tmp_path): + """Rows that each carry a property no other row has: the frame holds rows times the union of the names, not the values read.""" + from unittest import mock + + import arcadedb + from arcadedb._native import lib + + db = arcadedb.create_database(str(tmp_path / "sparse")) + try: + db.command("sql", "CREATE DOCUMENT TYPE Sparse") + db.insert_many("Sparse", [{"i": i, f"p{i}": i} for i in range(2000)]) + sizes = [] + real = lib.call_frame + + def spy(*args): + raw = real(*args) + sizes.append(len(raw)) + return raw + + with mock.patch.object(lib, "call_frame", spy): + t = db.query("sql", "SELECT FROM Sparse ORDER BY i").to_arrow() + assert t.num_rows == 2000 and t.num_columns == 2001 + assert t.column("p1999").to_pylist() == [None] * 1999 + [1999] + # about 1 MiB of fixed-width buffers (rows x columns x 9 bytes) is what a frame is allowed; 2000 x 2000 x 9 = 36 MiB in one frame is not + assert len(sizes) > 5 and max(sizes) < 3 * (1 << 20), sizes + finally: + db.close() + + +def test_an_empty_frame_for_a_cursor_with_rows_left_is_an_error_not_a_loop(ref_db): + from unittest import mock + + from arcadedb._native import lib + from arcadedb.exceptions import ArcadeDBError + + header = b'{"count":0,"done":false,"cols":[]}' + empty = len(header).to_bytes(4, "little") + header + rs = ref_db.query("sql", "SELECT FROM R ORDER BY big") - assert rs._head_n == 256 - for _ in range(300): - next(rs) - assert rs._head_n == 0 - rs.close() + with mock.patch.object(lib, "call_frame", lambda *args: empty): + with pytest.raises(ArcadeDBError, match="empty column frame"): + rs.to_arrow() + + +def test_a_cursor_reads_a_huge_batch_size_without_wrapping(ref_db): + sql = "SELECT big FROM R ORDER BY big" + for batch_size in (2**31, 2**32, 2**33 + 5): + assert ref_db.query("sql", sql).to_arrow(batch_size=batch_size).num_rows == ROWS + + +def test_a_list_property_is_not_a_vector_column_at_any_result_size(tmp_path): + import arcadedb + + db = arcadedb.create_database(str(tmp_path / "lists")) + try: + db.command("sql", "CREATE DOCUMENT TYPE L") + db.insert_many("L", [{"i": i, "lst": [0.5, 1.0, 2.0]} for i in range(600)]) + for rows in (5, 300, 600): + c = db.query( + "sql", + f"SELECT i, lst FROM L ORDER BY i LIMIT {rows}", # nosec B608 - an integer of the test + ).to_columns() + assert isinstance(c["lst"], list) and c["lst"] == [[0.5, 1.0, 2.0]] * rows + finally: + db.close() From dc55dec8d4b62cda52d4c34aedf7877a1bd6ad22 Mon Sep 17 00:00:00 2001 From: Taewoon Kim Date: Mon, 9 Nov 2026 22:22:23 +0900 Subject: [PATCH 22/49] tests: a duplicate key in a parallel load abandons the failing writer's open batch (CI of the first candidate) The first candidate run showed stored 1000 of 2950 distinct keys: a writer that meets a duplicate key rolls back its whole open batch (up to commit_every rows), so the valid rows of that batch are abandoned and reported too. stored == distinct was the wrong invariant. Assert stored + reported >= n, stored <= distinct, only sent ids, none stored twice. Co-Authored-By: Claude Sonnet 5.5 --- ...est_native_columnar_parallel_review_fixes.py | 17 ++++++++++------- 1 file changed, 10 insertions(+), 7 deletions(-) diff --git a/tests/test_native_columnar_parallel_review_fixes.py b/tests/test_native_columnar_parallel_review_fixes.py index 72313ee6..70f17bc3 100644 --- a/tests/test_native_columnar_parallel_review_fixes.py +++ b/tests/test_native_columnar_parallel_review_fixes.py @@ -1,8 +1,8 @@ """Review of the native parallel loads (#328): nothing is lost silently, an interrupt leaves no writer running and no token behind. ``insert_columns(parallel=True)`` hands the records to the async writers in one crossing and waits for them before it returns. These -tests pin what a caller relies on: a record that a writer rejects is counted (``stored + reported failed >= rows``) and the stored -count is exactly the distinct keys; a load at the writers' batch boundary stores every row and nothing lands after the call returns; +tests pin what a caller relies on: a record that a writer rejects is counted (``stored + reported failed >= rows``) and no key is stored twice +(a writer that meets a duplicate abandons its whole open batch, and reports it); a load at the writers' batch boundary stores every row and nothing lands after the call returns; a ^C during the crossing, or between the chunks of the row path, waits for the writers and reads (so removes) the failure token; a wait that fails still reads the tokens; and the message of a failed parallel batch inside a caller's transaction does not claim that a rollback removes rows the writers committed. Needs the native library: conftest.py collects this file on the native backend only. @@ -70,20 +70,23 @@ def _failed_count(error): return int(re.search(r"reported (\d+) failed record", str(error)).group(1)) -def test_a_duplicate_key_is_stored_once_and_the_rest_are_reported(temp_db): +def test_a_duplicate_key_is_reported_and_no_row_is_stored_twice(temp_db): _type(temp_db, unique=True) n, dupes = 3_000, 50 ids = np.arange(n, dtype=np.int64) ids[n - dupes :] = np.arange(dupes) # the last 50 repeat the first 50 keys with pytest.raises(ArcadeDBError, match="failed record") as err: temp_db.insert_columns("Col", {"id": ids, "txt": ["t"] * n}, parallel=True) - stored = _count(temp_db, "Col") + stored_ids = [r["id"] for r in temp_db.query("sql", "SELECT id FROM Col").to_list()] + stored = len(stored_ids) assert ( stored + _failed_count(err.value) >= n ) # every row is a stored row or a reported one - assert stored == len(set(ids.tolist())) == n - dupes - keys = {r["id"] for r in temp_db.query("sql", "SELECT id FROM Col").to_list()} - assert keys == set(ids.tolist()) # each key once, no key lost + # A writer that meets a duplicate key rolls back its WHOLE open batch (up to commit_every rows), so the valid rows of that batch are + # abandoned and reported too: `stored` can be below the number of distinct keys, never above it. + assert stored <= len(set(ids.tolist())) + assert set(stored_ids) <= set(ids.tolist()) # only ids that were sent + assert len(stored_ids) == len(set(stored_ids)) # none stored twice def test_a_load_at_the_writers_batch_boundary_stores_every_row_and_nothing_later( From 79bb5193271d5554f560296f5a82959a8afae77f Mon Sep 17 00:00:00 2001 From: Taewoon Kim Date: Mon, 9 Nov 2026 19:02:40 +0900 Subject: [PATCH 23/49] Native columnar insert: date-time, date, decimal and bytes columns stay columnar (refs #316) One column of a date-time, date, Decimal or bytes value used to make columnar.pack return None, so every column of the batch went through one dict per row and a JSON text with tagged values. BulkOps.insertColumns gets four kinds that read the same objects Json.decodeTag builds for those tagged values, so the stored value is the same: T (int64 ticks of the UTC wall clock since the epoch, dim = ticks per second so an Arrow timestamp keeps its unit, LocalDateTime), D (int32 epoch days, LocalDate), n (decimal text in the string layout, BigDecimal) and x (bytes in the string layout, byte[]). columnar.py lays out lists of datetime (also pandas.Timestamp), date, Decimal, bytes and bytearray and Arrow timestamp, date32, date64, decimal and binary arrays; a time-zone-aware value is its UTC wall clock and a naive one is taken as it is, the rule of _tag_datetime. The DataFrame and Arrow loader keeps those Arrow columns for the entry and turns them into tagged values only when a batch falls back to the row path. The native library is rebuilt from the unchanged engine commit. Co-Authored-By: Claude Sonnet 5.5 --- .../java/org/humemai/nativelib/BulkOps.java | 39 ++- src/arcadedb/_native/bulk.py | 4 +- src/arcadedb/_native/columnar.py | 114 ++++++- src/arcadedb/_native/database.py | 3 +- src/arcadedb/dataframe.py | 24 +- tests/conftest.py | 2 + tests/test_columnar_temporal_lists.py | 113 ++++++ tests/test_native_columnar_insert.py | 23 +- tests/test_native_columnar_pack.py | 97 +++++- tests/test_native_columnar_temporal.py | 322 ++++++++++++++++++ tests/test_native_mixed_chunk.py | 3 +- 11 files changed, 712 insertions(+), 32 deletions(-) create mode 100644 tests/test_columnar_temporal_lists.py create mode 100644 tests/test_native_columnar_temporal.py diff --git a/native/src/main/java/org/humemai/nativelib/BulkOps.java b/native/src/main/java/org/humemai/nativelib/BulkOps.java index cdf3cd98..00798947 100644 --- a/native/src/main/java/org/humemai/nativelib/BulkOps.java +++ b/native/src/main/java/org/humemai/nativelib/BulkOps.java @@ -19,8 +19,12 @@ import org.graalvm.word.Pointer; import org.graalvm.word.WordFactory; +import java.math.BigDecimal; import java.nio.ByteBuffer; import java.nio.charset.StandardCharsets; +import java.time.LocalDate; +import java.time.LocalDateTime; +import java.time.ZoneOffset; import java.util.List; import java.util.Map; @@ -82,7 +86,10 @@ static String insertColumnsReply(final Database db, final String type, final Str /** * header: {"n":rows,"commit":every,"cols":[[name,kind,dim],..]}. Kinds: q (int64), d (float64), b (bool byte), i (int32), e (float32), * f (float32 matrix, dim floats per row), s (UTF-8 string: int32 offsets[n+1] in the data slot, the bytes they index in the aux slot, as - * an Arrow string array; offsets[0] need not be 0). spec: per column six longs, [data address, data bytes, aux address, aux bytes, null + * an Arrow string array; offsets[0] need not be 0). The kinds that the JSON rows carry as tagged values (Json.decodeTag), read as the same + * objects: T (int64 ticks since 1970-01-01T00:00:00 of the UTC wall clock, dim = ticks per second: 1, 1000, 10^6 or 10^9, read as a + * LocalDateTime), D (int32 days since the epoch, a LocalDate), n (a decimal as text in the string layout, a BigDecimal) and x (bytes in + * the string layout, a byte[]), so a column of these is stored as the row path stores it. spec: per column six longs, [data address, data bytes, aux address, aux bytes, null * bitmap address, null bitmap bytes]; the bitmap is Arrow's (bit i of byte i/8, least significant first; 1 = present), an address of 0 * means no nulls. The memory belongs to the caller for the duration of the call; nothing is kept. Every length is checked here before * the first document is built, so a wrong table is an exception and nothing is written. Same transaction contract as @@ -121,13 +128,18 @@ private static long insertColumns(final Database db, final String type, final Ma final long need = switch (kinds[c]) { case 'q', 'd' -> 8L * n; case 'b' -> n; - case 'i', 'e' -> 4L * n; + case 'i', 'e', 'D' -> 4L * n; + case 'T' -> { + if (dims[c] != 1 && dims[c] != 1_000 && dims[c] != 1_000_000 && dims[c] != 1_000_000_000) + throw new IllegalArgumentException("column '" + names[c] + "': a date-time column has 1, 1000, 10^6 or 10^9 ticks per second, not " + dims[c]); + yield 8L * n; + } case 'f' -> { if (dims[c] <= 0) throw new IllegalArgumentException("column '" + names[c] + "': a vector column needs a dimension above 0"); yield 4L * n * dims[c]; } - case 's' -> 4L * (n + 1L); + case 's', 'n', 'x' -> 4L * (n + 1L); default -> throw new IllegalArgumentException("unknown column kind " + kinds[c]); }; if (dataBytes < need || (need > 0 && dataAddr == 0)) @@ -138,7 +150,7 @@ private static long insertColumns(final Database db, final String type, final Ma throw new IllegalArgumentException("column '" + names[c] + "' has a null bitmap of " + nullBytes + " bytes, " + (n + 7L) / 8 + " needed"); nulls[c] = nullAddr; } - if (kinds[c] == 's') { + if (kinds[c] == 's' || kinds[c] == 'n' || kinds[c] == 'x') { if (auxBytes < 0 || auxBytes > Integer.MAX_VALUE || (auxBytes > 0 && auxAddr == 0)) throw new IllegalArgumentException("column '" + names[c] + "': the string bytes are missing or beyond 2 GiB"); blobs[c] = auxBytes == 0 ? ByteBuffer.allocate(0) : CTypeConversion.asByteBuffer(WordFactory.pointer(auxAddr), (int) auxBytes); @@ -220,6 +232,25 @@ private static void setRow(final MutableDocument doc, final int i, final String[ case 'b' -> doc.set(names[c], base.readByte(WordFactory.signed(i)) != 0); case 'i' -> doc.set(names[c], base.readInt(WordFactory.signed(4L * i))); case 'e' -> doc.set(names[c], base.readFloat(WordFactory.signed(4L * i))); + case 'T' -> { + final long ticks = base.readLong(WordFactory.signed(8L * i)); + final long perSecond = dims[c]; + doc.set(names[c], LocalDateTime.ofEpochSecond(Math.floorDiv(ticks, perSecond), + (int) (Math.floorMod(ticks, perSecond) * (1_000_000_000L / perSecond)), ZoneOffset.UTC)); + } + case 'D' -> doc.set(names[c], LocalDate.ofEpochDay(base.readInt(WordFactory.signed(4L * i)))); + case 'n' -> { + final int from = base.readInt(WordFactory.signed(4L * i)); + final int len = base.readInt(WordFactory.signed(4L * (i + 1L))) - from; + blobs[c].get(from, scratch, 0, len); + doc.set(names[c], new BigDecimal(new String(scratch, 0, len, StandardCharsets.US_ASCII))); + } + case 'x' -> { + final int from = base.readInt(WordFactory.signed(4L * i)); + final byte[] bytes = new byte[base.readInt(WordFactory.signed(4L * (i + 1L))) - from]; + blobs[c].get(from, bytes, 0, bytes.length); + doc.set(names[c], bytes); + } case 's' -> { final int from = base.readInt(WordFactory.signed(4L * i)); final int len = base.readInt(WordFactory.signed(4L * (i + 1L))) - from; diff --git a/src/arcadedb/_native/bulk.py b/src/arcadedb/_native/bulk.py index 9851848f..c16d225b 100644 --- a/src/arcadedb/_native/bulk.py +++ b/src/arcadedb/_native/bulk.py @@ -1,7 +1,7 @@ """Columnar bulk insert through one crossing (``arcadedb_n_insert_colp``): whole columns go over by address, with no JSON text. Numpy ints, floats and bools, lists of one value type (None is a null), strings (UTF-8 bytes plus offsets, as Arrow lays them out), float32 -vector columns and Arrow arrays are laid out by :mod:`.columnar`; anything else (nested values, mixed-type columns, date-times, -decimals, integers past 64 bits) returns None and the caller uses the per-row JSON path. The documents are built image-side as +vector columns, date-times, dates, decimals, bytes and Arrow arrays are laid out by :mod:`.columnar`; anything else (nested values, +mixed-type columns, integers past 64 bits) returns None and the caller uses the per-row JSON path. The documents are built image-side as ``DocumentBatcher.insertColumns`` builds them on JPype; with ``parallel`` they are handed to the async writers instead of a transaction (``DocumentBatcher.insertColumnsParallel``). ``ARCADEDB_NATIVE_BULK=0`` turns the columnar path off. """ diff --git a/src/arcadedb/_native/columnar.py b/src/arcadedb/_native/columnar.py index 27d261a4..46537e38 100644 --- a/src/arcadedb/_native/columnar.py +++ b/src/arcadedb/_native/columnar.py @@ -2,19 +2,29 @@ A column crosses as raw memory, not as JSON text: the image reads each buffer in place and builds the documents without a parser. This module only decides how a column is laid out and hands the addresses over; it needs no native library, so the layouts are unit-tested -without one. A column that cannot be laid out (a nested value, a mixed-type column, a date-time, an integer past 64 bits, a string with -a lone surrogate) makes :func:`pack` return ``None`` and the caller uses the JSON path for that call; nothing is half sent. +without one. A column that cannot be laid out (a nested value, a mixed-type column, a numpy datetime64, an integer past 64 bits, a string +with a lone surrogate) makes :func:`pack` return ``None`` and the caller uses the JSON path for that call; nothing is half sent. Kinds (the image's, ``BulkOps.insertColumns``): ``q`` int64, ``d`` float64, ``b`` bool byte, ``i`` int32, ``e`` float32, ``f`` float32 matrix (``dim`` per row), ``s`` UTF-8 string (``int32 offsets[n+1]`` and the bytes they index: Arrow's string layout). A column may carry a null bitmap (Arrow's: bit ``i`` of byte ``i // 8``, least significant first, 1 = present). +Temporal, decimal and binary kinds (the values the JSON path sends as tagged values, so the stored value is the same object): ``T`` a +date-time, int64 ticks since 1970-01-01T00:00:00 of the UTC wall clock, ``dim`` ticks per second (1, 1000, 10**6 or 10**9: an Arrow +timestamp keeps its unit, a Python ``datetime`` is microseconds), read as ``LocalDateTime``; ``D`` a date, int32 days since the epoch, +read as ``LocalDate``; ``n`` a decimal, the string layout holding ``str(Decimal)``, read as ``BigDecimal``; ``x`` bytes, the string +layout, read as ``byte[]``. A time-zone-aware ``datetime`` or Arrow timestamp is its UTC wall clock and a naive one is taken as it is, +the rule of ``_tag_datetime``; nothing here decides a time zone. + ``i`` and ``e`` and the float matrix from a list of lists are chosen only for a property declared ``INTEGER``, ``FLOAT`` and ``ARRAY_OF_FLOATS``: the engine would convert to those types anyway, so the stored value is the one the JSON path stores. For an undeclared property an ``int32`` column crosses as ``q`` (a Long) and a ``float32`` column as ``d`` (a Double), as the JSON path stores them. """ +from datetime import date, datetime, timezone +from decimal import Decimal + from .schema_objects import ( # the schema-entry reader shared with NativeType (#195) _name_type, ) @@ -27,6 +37,12 @@ _INT64_MAX = 2**63 - 1 _INT32_MAX = 2**31 - 1 _NoneType = type(None) +_EPOCH_ORDINAL = date(1970, 1, 1).toordinal() +_UTC = timezone.utc +_BLOB_KINDS = ( + "snx" # the kinds laid out as int32 offsets[n + 1] plus the bytes they index +) +_TICKS = {"s": 1, "ms": 1_000, "us": 1_000_000, "ns": 1_000_000_000} class Packed: @@ -39,8 +55,8 @@ def __init__(self, name, kind, dim, array, aux=None, nulls=None, keep=()): self.kind = kind self.dim = dim self.n = int(array.shape[0]) - ( - 1 if kind == "s" else 0 - ) # a string column's array is its n + 1 offsets + 1 if kind in _BLOB_KINDS else 0 + ) # a string, decimal or bytes column's array is its n + 1 offsets self.data = (array.ctypes.data, array.nbytes) self.aux = aux or (0, 0) self.nulls = (nulls.ctypes.data, nulls.nbytes) if nulls is not None else (0, 0) @@ -166,6 +182,35 @@ def _string_column(name, texts, nulls=None): return Packed(name, "s", 0, offsets, aux=aux, nulls=nulls, keep=(blob_array, data)) +def _blob_column(name, kind, items, nulls=None): + """A list of bytes (a null already replaced by b'') as an offsets-plus-bytes column of ``kind``, or None past 2 GiB.""" + n = len(items) + data = b"".join(items) + if len(data) > _INT32_MAX: + return None + lengths = _np.fromiter(map(len, items), dtype=_np.int64, count=n) + offsets = _np.zeros(n + 1, dtype=_np.int32) + _np.cumsum(lengths, out=offsets[1:], dtype=_np.int32) + blob_array = ( + _np.frombuffer(data, dtype=_np.uint8) if data else _np.zeros(0, dtype=_np.uint8) + ) + aux = (blob_array.ctypes.data, blob_array.nbytes) if data else (0, 0) + return Packed(name, kind, 0, offsets, aux=aux, nulls=nulls, keep=(blob_array, data)) + + +def _epoch_micros(value): + """Microseconds since 1970-01-01T00:00:00 of the UTC wall clock of a ``datetime`` (a naive one is taken as it is, as ``_tag_datetime``).""" + if value.utcoffset() is not None: + value = value.astimezone(_UTC) + seconds = ( + (value.toordinal() - _EPOCH_ORDINAL) * 86400 + + value.hour * 3600 + + value.minute * 60 + + value.second + ) + return seconds * 1_000_000 + value.microsecond + + def _list_column(name, values, declared): """A list or tuple of Python values as a column: one type (None allowed) or a list of equal-length float arrays; else None.""" n = len(values) @@ -203,6 +248,33 @@ def _list_column(name, values, declared): return None # an integer beyond 64 bits kind = "q" if only is int else "d" if only is float else "b" return Packed(name, kind, 0, arr, nulls=nulls) + if issubclass(only, datetime): + if only.__name__ == "NaTType": + return None # the JSON path stores it as a null + try: + ticks = [0 if v is None else _epoch_micros(v) for v in values] + except (OverflowError, ValueError): + return None # the JSON path raises what it raised before + return Packed( + name, "T", 1_000_000, _np.array(ticks, dtype=_np.int64), nulls=nulls + ) + if only is date: + days = [0 if v is None else v.toordinal() - _EPOCH_ORDINAL for v in values] + return Packed(name, "D", 0, _np.array(days, dtype=_np.int32), nulls=nulls) + if only is Decimal: + texts = [] + for v in values: + if v is None: + texts.append(b"") + elif v.is_finite(): + texts.append(str(v).encode("ascii")) + else: + return None # the JSON path sends a non-finite Decimal down the per-row path + return _blob_column(name, "n", texts, nulls) + if only is bytes or only is bytearray: + return _blob_column( + name, "x", values if present is None else [v or b"" for v in values], nulls + ) if only is list and declared.get(name) == "ARRAY_OF_FLOATS" and present is None: try: arr = _np.array(values, dtype=_np.float32) @@ -248,10 +320,11 @@ def _arrow_column(name, arr, declared): if arr.null_count else None ) - if types.is_string(t) or types.is_large_string(t): + is_text = types.is_string(t) or types.is_large_string(t) + if is_text or types.is_binary(t) or types.is_large_binary(t): bufs = arr.buffers() off_buf, data_buf = bufs[1], bufs[2] - if types.is_string(t): + if types.is_string(t) or types.is_binary(t): offsets = _np.frombuffer(off_buf, dtype=_np.int32)[ arr.offset : arr.offset + n + 1 ] @@ -266,8 +339,35 @@ def _arrow_column(name, arr, declared): size = 0 if data_buf is None else min(data_buf.size, _INT32_MAX) aux = (data_buf.address, size) if size else (0, 0) return Packed( - name, "s", 0, offsets, aux=aux, nulls=nulls, keep=(arr, off_buf, data_buf) + name, + "s" if is_text else "x", + 0, + offsets, + aux=aux, + nulls=nulls, + keep=(arr, off_buf, data_buf), + ) + if types.is_timestamp(t): + values = arr.cast(pa.int64()).fill_null(0).to_numpy(zero_copy_only=False) + return Packed( + name, + "T", + _TICKS[t.unit], + _np.ascontiguousarray(values, dtype=_np.int64), + nulls=nulls, + ) + if types.is_date(t): + if types.is_date64(t): # milliseconds since the epoch, floor-divided to days + raw = arr.cast(pa.int64()).fill_null(0).to_numpy(zero_copy_only=False) + days = raw // 86_400_000 + else: + days = arr.cast(pa.int32()).fill_null(0).to_numpy(zero_copy_only=False) + return Packed( + name, "D", 0, _np.ascontiguousarray(days, dtype=_np.int32), nulls=nulls ) + if types.is_decimal(t): + texts = [b"" if v is None else str(v).encode("ascii") for v in arr.to_pylist()] + return _blob_column(name, "n", texts, nulls) if types.is_integer(t) or types.is_floating(t) or types.is_boolean(t): if arr.null_count: arr = arr.fill_null(False if types.is_boolean(t) else 0) diff --git a/src/arcadedb/_native/database.py b/src/arcadedb/_native/database.py index 6fab6c58..3ee0f51b 100644 --- a/src/arcadedb/_native/database.py +++ b/src/arcadedb/_native/database.py @@ -877,7 +877,8 @@ def insert_many( a newDocument/set/save loop inside the image, as ``DocumentBatcher.insertManyJson`` on the JPype path). A ``datetime`` (also ``pandas.Timestamp`` and ``numpy.datetime64``), ``date``, ``Decimal``, ``bytes``/``bytearray``, numpy scalar, ``pandas.NA`` or ``pandas.NaT`` sitting directly in a row crosses in that text as a tagged value (``{"\\u0001": [kind, payload]}``, decoded by - ``Json.java``), so it does not move the chunk off this path; such a chunk loses the columnar entry. A row with a value the JSON + ``Json.java``), so it does not move the chunk off this path; but when a column holds only date-times, dates, Decimals or + bytes (None is a null) the chunk goes as columns, the same stored objects. A row with a value the JSON text cannot carry unchanged (NaN or Infinity, an integer beyond 64 bits, a date-time or decimal inside a list or map, a non-str key, a lone surrogate, a ``datetime.time``) goes through ``INSERT INTO t CONTENT :row`` on its own, in one crossing for a run of such rows; the rows around it keep the bulk path and every row is inserted in the order given. Inside the caller's diff --git a/src/arcadedb/dataframe.py b/src/arcadedb/dataframe.py index 46362e53..37c67a29 100644 --- a/src/arcadedb/dataframe.py +++ b/src/arcadedb/dataframe.py @@ -325,8 +325,10 @@ def _column_values(name: str, chunk, pa, native: bool = False, arrow: bool = Fal """One Arrow array as what ``Database.insert_columns`` takes: a numpy array when the column crosses as one buffer (integers, floats and booleans without nulls), else a list of values (``None`` is a null). - ``arrow`` (the native backend's document load): a string column, and a numeric one with nulls, stay Arrow arrays, which - the columnar entry reads in place (the offsets, the bytes and the validity bitmap) instead of as Python values. + ``arrow`` (the native backend's document load): a string, binary, timestamp, date or decimal column, and a numeric one with + nulls, stay Arrow arrays, which the columnar entry reads in place (the offsets, the bytes, the integer ticks and the validity + bitmap) instead of as Python values; ``_insert_documents_native`` turns them into tagged values for the row path when a batch + cannot go columnar. """ types = pa.types if types.is_dictionary(chunk.type): # a pandas categorical or a Polars Categorical @@ -361,6 +363,8 @@ def _column_values(name: str, chunk, pa, native: bool = False, arrow: bool = Fal values = values.astype("float64") # float16/32 widen exactly return values return chunk.to_pylist() + if arrow and kind in ("timestamp", "date", "decimal", "binary"): + return chunk if kind in ("timestamp", "date"): return ( _temporal_to_tagged(chunk, pa) if native else _temporal_to_java(chunk, pa) @@ -496,6 +500,20 @@ def _insert_vertices( ) +def _arrow_row_values(col) -> list: + """A column kept as an Arrow array, as the values the row path takes: Python values, or tagged values for a timestamp, date, + decimal or binary column (the form it had before the columnar entry took them, so the stored value is unchanged). + """ + import pyarrow as pa + + kind = _kind(pa, col.type) + if kind in ("timestamp", "date"): + return _temporal_to_tagged(col, pa) + if kind in ("decimal", "binary"): + return _scalars_to_tagged(kind, col) + return col.to_pylist() + + def _insert_documents_native( db, type_name: str, @@ -542,7 +560,7 @@ def _insert_documents_native( if hasattr( col, "to_pylist" ): # a column kept as an Arrow array for the columnar entry - lists.append(col.to_pylist()) + lists.append(_arrow_row_values(col)) else: # jvm-payload: not-a-crossing -- no JVM here: insert_many serialises the rows to JSON text lists.append(col.tolist() if hasattr(col, "tolist") else col) diff --git a/tests/conftest.py b/tests/conftest.py index 707833d1..dc34241e 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -51,6 +51,8 @@ collect_ignore.append("test_native_columnar_parallel.py") # the review of the native parallel loads: no silent loss, an interrupt waits for the writers and reads the failure token collect_ignore.append("test_native_columnar_parallel_review_fixes.py") + # a column of date-times, dates, Decimals or bytes crosses as a column (kinds T D n x), observed on the native library + collect_ignore.append("test_native_columnar_temporal.py") # the rows of an insert_many chunk with date-times, Decimals, bytes and numpy scalars are told apart by the entry points they cross collect_ignore.append("test_native_mixed_chunk.py") # the native build's ingest defaults (insert_many and graph_batch commit size), observed on the native library diff --git a/tests/test_columnar_temporal_lists.py b/tests/test_columnar_temporal_lists.py new file mode 100644 index 00000000..35aacabb --- /dev/null +++ b/tests/test_columnar_temporal_lists.py @@ -0,0 +1,113 @@ +"""A column of date-times, dates, Decimals or bytes stores the same values on both wheels (ArcadeData/arcadedb-python #316). + +On the native wheel such a column used to move the whole batch off the columnar entry; now ``insert_columns`` and ``insert_many`` send +it as a column (``tests/test_native_columnar_temporal.py`` proves the entry and compares with the row path). Here the expected values +are literals, so the same file on the JVM wheel and on the native wheel says that both store them: a naive date-time as it is, a +time-zone-aware one as its UTC wall clock, a date-time before 1970, a null, a date, a Decimal and bytes. +""" + +import datetime as dt +import decimal + +import pytest + +UTC9 = dt.timezone(dt.timedelta(hours=9)) +TS = [ + dt.datetime(2020, 1, 2, 3, 4, 5, 123456), + dt.datetime(1969, 12, 31, 23, 59, 59, 999999), + dt.datetime(1900, 1, 1, 0, 0, 0, 1), + dt.datetime(2400, 2, 29, 23, 59, 59), + None, +] +TS_STORED = TS # a naive date-time is the wall clock +AWARE = [ + dt.datetime(2020, 1, 2, 12, 0, 0, 5, tzinfo=UTC9), + dt.datetime(1969, 12, 31, 23, 0, 0, tzinfo=dt.timezone.utc), + dt.datetime(2020, 6, 1, 0, 0, 0, tzinfo=dt.timezone(dt.timedelta(hours=-7))), + None, + dt.datetime(2000, 1, 1, tzinfo=dt.timezone.utc), +] +AWARE_STORED = [ + dt.datetime(2020, 1, 2, 3, 0, 0, 5), + dt.datetime(1969, 12, 31, 23, 0, 0), + dt.datetime(2020, 6, 1, 7, 0, 0), + None, + dt.datetime(2000, 1, 1), +] +DAYS = [ + dt.date(2020, 2, 29), + dt.date(1960, 1, 1), + None, + dt.date(1970, 1, 1), + dt.date(1969, 12, 31), +] +AMOUNTS = [ + decimal.Decimal("12345.6789"), + decimal.Decimal("-0.50"), + decimal.Decimal("1E+3"), + None, + decimal.Decimal("0"), +] +BLOBS = [b"\x00\xff abc\x7f", b"", None, bytearray(b"ab\x80"), b"z"] + + +def _type(db, name): + db.command("sql", f"CREATE DOCUMENT TYPE {name}") + db.command("sql", f"CREATE PROPERTY {name}.id INTEGER") + db.command("sql", f"CREATE PROPERTY {name}.ts DATETIME_MICROS") + db.command("sql", f"CREATE PROPERTY {name}.day DATE") + db.command("sql", f"CREATE PROPERTY {name}.amount DECIMAL") + db.command("sql", f"CREATE PROPERTY {name}.blob BINARY") + + +def _stored(db, name, column): + sql = f"SELECT id, {column} AS v FROM {name} ORDER BY id" # nosec B608 - test-owned names + return [r.get("v") for r in db.query("sql", sql).to_list()] + + +def _bytes(values): + return [None if v is None else bytes(v) for v in values] + + +@pytest.mark.parametrize("path", ["insert_columns", "insert_many"]) +def test_naive_and_aware_datetimes_dates_decimals_and_bytes_are_stored_the_same( + temp_db, path +): + _type(temp_db, "Ev") + n = len(TS) + columns = { + "id": list(range(n)), + "ts": TS, + "day": DAYS, + "amount": AMOUNTS, + "blob": BLOBS, + } + if path == "insert_columns": + assert temp_db.insert_columns("Ev", columns) == n + else: + rows = [dict(zip(columns, vals)) for vals in zip(*columns.values())] + assert temp_db.insert_many("Ev", rows) == n + assert _stored(temp_db, "Ev", "ts") == TS_STORED + assert _stored(temp_db, "Ev", "day") == DAYS + assert _stored(temp_db, "Ev", "amount") == AMOUNTS + assert _bytes(_stored(temp_db, "Ev", "blob")) == _bytes(BLOBS) + + +def test_time_zone_aware_datetimes_are_stored_as_their_utc_wall_clock(temp_db): + _type(temp_db, "Zoned") + n = len(AWARE) + assert temp_db.insert_columns("Zoned", {"id": list(range(n)), "ts": AWARE}) == n + assert _stored(temp_db, "Zoned", "ts") == AWARE_STORED + + +def test_a_parallel_load_of_datetimes_stores_the_same_values(temp_db): + _type(temp_db, "Par") + n = len(TS) + assert ( + temp_db.insert_columns( + "Par", {"id": list(range(n)), "ts": TS, "day": DAYS}, parallel=True + ) + == n + ) + assert _stored(temp_db, "Par", "ts") == TS_STORED + assert _stored(temp_db, "Par", "day") == DAYS diff --git a/tests/test_native_columnar_insert.py b/tests/test_native_columnar_insert.py index 1ec2984c..e7296960 100644 --- a/tests/test_native_columnar_insert.py +++ b/tests/test_native_columnar_insert.py @@ -315,15 +315,18 @@ def test_rows_with_different_keys_fall_back_to_json(temp_db, spy, monkeypatch): _fallback_case(temp_db, spy, monkeypatch, rows, ROWS) -def test_a_datetime_leaves_the_columnar_entry_but_stays_on_the_json_entry( - temp_db, spy, monkeypatch -): - """columnar.pack refuses a date-time, so the chunk takes the JSON entry, which carries it as a tagged value - (test_native_mixed_chunk.py compares what it stores with INSERT ... CONTENT).""" +def test_a_datetime_column_stays_on_the_columnar_entry(temp_db, spy, monkeypatch): + """A column of date-times is the kind T of the columnar entry (#316): the stored value is the one the JSON entry stores as a tagged + value (tests/test_native_columnar_temporal.py has the full comparison, test_native_mixed_chunk.py the rows that stay on JSON). + """ when = datetime.datetime(2024, 5, 6, 7, 8, 9) rows = [{"id": i, "txt": "t%d" % i, "when": when} for i in range(10)] - _fallback_case(temp_db, spy, monkeypatch, rows, ROWS) - assert spy.count(MANY) == 0 + _types(temp_db, "Ref", "Col") + _json_reference(temp_db, "Ref", rows, monkeypatch) + spy.calls.clear() + temp_db.insert_many("Col", rows) + assert spy.count(COLP) == 1 and spy.count(ROWS) == 0 and spy.count(MANY) == 0 + assert _stored(temp_db, "Col") == _stored(temp_db, "Ref") def test_values_json_cannot_carry_still_go_row_by_row_in_one_crossing( @@ -344,9 +347,7 @@ def test_an_integer_past_64_bits_is_refused_on_both_paths(temp_db, spy): temp_db.insert_columns("Col", {"id": np.array([2**63], dtype=np.uint64)}) -def test_a_dataframe_with_a_column_the_buffers_cannot_carry_falls_back( - temp_db, spy, monkeypatch -): +def test_a_dataframe_with_a_timestamp_column_stays_columnar(temp_db, spy, monkeypatch): import pandas as pd import pyarrow as pa @@ -362,7 +363,7 @@ def test_a_dataframe_with_a_column_the_buffers_cannot_carry_falls_back( } ) temp_db.insert_arrow("Col", table) - assert spy.count(COLP) == 0 + assert spy.count(COLP) == 1 and spy.count(ROWS) == 0 assert pd is not None rows = temp_db.query("sql", "SELECT id, txt FROM Col ORDER BY id").to_list() assert [(r["id"], r["txt"]) for r in rows] == [(1, "a"), (2, "b"), (3, "c")] diff --git a/tests/test_native_columnar_pack.py b/tests/test_native_columnar_pack.py index af26f211..92a0eb89 100644 --- a/tests/test_native_columnar_pack.py +++ b/tests/test_native_columnar_pack.py @@ -198,16 +198,107 @@ def test_columns_the_buffer_cannot_carry_unchanged_are_refused(column): assert columnar.pack({"c": column}, NO_TYPES) is None -def test_datetimes_decimals_and_object_arrays_of_them_are_refused(): +def test_numpy_datetime64_arrays_and_mixed_temporal_columns_are_refused(): import datetime import decimal - assert columnar.pack({"d": [datetime.datetime(2020, 1, 1)]}, NO_TYPES) is None - assert columnar.pack({"d": [decimal.Decimal("1.5")]}, NO_TYPES) is None assert ( columnar.pack({"d": np.array(["2020-01-01"], dtype="datetime64[D]")}, NO_TYPES) is None ) + assert ( + columnar.pack( + {"d": [datetime.datetime(2020, 1, 1), datetime.date(2020, 1, 1)]}, NO_TYPES + ) + is None + ) + assert ( + columnar.pack({"d": [decimal.Decimal("1"), decimal.Decimal("NaN")]}, NO_TYPES) + is None + ) + assert columnar.pack({"d": [b"a", "b"]}, NO_TYPES) is None + + +def _ticks(col, dtype): + return np.frombuffer(_bytes(col.data), dtype=dtype).tolist() + + +def test_python_datetimes_are_utc_wall_clock_microseconds(): + import datetime + + utc9 = datetime.timezone(datetime.timedelta(hours=9)) + values = [ + datetime.datetime(1970, 1, 1), + datetime.datetime(1969, 12, 31, 23, 59, 59, 999999), + datetime.datetime(2020, 1, 2, 12, 0, 0, 5, tzinfo=utc9), # 03:00:00.000005 UTC + None, + ] + rows, cols = columnar.pack({"t": values}, NO_TYPES) + col = cols[0] + assert rows == 4 and col.kind == "T" and col.dim == 1_000_000 + assert _ticks(col, np.int64) == [0, -1, 1_577_934_000_000_005, 0] + assert _present(col) == [True, True, True, False] + + +def test_python_dates_are_epoch_days(): + import datetime + + rows, cols = columnar.pack( + {"d": [datetime.date(1970, 1, 2), datetime.date(1969, 12, 31), None]}, NO_TYPES + ) + assert cols[0].kind == "D" and _ticks(cols[0], np.int32) == [1, -1, 0] + assert _present(cols[0]) == [True, True, False] + + +def test_decimals_and_bytes_use_the_string_layout(): + import decimal + + rows, cols = columnar.pack( + { + "a": [decimal.Decimal("1E+3"), None, decimal.Decimal("-0.50")], + "b": [b"\x00\xff", b"", None], + "c": [bytearray(b"xy"), bytearray(b"z"), bytearray(b"")], + }, + NO_TYPES, + ) + a, b, c = cols + assert [x.kind for x in cols] == ["n", "x", "x"] + offsets = np.frombuffer(_bytes(a.data), dtype=np.int32).tolist() + assert offsets == [0, 4, 4, 9] and _bytes(a.aux) == b"1E+3-0.50" + offsets = np.frombuffer(_bytes(b.data), dtype=np.int32).tolist() + assert offsets == [0, 2, 2, 2] and _bytes(b.aux) == b"\x00\xff" + assert _bytes(c.aux) == b"xyz" and _present(a) == [True, False, True] + + +def test_arrow_timestamps_dates_decimals_and_binary(): + import decimal + + import pyarrow as pa + + stamps = pa.array([1, None, -2], pa.int64()).cast( + pa.timestamp("ms", tz="Asia/Seoul") + ) + days = pa.array([3, -4, None], pa.int32()).cast(pa.date32()) + day_ms = pa.array([86_400_000 * 2, -1, None], pa.int64()).cast(pa.date64()) + amounts = pa.array([decimal.Decimal("1.50"), None], pa.decimal128(10, 2)) + blobs = pa.array([b"ab", None, b"c"]) + for name, col, expected in ( + ("t", stamps, ("T", 1_000)), + ("d", days, ("D", 0)), + ("m", day_ms, ("D", 0)), + ): + packed = columnar.pack({name: col}, NO_TYPES)[1][0] + assert (packed.kind, packed.dim) == expected + assert _ticks(columnar.pack({"t": stamps}, NO_TYPES)[1][0], np.int64) == [1, 0, -2] + assert _ticks(columnar.pack({"d": days}, NO_TYPES)[1][0], np.int32) == [3, -4, 0] + # a date64 is floor-divided to days: 2 days, then -1 ms is the day before the epoch + assert _ticks(columnar.pack({"m": day_ms}, NO_TYPES)[1][0], np.int32) == [2, -1, 0] + n = columnar.pack({"n": amounts}, NO_TYPES)[1][0] + assert n.kind == "n" and _bytes(n.aux) == b"1.50" + x = columnar.pack({"x": blobs}, NO_TYPES)[1][0] + assert ( + x.kind == "x" and _bytes(x.aux) == b"abc" and _present(x) == [True, False, True] + ) def test_ragged_columns_and_empty_input_are_left_to_the_row_path(): diff --git a/tests/test_native_columnar_temporal.py b/tests/test_native_columnar_temporal.py new file mode 100644 index 00000000..a97d0b61 --- /dev/null +++ b/tests/test_native_columnar_temporal.py @@ -0,0 +1,322 @@ +"""Native: a column of date-times, dates, Decimals or bytes stays on the columnar entry and stores what the row path stores (#316). + +Before, one such column made ``columnar.pack`` return None, so every column of the batch went through one dict per row and a JSON text +with tagged values. Now the kinds T, D, n and x of ``arcadedb_n_insert_colp`` carry them and the image builds the same objects +(``LocalDateTime``, ``LocalDate``, ``BigDecimal``, ``byte[]``) that ``Json.decodeTag`` builds for the tagged values. Each test loads the +same data through the row path (the private reference: the columnar path switched off, so the chunk goes as tagged JSON) and through +the columns, compares every stored value and its type, and spies on the entry points so a silent fall back cannot pass. Needs the +native library, so conftest.py collects this file on the native backend only. +""" + +import datetime as dt +import decimal + +import pytest + +from arcadedb._native import bulk, columnar, lib # noqa: E402 + +COLP, ROWS, MANY = ( + "arcadedb_n_insert_colp", + "arcadedb_n_insert_rows", + "arcadedb_n_exec_many", +) +UTC9 = dt.timezone(dt.timedelta(hours=9)) + + +class Spy: + def __init__(self, monkeypatch): + self.calls = [] + real = lib.call + + def call(name, *args): + self.calls.append(name) + return real(name, *args) + + monkeypatch.setattr(lib, "call", call) + + def count(self, name): + return self.calls.count(name) + + +@pytest.fixture +def spy(monkeypatch): + return Spy(monkeypatch) + + +def _type(db, name, declared=True): + db.command("sql", f"CREATE DOCUMENT TYPE {name}") + db.command("sql", f"CREATE PROPERTY {name}.id INTEGER") + if declared: + for prop, ty in ( + ("ts", "DATETIME_MICROS"), + ("ns", "DATETIME_NANOS"), + ("day", "DATE"), + ("amount", "DECIMAL"), + ("blob", "BINARY"), + ): + db.command("sql", f"CREATE PROPERTY {name}.{prop} {ty}") + + +def _stored(db, name): + """Every record by id as {property: (python type name, repr)}, so a changed type is as visible as a changed value.""" + sql = f"SELECT FROM {name} ORDER BY id" # nosec B608 - test-owned type name + return [ + { + k: (type(v).__name__, repr(v)) + for k, v in sorted(rec.items()) + if not k.startswith("@") and k != "rid" + } + for rec in db.query("sql", sql).to_list() + ] + + +def _rows(columns): + return [dict(zip(columns, vals)) for vals in zip(*columns.values())] + + +def _reference(db, name, rows, monkeypatch): + with monkeypatch.context() as m: + m.setattr(bulk, "_ENABLED", False) + db.insert_many(name, rows) + + +def _columns(n=60): + base = dt.datetime(1969, 12, 31, 23, 59, 59, 999999) + return { + "id": list(range(n)), + "ts": [ + ( + None + if i % 11 == 0 + else (base + dt.timedelta(seconds=i * 86_399, microseconds=i * 7)) + ) + for i in range(n) + ], + "day": [ + None if i % 13 == 0 else dt.date(1960, 1, 1) + dt.timedelta(days=i * 401) + for i in range(n) + ], + "amount": [ + ( + None + if i % 9 == 0 + else decimal.Decimal(f"{i * 1234567}.{i % 100:02d}") + * (-1 if i % 2 else 1) + ) + for i in range(n) + ], + "blob": [ + None if i % 8 == 0 else bytes(range(i % 256)) + b"\x00\xff" + for i in range(n) + ], + } + + +@pytest.mark.parametrize("declared", [True, False], ids=["declared", "schemaless"]) +def test_a_batch_with_temporal_decimal_and_bytes_columns_stays_columnar( + temp_db, spy, monkeypatch, declared +): + for name in ("Ref", "ByRows", "ByColumns"): + _type(temp_db, name, declared) + columns = _columns() + _reference(temp_db, "Ref", _rows(columns), monkeypatch) + assert ( + spy.count(COLP) == 0 and spy.count(ROWS) == 1 + ) # the reference is the JSON path + spy.calls.clear() + assert temp_db.insert_many("ByRows", _rows(columns)) == 60 + assert spy.count(COLP) == 1 and spy.count(ROWS) == 0 and spy.count(MANY) == 0 + spy.calls.clear() + assert temp_db.insert_columns("ByColumns", columns) == 60 + assert spy.count(COLP) == 1 and spy.count(ROWS) == 0 + expected = _stored(temp_db, "Ref") + assert len(expected) == 60 and expected[1]["ts"][0] == "datetime" + assert _stored(temp_db, "ByRows") == expected + assert _stored(temp_db, "ByColumns") == expected + + +def test_time_zone_aware_and_naive_datetimes_follow_the_json_rule( + temp_db, spy, monkeypatch +): + """An aware date-time is its UTC wall clock, a naive one is taken as it is (_tag_datetime): the column does not decide a zone.""" + _type(temp_db, "Ref") + _type(temp_db, "Col") + values = [ + dt.datetime(2020, 1, 2, 12, 0, 0, 5, tzinfo=UTC9), + dt.datetime(1969, 12, 31, 23, 0, 0, tzinfo=dt.timezone.utc), + dt.datetime(2020, 6, 1, 0, 0, 0, tzinfo=dt.timezone(dt.timedelta(hours=-7))), + ] + rows = [{"id": i, "ts": v} for i, v in enumerate(values)] + _reference(temp_db, "Ref", rows, monkeypatch) + spy.calls.clear() + temp_db.insert_many("Col", rows) + assert spy.count(COLP) == 1 + assert _stored(temp_db, "Col") == _stored(temp_db, "Ref") + + +def test_pandas_timestamps_keep_microseconds_as_the_json_path_does( + temp_db, spy, monkeypatch +): + import pandas as pd + + _type(temp_db, "Ref") + _type(temp_db, "Col") + stamps = [ + pd.Timestamp("2026-05-06 07:08:09.123456789"), + pd.Timestamp("1960-05-06 07:08:09.123456", tz="Asia/Seoul"), + ] + rows = [{"id": i, "ts": v} for i, v in enumerate(stamps)] + _reference(temp_db, "Ref", rows, monkeypatch) + spy.calls.clear() + temp_db.insert_many("Col", rows) + assert spy.count(COLP) == 1 and spy.count(ROWS) == 0 + assert _stored(temp_db, "Col") == _stored(temp_db, "Ref") + + +@pytest.mark.parametrize( + "column", + [ + [dt.datetime(2020, 1, 1), dt.date(2020, 1, 1)], # a date-time and a date + [decimal.Decimal("1.5"), decimal.Decimal("NaN")], # a non-finite Decimal + [b"a", "b"], # bytes and text + ], + ids=["datetime-and-date", "nan-decimal", "bytes-and-text"], +) +def test_a_column_the_buffers_cannot_carry_unchanged_still_falls_back( + temp_db, spy, monkeypatch, column +): + _type(temp_db, "Ref", declared=False) + _type(temp_db, "Col", declared=False) + rows = [{"id": i, "x": v} for i, v in enumerate(column)] + try: + _reference(temp_db, "Ref", rows, monkeypatch) + except ( + Exception + ): # the statement path may refuse it; then the new path must refuse it the same way + with pytest.raises(Exception): + temp_db.insert_many("Col", rows) + assert spy.count(COLP) == 0 + return + spy.calls.clear() + temp_db.insert_many("Col", rows) + assert spy.count(COLP) == 0 + assert _stored(temp_db, "Col") == _stored(temp_db, "Ref") + + +def _arrow_table(pa): + import numpy as np + + n = 40 + ids = np.arange(n, dtype=np.int64) + epoch_us = (ids - 20) * 1_000_003_000 + 7 # microseconds, before and after 1970 + mask = ids % 6 == 0 + + def stamps(unit, tz=None): + raw = { + "s": epoch_us // 1_000_000, + "ms": epoch_us // 1_000, + "us": epoch_us, + "ns": epoch_us * 1_000, + }[unit] + return pa.array(raw.astype(np.int64), mask=mask).cast(pa.timestamp(unit, tz=tz)) + + return pa.table( + { + "id": ids, + "ts": stamps("us"), + "ns": stamps("ns"), + "sec": stamps("s"), + "zoned": stamps("ms", tz="Asia/Seoul"), + "day": pa.array( + ((ids - 10) * 397).astype(np.int32), mask=ids % 5 == 0 + ).cast(pa.date32()), + "amount": pa.array( + [ + None if i % 7 == 0 else decimal.Decimal(f"{i}.{i % 100:02d}") + for i in range(n) + ], + type=pa.decimal128(20, 2), + ), + "blob": pa.array( + [None if i % 4 == 0 else bytes(range(i)) for i in range(n)] + ), + } + ) + + +def test_arrow_timestamps_dates_decimals_and_binary_stay_columnar( + temp_db, spy, monkeypatch +): + import pyarrow as pa + + for name in ("Ref", "Col"): + _type(temp_db, name) + for prop, ty in (("sec", "DATETIME_SECOND"), ("zoned", "DATETIME")): + temp_db.command("sql", f"CREATE PROPERTY {name}.{prop} {ty}") + table = _arrow_table(pa) + with ( + monkeypatch.context() as m + ): # the reference: tagged values through the JSON path + m.setattr(bulk, "_ENABLED", False) + assert temp_db.insert_arrow("Ref", table) == 40 + assert spy.count(COLP) == 0 + spy.calls.clear() + assert temp_db.insert_arrow("Col", table) == 40 + assert spy.count(COLP) == 1 and spy.count(ROWS) == 0 and spy.count(MANY) == 0 + expected = _stored(temp_db, "Ref") + assert len(expected) == 40 and expected[1]["ns"][0] == "datetime" + assert _stored(temp_db, "Col") == expected + + +def test_an_arrow_batch_with_one_list_column_falls_back_with_the_same_values( + temp_db, spy, monkeypatch +): + """A list column cannot cross as a buffer: the whole batch takes the row path, and the temporal columns reach it as tagged values.""" + import pyarrow as pa + + for name in ("Ref", "Col"): + _type(temp_db, name) + for prop, ty in (("sec", "DATETIME_SECOND"), ("zoned", "DATETIME")): + temp_db.command("sql", f"CREATE PROPERTY {name}.{prop} {ty}") + table = _arrow_table(pa).append_column( + "tags", pa.array([["a", "b"]] * 40, type=pa.list_(pa.string())) + ) + with monkeypatch.context() as m: + m.setattr(bulk, "_ENABLED", False) + temp_db.insert_arrow("Ref", table) + spy.calls.clear() + temp_db.insert_arrow("Col", table) + assert spy.count(COLP) == 0 + assert _stored(temp_db, "Col") == _stored(temp_db, "Ref") + + +def test_a_parallel_load_of_datetimes_is_one_columnar_crossing(temp_db, spy): + _type(temp_db, "Par") + columns = _columns(300) + assert temp_db.insert_columns("Par", columns, parallel=True) == 300 + assert spy.count(COLP) == 1 and spy.count(ROWS) == 0 + assert temp_db.query("sql", "SELECT count(*) AS n FROM Par").first().get("n") == 300 + + +def test_the_image_refuses_a_wrong_ticks_per_second_and_writes_nothing(temp_db): + import json + + _type(temp_db, "Col") + rows, packed = columnar.pack( + {"id": [1, 2], "ts": [dt.datetime(2020, 1, 1)] * 2}, columnar.Declared(known={}) + ) + assert packed[1].kind == "T" and packed[1].dim == 1_000_000 + packed[1].dim = 7 + spec = columnar.spec_of(packed) + reply = json.loads( + lib.call( + COLP, + temp_db._h, + lib.b("Col"), + lib.b(columnar.header_of(rows, 10_000, packed)), + spec, + len(spec), + ) + ) + assert not reply["ok"] and "ticks per second" in reply["error"] + assert temp_db.query("sql", "SELECT count(*) AS n FROM Col").first().get("n") == 0 diff --git a/tests/test_native_mixed_chunk.py b/tests/test_native_mixed_chunk.py index db4162cf..0c746bdf 100644 --- a/tests/test_native_mixed_chunk.py +++ b/tests/test_native_mixed_chunk.py @@ -169,7 +169,8 @@ def test_a_pandas_timestamp_is_stored_as_the_statement_path_stores_it( _statement_path(db, "Ref", rows) spy.clear() db.insert_many("Bulk", rows) - assert spy.count(MANY) == 0 and spy.count(ROWS) == 1 + # one column of Timestamps is the kind T of the columnar entry (#316): microseconds, as the tagged path keeps them + assert spy.count(MANY) == 0 and spy.count(ROWS) == 0 and spy.count(COLP) == 1 assert _stored(db, "Bulk") == _stored(db, "Ref") From ca91fd836334aa587e5fb4c6feb1ad39dbbd7327 Mon Sep 17 00:00:00 2001 From: Taewoon Kim Date: Mon, 9 Nov 2026 21:25:41 +0900 Subject: [PATCH 24/49] Temporal columnar kinds: refuse out-of-range values in Python, before the crossing (review of #333) A date64 past int32 days wrapped in the int32 cast (the row path raised) and a timestamp[s] past the years of java.time threw DateTimeException in the middle of a batch (BulkOps T kind, LocalDateTime.ofEpochSecond). Both now make pack() return None, so the call takes the row path and raises what it raised before, with nothing stored. Only a timestamp[s] can pass the java.time range: an int64 of milliseconds is at most 9.2e15 seconds against 3.15e16. Tests: the pack-level limits, the sync call (nothing stored, no transaction left open), the parallel call (may have been stored), and a date-time chunk with NaN stores the NaN that INSERT ... CONTENT :row stores (both end in MutableDocument.putProperty). Co-Authored-By: Claude Sonnet 5.5 --- src/arcadedb/_native/columnar.py | 11 ++ tests/conftest.py | 2 + ...t_native_columnar_temporal_review_fixes.py | 176 ++++++++++++++++++ 3 files changed, 189 insertions(+) create mode 100644 tests/test_native_columnar_temporal_review_fixes.py diff --git a/src/arcadedb/_native/columnar.py b/src/arcadedb/_native/columnar.py index 46537e38..d001b3da 100644 --- a/src/arcadedb/_native/columnar.py +++ b/src/arcadedb/_native/columnar.py @@ -43,6 +43,10 @@ "snx" # the kinds laid out as int32 offsets[n + 1] plus the bytes they index ) _TICKS = {"s": 1, "ms": 1_000, "us": 1_000_000, "ns": 1_000_000_000} +# the seconds that java.time.LocalDateTime.ofEpochSecond takes (years -999999999 to 999999999): beyond them it throws in the image, in the +# middle of a batch. Only a timestamp[s] can pass them: an int64 of milliseconds is at most 9.2e15 seconds +_LDT_MIN_SECONDS = -31_557_014_167_219_200 +_LDT_MAX_SECONDS = 31_556_889_864_403_199 class Packed: @@ -349,6 +353,9 @@ def _arrow_column(name, arr, declared): ) if types.is_timestamp(t): values = arr.cast(pa.int64()).fill_null(0).to_numpy(zero_copy_only=False) + if t.unit == "s" and values.size: + if values.min() < _LDT_MIN_SECONDS or values.max() > _LDT_MAX_SECONDS: + return None # the row path raises its usual error for it, before anything is stored return Packed( name, "T", @@ -360,6 +367,10 @@ def _arrow_column(name, arr, declared): if types.is_date64(t): # milliseconds since the epoch, floor-divided to days raw = arr.cast(pa.int64()).fill_null(0).to_numpy(zero_copy_only=False) days = raw // 86_400_000 + if days.size and ( + days.min() < -(_INT32_MAX + 1) or days.max() > _INT32_MAX + ): + return None # int32 days would wrap: the row path raises its usual error for it else: days = arr.cast(pa.int32()).fill_null(0).to_numpy(zero_copy_only=False) return Packed( diff --git a/tests/conftest.py b/tests/conftest.py index dc34241e..d7d95ce1 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -53,6 +53,8 @@ collect_ignore.append("test_native_columnar_parallel_review_fixes.py") # a column of date-times, dates, Decimals or bytes crosses as a column (kinds T D n x), observed on the native library collect_ignore.append("test_native_columnar_temporal.py") + # the review of the temporal kinds: a value the image cannot take is refused in Python, never wrapped or thrown mid-batch + collect_ignore.append("test_native_columnar_temporal_review_fixes.py") # the rows of an insert_many chunk with date-times, Decimals, bytes and numpy scalars are told apart by the entry points they cross collect_ignore.append("test_native_mixed_chunk.py") # the native build's ingest defaults (insert_many and graph_batch commit size), observed on the native library diff --git a/tests/test_native_columnar_temporal_review_fixes.py b/tests/test_native_columnar_temporal_review_fixes.py new file mode 100644 index 00000000..31c8430e --- /dev/null +++ b/tests/test_native_columnar_temporal_review_fixes.py @@ -0,0 +1,176 @@ +"""Review of the temporal columnar kinds (#333): a value the image cannot take is refused in Python, before the crossing, and never wraps. + +``T`` and ``D`` columns carry integers that the image turns into ``LocalDateTime`` and ``LocalDate``. A ``date64`` past int32 days used +to wrap in the int32 cast and a ``timestamp[s]`` past the years of ``java.time`` threw ``DateTimeException`` in the middle of a batch. Now +``columnar.pack`` returns None for both, so the call takes the row path, which raises what it raised before (nothing stored, no +transaction left open on the synchronous call; ``may have been stored`` for a parallel call, which has handed rows over by then). Also +pinned: a chunk with a date-time column and a float column holding NaN goes columnar now and stores the same NaN that ``INSERT INTO t +CONTENT :row`` stores (both end in ``MutableDocument.putProperty``). Needs the native library: conftest.py collects this file on the +native backend only. +""" + +import datetime as dt +import math + +import pytest + +from arcadedb._native import columnar, lib # noqa: E402 +from arcadedb.exceptions import ArcadeDBError + +pa = pytest.importorskip("pyarrow") + +COLP = "arcadedb_n_insert_colp" +DAY_MS = 86_400_000 +LDT_MIN_SECONDS, LDT_MAX_SECONDS = -31_557_014_167_219_200, 31_556_889_864_403_199 +NO_TYPES = columnar.Declared(known={}) + + +class Spy: + def __init__(self, monkeypatch): + self.calls = [] + real = lib.call + + def call(name, *args): + self.calls.append(name) + return real(name, *args) + + monkeypatch.setattr(lib, "call", call) + + +def _count(db, name): + sql = f"SELECT count(*) AS n FROM {name}" # nosec B608 - test-owned type name + return db.query("sql", sql).first().get("n") + + +def _kinds(columns): + packed = columnar.pack(columns, NO_TYPES) + return None if packed is None else [c.kind for c in packed[1]] + + +def test_a_date64_past_int32_days_is_not_laid_out_and_never_wraps(): + wraps = pa.array([DAY_MS * (2**31 + 5)], pa.int64()).cast(pa.date64()) + assert columnar.pack({"d": wraps}, NO_TYPES) is None + below = pa.array([DAY_MS * (-(2**31) - 1)], pa.int64()).cast(pa.date64()) + assert columnar.pack({"d": below}, NO_TYPES) is None + one_bad = pa.array([0, DAY_MS, DAY_MS * 2**40], pa.int64()).cast(pa.date64()) + assert columnar.pack({"d": one_bad}, NO_TYPES) is None # one value is enough + + +def test_a_date64_at_the_int32_limits_stays_columnar_and_exact(): + edge = pa.array( + [DAY_MS * (2**31 - 1), DAY_MS * -(2**31), None], pa.int64() + ).cast(pa.date64()) + rows, packed = columnar.pack({"d": edge}, NO_TYPES) + assert rows == 3 and packed[0].kind == "D" + import ctypes + + import numpy as np + + addr, length = packed[0].data + days = np.frombuffer(ctypes.string_at(addr, length), dtype=np.int32) + assert days.tolist() == [2**31 - 1, -(2**31), 0] + + +def test_a_timestamp_past_the_years_of_java_time_is_not_laid_out(): + assert ( + columnar.pack({"t": pa.array([0, 2**62], pa.timestamp("s"))}, NO_TYPES) is None + ) + over = pa.array([LDT_MAX_SECONDS + 1], pa.timestamp("s")) + assert columnar.pack({"t": over}, NO_TYPES) is None + under = pa.array([LDT_MIN_SECONDS - 1], pa.timestamp("s")) + assert columnar.pack({"t": under}, NO_TYPES) is None + + +def test_a_timestamp_at_the_limits_stays_columnar(): + edge = pa.array([LDT_MAX_SECONDS, LDT_MIN_SECONDS, None], pa.timestamp("s")) + assert _kinds({"t": edge}) == ["T"] + # an int64 of milliseconds, microseconds or nanoseconds is always inside the years of java.time + for unit in ("ms", "us", "ns"): + big = pa.array([2**63 - 1, -(2**63)], pa.timestamp(unit)) + assert _kinds({"t": big}) == ["T"], unit + + +def _type(db, name): + db.command("sql", f"CREATE DOCUMENT TYPE {name}") + db.command("sql", f"CREATE PROPERTY {name}.id INTEGER") + + +def test_a_timestamp_the_image_cannot_take_stores_nothing_and_leaves_no_transaction( + temp_db, +): + _type(temp_db, "Col") + table = pa.table( + { + "id": pa.array([1, 2], pa.int64()), + "t": pa.array([0, 2**62], pa.timestamp("s")), + } + ) + with pytest.raises(ArcadeDBError): + temp_db.insert_arrow("Col", table) + assert not temp_db.is_transaction_active() + assert _count(temp_db, "Col") == 0 + + +def test_a_date_the_image_cannot_take_stores_nothing_and_leaves_no_transaction(temp_db): + _type(temp_db, "Col") + wraps = pa.array([0, DAY_MS * (2**31 + 5)], pa.int64()).cast(pa.date64()) + table = pa.table({"id": pa.array([1, 2], pa.int64()), "d": wraps}) + with pytest.raises(ArcadeDBError): + temp_db.insert_arrow("Col", table) + assert not temp_db.is_transaction_active() + assert _count(temp_db, "Col") == 0 + + +def test_a_parallel_load_of_such_a_timestamp_says_that_rows_may_have_been_stored( + temp_db, +): + _type(temp_db, "Col") + table = pa.table( + { + "id": pa.array([1, 2], pa.int64()), + "t": pa.array([0, 2**62], pa.timestamp("s")), + } + ) + with pytest.raises(ArcadeDBError, match="may have been stored"): + temp_db.insert_arrow("Col", table, parallel=True) + temp_db.async_executor().wait_completion() + assert ( + _count(temp_db, "Col") <= 1 + ) # the row before the bad one may have been handed over + + +def test_a_datetime_chunk_with_nan_stores_the_nan_the_statement_path_stores( + temp_db, monkeypatch +): + for name in ("Col", "Ref"): + _type(temp_db, name) + temp_db.command("sql", f"CREATE PROPERTY {name}.ts DATETIME_MICROS") + temp_db.command("sql", f"CREATE PROPERTY {name}.val DOUBLE") + rows = [ + { + "id": i, + "ts": dt.datetime(2020, 1, 1, 0, 0, i), + "val": float("nan") if i % 2 else 1.5 * i, + } + for i in range(10) + ] + spy = Spy(monkeypatch) + temp_db.insert_many("Col", rows) + assert ( + spy.calls.count(COLP) == 1 + ) # the date-time column no longer forces the JSON path + sql = "INSERT INTO `Ref` CONTENT :row" + temp_db.begin() + temp_db._command_many("sql", sql, [{"row": row} for row in rows]) + temp_db.commit() + + def stored(name): + select = f"SELECT FROM {name} ORDER BY id" # nosec B608 - test-owned type name + return [ + (r["id"], type(r["val"]).__name__, repr(r["val"]), repr(r["ts"])) + for r in temp_db.query("sql", select).to_list() + ] + + got, want = stored("Col"), stored("Ref") + assert got == want + assert all(math.isnan(float(v[2])) for v in got if v[0] % 2) From 9da36d634ccd025c43aa65607171021306efdf87 Mon Sep 17 00:00:00 2001 From: Taewoon Kim Date: Mon, 9 Nov 2026 22:22:43 +0900 Subject: [PATCH 25/49] Native spy test: count the codec hooks around create_vertices, not around the whole batch (refs #337) codec._prep is counted by name, so it also counted the calls the batch makes outside create_vertices: creating the batch (extras._enc of its options runs the original _prep, whose recursion goes through the patched name, once per option) and close() (encode_gb_args({}) once). That is 2 calls, matching the Free leg; they do not depend on the rows. The create_vertices path itself calls neither _prep nor a hook (take_vector_columns, _plain_rows and one json.dumps). The first test now takes the hook counts and the crossings around the create_vertices call (gb_default and default exactly 0, prep at most one per crossing), and a new test runs 250 rows in three chunks and 2,500 rows in three chunks with the same bounds: the old path called the hooks per row. Co-Authored-By: Claude Sonnet 5.5 --- tests/test_native_vertex_vector_crossings.py | 33 ++++++++++++++++++-- 1 file changed, 31 insertions(+), 2 deletions(-) diff --git a/tests/test_native_vertex_vector_crossings.py b/tests/test_native_vertex_vector_crossings.py index 8523df95..e6848451 100644 --- a/tests/test_native_vertex_vector_crossings.py +++ b/tests/test_native_vertex_vector_crossings.py @@ -59,6 +59,19 @@ def wrapper(*args, **kwargs): def entries(self): return [c[0] for c in self.crossings] + def around(self, call): + """``(result, hooks, crossings)`` of one ``call()``: the codec hook calls and the library crossings made inside it only. + + ``codec._prep`` is counted by name, so it also counts the calls the batch itself makes outside ``create_vertices``: creating + the batch (``_enc`` of its options, whose ``_prep`` recurses into the patched name once per option) and ``close()`` + (``encode_gb_args({})``). Those are a few calls and do not depend on the rows, so the counts here are taken around the + ``create_vertices`` call alone. + """ + before, seen = dict(self.hooks), len(self.crossings) + result = call() + used = {key: self.hooks[key] - before[key] for key in before} + return result, used, len(self.crossings) - seen + @pytest.fixture def spy(monkeypatch): @@ -95,9 +108,12 @@ def test_a_float32_column_is_one_matrix_with_no_hook_call_per_row(db, spy, type_ x = _x() rows = [{"id": i, "embedding": x[i]} for i in range(ROWS)] with db.graph_batch(commit_every=CHUNK) as gb: - assert len(gb.create_vertices(type_name, rows)) == ROWS + rids, used, crossed = spy.around(lambda: gb.create_vertices(type_name, rows)) + assert len(rids) == ROWS assert spy.entries() == ["arcadedb_n_gb_bin"] * 3 # one crossing per chunk of 100 - assert spy.hooks == {"gb_default": 0, "default": 0, "prep": 0}, spy.hooks + # no per-row hook: the per-row hooks of the old path are never called, and nothing is called more than once per crossing + assert used["gb_default"] == 0 and used["default"] == 0, used + assert used["prep"] <= crossed, used # the old path called it for every row assert [c[3] for c in spy.crossings] == [CHUNK * DIM, CHUNK * DIM, 50 * DIM] for _, text, buf, count in spy.crossings: args = json.loads(text) @@ -110,6 +126,19 @@ def test_a_float32_column_is_one_matrix_with_no_hook_call_per_row(db, spy, type_ _assert_bit_equal(_stored(db, type_name), x) +def test_the_python_work_does_not_grow_with_the_row_count(db, spy): + # 250 rows in three chunks of 100, then 2,500 rows in three chunks of 1,000: ten times the rows, the same number of crossings, and + # the same bound on the codec hooks (the old path called the hooks once or more for every row, so it would show 250 and 2,500) + for rows_n, chunk in ((250, 100), (2500, 1000)): + x = _x(rows_n) + rows = [{"id": i, "embedding": x[i]} for i in range(rows_n)] + with db.graph_batch(commit_every=chunk) as gb: + rids, used, crossed = spy.around(lambda: gb.create_vertices("Free", rows)) + assert len(rids) == rows_n and crossed == 3 + assert used["gb_default"] == 0 and used["default"] == 0, used + assert used["prep"] <= crossed, used + + def test_two_columns_and_a_date_time_keep_the_buffer_layout(db, spy): x, y = _x(), _x(dim=8, seed=8) when = datetime(2020, 1, 2, 3, 4, 5) From cce199f6ba0b28911344390689ba5ac8f453e0eb Mon Sep 17 00:00:00 2001 From: Taewoon Kim Date: Mon, 9 Nov 2026 22:22:59 +0900 Subject: [PATCH 26/49] Test: a statement error has the timing, class and message of the lazy form (#317) A CI run showed that an INSERT or UPDATE error is raised by the call in both forms, not when the row is read. The test pins that the two forms agree, and a SELECT run as a command pins the read-time case for both. The helper's docstring states the timing. Co-Authored-By: Claude Sonnet 5.5 --- src/arcadedb/_native/database.py | 5 ++-- tests/test_native_one_row_crossings.py | 37 +++++++++++++++++++------- 2 files changed, 30 insertions(+), 12 deletions(-) diff --git a/src/arcadedb/_native/database.py b/src/arcadedb/_native/database.py index 8016245e..4c5d4b3b 100644 --- a/src/arcadedb/_native/database.py +++ b/src/arcadedb/_native/database.py @@ -686,8 +686,9 @@ def command(self, language: str, command: str, *args) -> NativeResultSet: def _command_first_row(self, language: str, command: str, *args) -> NativeResultSet: """:meth:`command` for a caller that reads the first row right away (a create or an update that returns its row): the statement runs and its first rows come back in the same crossing (``arcadedb_n_exec`` without the lazy bit) instead of a cursor id that a second - crossing (``arcadedb_n_next_bin``) reads. The rows, the values and the errors are those of :meth:`command`; an error met while - the first rows are computed is raised when they are read, as on the lazy path. + crossing (``arcadedb_n_next_bin``) reads. The rows, the values and the errors are those of :meth:`command`, at the same moment: + an error the engine raises while it runs the statement (an INSERT or an UPDATE) is raised by this call, as by :meth:`command`; + one met while rows are computed lazily comes in the reply (``rerr``) and is raised when the row is read, as on the lazy path. """ return self._exec(1 | _EAGER, "Command failed", language, command, args) diff --git a/tests/test_native_one_row_crossings.py b/tests/test_native_one_row_crossings.py index 8f768565..73f32a92 100644 --- a/tests/test_native_one_row_crossings.py +++ b/tests/test_native_one_row_crossings.py @@ -302,17 +302,19 @@ def _raises_at(call): return None, None -FIRST_ROW_ERRORS = { +STATEMENT_ERRORS = { "insert": ("INSERT INTO D SET v = 1 / (1 - 1)", False), "update": ("UPDATE :rid SET v = 1 / (1 - 1)", True), } -@pytest.mark.parametrize("case", list(FIRST_ROW_ERRORS)) -def test_an_error_met_while_the_first_row_is_computed_is_raised_when_it_is_read( - db, case -): - sql, with_rid = FIRST_ROW_ERRORS[case] +@pytest.mark.parametrize("case", list(STATEMENT_ERRORS)) +def test_a_statement_error_has_the_timing_class_and_message_of_the_lazy_form(db, case): + """An INSERT or UPDATE is run by the engine while the statement is called, so both forms raise from the call (a CI run showed + it: the first version of this test expected the error when the row is read, and the call raised). What a caller of save() sees + is the same either way: one ArcadeDBError with one message. What is pinned is that the two forms agree on all three. + """ + sql, with_rid = STATEMENT_ERRORS[case] rid = _saved(db, v=1) params = ({"rid": records.RID(rid)},) if with_rid else () with _rolled_back(db): @@ -321,14 +323,29 @@ def test_an_error_met_while_the_first_row_is_computed_is_raised_when_it_is_read( eager_phase, eager = _raises_at( lambda: db._command_first_row("sql", sql, *params) ) - assert ( - eager_phase == lazy_phase == "read" - ) # the engine computes the row when it is pulled: the reply carries it as rerr - assert str(eager).startswith("Reading the result set failed: ") + assert lazy_phase is not None and eager_phase == lazy_phase assert str(eager) == str(lazy) and type(eager) is type(lazy) assert eager.native.get("cls") == lazy.native.get("cls") +def test_a_row_error_is_raised_when_the_row_is_read_by_both_forms(db): + """A statement whose rows the engine computes lazily (a SELECT run as a command) fails when its first row is read: the public + ``command`` as before (the cursor's read), and the eager form (the error travels in the reply as ``rerr`` and is raised by the + read of the row it replaces). The call itself returns.""" + _saved(db, i=1) + sql = "SELECT 1 / (i - i) AS f FROM D" + with _rolled_back(db): + lazy = db.command("sql", sql) + with pytest.raises(ArcadeDBError) as lazy_error: + lazy.first() + eager = db._command_first_row("sql", sql) + with pytest.raises(ArcadeDBError) as eager_error: + eager.first() + assert str(lazy_error.value).startswith("Reading the result set failed: ") + assert str(eager_error.value) == str(lazy_error.value) + assert type(eager_error.value) is type(lazy_error.value) + + def test_a_before_create_callback_that_raises_fails_save_as_the_lazy_form_does(db): def explode(record): raise RuntimeError("no creating today") From 17ccea5b3ba3050e7a1cbc92546962bd8aba2db0 Mon Sep 17 00:00:00 2001 From: Taewoon Kim Date: Mon, 9 Nov 2026 22:23:03 +0900 Subject: [PATCH 27/49] Native frames: vector types unify to float64, a null vector is None in to_dataframe A dimension that changes inside a frame makes a json column (list), so the same change between two frames now gives list too (it gave list); float32 next to float64 of one size is fixed_size_list. In to_dataframe a vector cell is a numpy float32 array or None for a null, on both wheels (a NaN row, which to_columns keeps in its matrix, read as a vector of NaN where the old route had a missing value); to_arrow and to_polars already kept the null. Co-Authored-By: Claude Sonnet 5.5 --- src/arcadedb/_columnar.py | 16 +++-- src/arcadedb/_native/database.py | 9 +-- src/arcadedb/results.py | 15 ++--- tests/test_column_frames.py | 65 +++++++++++++++++++- tests/test_native_column_frames_reference.py | 5 +- 5 files changed, 87 insertions(+), 23 deletions(-) diff --git a/src/arcadedb/_columnar.py b/src/arcadedb/_columnar.py index 16c66d81..75dd8f5e 100644 --- a/src/arcadedb/_columnar.py +++ b/src/arcadedb/_columnar.py @@ -170,14 +170,15 @@ def _unify_arrow_chunk_types(arrs: list, pa) -> list: # (the widening is exact), never text values = {t.value_type for t in types} if values <= {pa.float32(), pa.float64()}: - value = pa.float64() if pa.float64() in values else pa.float32() + # always float64: the widening is exact, and a frame whose rows differ in length is a json column (list) on its + # own, so float32 here would make the type depend on whether the change falls inside a frame or between two sizes = { t.list_size if pa.types.is_fixed_size_list(t) else None for t in types } target = ( - pa.list_(value, next(iter(sizes))) + pa.list_(pa.float64(), next(iter(sizes))) if len(sizes) == 1 and None not in sizes - else pa.list_(value) + else pa.list_(pa.float64()) ) try: return [pa.array(a.to_pylist(), type=target) for a in arrs] @@ -238,7 +239,10 @@ def decode_columns_frame( ) -> Tuple[int, Dict[str, Any], Dict[str, Any]]: """``(count, parts, header)`` of one frame as ``to_columns`` columns: ``parts`` maps a column name to a numpy array (int64, float64, bool, datetime64) or a Python list (text, JSON-typed values, a bool column with nulls); a count of 0 has - no parts. With ``micros`` a naive datetime column is ``datetime64[us]`` (``dtu``). + no parts. A vector column is a (count, dim) matrix, a null row of it NaN. + + ``micros`` is the ``to_dataframe`` form: a naive datetime column is ``datetime64[us]`` (``dtu``), and a vector column is a + list with one numpy array per cell and None for a null (a NaN row would read as a vector of NaN, not as a missing value). """ import numpy as np @@ -291,6 +295,10 @@ def decode_columns_frame( values = np.empty((count, 0), dtype=dt) else: values = np.frombuffer(data, dtype=dt).reshape(count, dim).copy() + if micros: + values = [ + None if has_nulls and mask[i] else values[i] for i in range(count) + ] elif ctype == "json": values = json_values(data, side, count) elif ctype == "dec": diff --git a/src/arcadedb/_native/database.py b/src/arcadedb/_native/database.py index 792b82ce..4cd943d3 100644 --- a/src/arcadedb/_native/database.py +++ b/src/arcadedb/_native/database.py @@ -470,15 +470,11 @@ def to_dataframe(self, convert_types: bool = True): with two exceptions that are the same on both builds. A DATETIME keeps its microseconds (a ``datetime64[us]`` column, where ``to_columns()`` and ``to_arrow()`` carry milliseconds), a DATE is a ``datetime.date`` in an object column, and a ``Decimal`` stays one. A column of ``float[]`` vectors holds one numpy ``float32`` array per cell (a row of the matrix ``to_columns()`` - returns, not a Python list per row); ``to_list()`` gives lists of floats. + returns, not a Python list per row) and None for a null; ``to_list()`` gives lists of floats. """ import pandas as pd columns = self._columns(25_000, None, True) - # pandas rejects 2-D ndarrays as column values; vector columns become object columns of row slices - columns = { - k: (list(v) if getattr(v, "ndim", 1) > 1 else v) for k, v in columns.items() - } return pd.DataFrame(columns) def _columns(self, batch_size, columns, micros): @@ -498,7 +494,8 @@ def to_columns(self, batch_size: int = 25_000, columns=None): """Columns as the JVM build returns them (numpy int64/float64/bool/datetime64[ms] arrays, lists for text and JSON-typed values, an object array of Decimal), decoded from column frames by the decoder both builds share. The arrays are writable, and a column's type belongs to the whole result, not to where a batch ends. A column is a float32 matrix when the property is - a ``float[]`` (ARRAY_OF_FLOATS), and a list of lists when it is a LIST of numbers whatever the values. An empty result is + a ``float[]`` (ARRAY_OF_FLOATS), and a list of lists when it is a LIST of numbers whatever the values; a null vector is a row of + NaN in the matrix (``to_dataframe()`` and ``to_arrow()`` keep it a null). An empty result is ``{}``, or the ``columns`` asked for as empty lists. None when numpy is missing. """ try: diff --git a/src/arcadedb/results.py b/src/arcadedb/results.py index 2abc95dc..369c1dfa 100644 --- a/src/arcadedb/results.py +++ b/src/arcadedb/results.py @@ -555,8 +555,8 @@ def to_dataframe(self, convert_types: bool = True): carry milliseconds), a DATE is a ``datetime.date`` in an object column, and a ``Decimal`` stays one. A column of ``float[]`` vectors holds one numpy ``float32`` array per cell (a row of the matrix ``to_columns()`` - returns, no copy into a Python list per row); ``to_list()`` gives - lists of floats. + returns, no copy into a Python list per row) and None for a null; + ``to_list()`` gives lists of floats. Args: convert_types: Convert Java types to Python (default: True) @@ -588,12 +588,7 @@ def to_dataframe(self, convert_types: bool = True): # to_arrow() keep the documented milliseconds. columns = self._read_columns(25_000, None, micros=True) if columns is not None: - # pandas rejects 2-D ndarrays as column values; vector - # columns (f4v/f8v) become object columns of row slices - columns = { - k: (list(v) if getattr(v, "ndim", 1) > 1 else v) - for k, v in columns.items() - } + # a vector column is already one numpy array per cell, None for a null return pd.DataFrame(columns) return pd.DataFrame(self.to_list(convert_types=convert_types)) @@ -657,7 +652,9 @@ def to_columns( ``Decimal`` values (#115). The type of a column belongs to the whole result, not to where a batch ends: a batch that lacks the column or holds only nulls for it takes the type of the batches that have - values. The arrays are writable. An empty result is ``{}``, or, when + values. The arrays are writable. A vector column is a ``(count, dim)`` matrix + and a null vector is a row of NaN in it (``to_dataframe()`` and ``to_arrow()`` keep + it a null). An empty result is ``{}``, or, when ``columns`` is given, those columns as empty lists. The columns are the union of every row's property names, in order of diff --git a/tests/test_column_frames.py b/tests/test_column_frames.py index dc06f830..177c2d79 100644 --- a/tests/test_column_frames.py +++ b/tests/test_column_frames.py @@ -529,5 +529,68 @@ def test_vectors_whose_dimension_changes_between_frames_are_one_list_column(temp [{"i": i, "v": [0.5, 1.0, 2.0] if i < 300 else [0.5, 1.0]} for i in range(600)], ) t = db.query("sql", "SELECT i, v FROM Dim ORDER BY i").to_arrow(batch_size=100) - assert t.schema.field("v").type == pa.list_(pa.float32()) + assert t.schema.field("v").type == pa.list_( + pa.float64() + ) # a lossless widening, wherever the dimension changes assert t.column("v").to_pylist() == [[0.5, 1.0, 2.0]] * 300 + [[0.5, 1.0]] * 300 + + +def test_vector_types_unify_by_value_and_dimension_never_to_text(): + def vec(dim, ctype, dtype, rows=2): + data = np.arange(rows * dim, dtype=dtype).reshape(rows, dim).tobytes() + return _frame(rows, [("v", ctype, [], data, dim)]) + + f3, f2 = _arrow(vec(3, "f4v", " Date: Mon, 9 Nov 2026 22:23:25 +0900 Subject: [PATCH 28/49] tests: fix the cases of the first candidate run that the temporal kinds changed or that were mis-built - review fixes: the date64 case built an int64 beyond its own range (DAY_MS * 2**40); use 2**33, still past int32 days. A non-finite Decimal (NaN, -NaN, sNaN, Infinity, -Infinity) is refused in lists, with None, and in object columns. - nan-decimal fall back: the NaN row goes to the statement as before; the finite row around it is a run of carriable rows and crosses on the columnar entry now (a finite Decimal is carried), so one colp call is expected, not none. - pack refusals: bytes alone are carried (kind x) now; refuse bytes with text and bytes with bytearray instead. - temporal lists: a BINARY property reads back as the signed values of a Java byte[] or as bytes; compare after & 0xFF. - mixed chunk: the rows around an uncarriable row cross on the columnar entry (the date-time column is carried), one colp call per run. Co-Authored-By: Claude Sonnet 5.5 --- tests/test_columnar_temporal_lists.py | 3 ++- tests/test_native_columnar_pack.py | 6 +++++- tests/test_native_columnar_temporal.py | 18 +++++++++++------- ...st_native_columnar_temporal_review_fixes.py | 16 +++++++++++++++- tests/test_native_mixed_chunk.py | 4 +++- 5 files changed, 36 insertions(+), 11 deletions(-) diff --git a/tests/test_columnar_temporal_lists.py b/tests/test_columnar_temporal_lists.py index 35aacabb..9a9d62b7 100644 --- a/tests/test_columnar_temporal_lists.py +++ b/tests/test_columnar_temporal_lists.py @@ -66,7 +66,8 @@ def _stored(db, name, column): def _bytes(values): - return [None if v is None else bytes(v) for v in values] + # a BINARY property reads back as the values of a Java byte[] (signed, -128..127) or as bytes; & 0xFF takes both to the same bytes + return [None if v is None else bytes(b & 0xFF for b in v) for v in values] @pytest.mark.parametrize("path", ["insert_columns", "insert_many"]) diff --git a/tests/test_native_columnar_pack.py b/tests/test_native_columnar_pack.py index 92a0eb89..e8925661 100644 --- a/tests/test_native_columnar_pack.py +++ b/tests/test_native_columnar_pack.py @@ -190,7 +190,11 @@ def test_a_bitmap_longer_than_a_byte(): [1, 2**63], # past 64 bits [{"a": 1}, {"a": 2}], # nested [[1], [2]], # a list without a declared ARRAY_OF_FLOATS - [b"ab", b"cd"], + [ + b"ab", + "cd", + ], # bytes and text (a column of bytes alone crosses as kind x since #316) + [b"ab", bytearray(b"cd")], # two bytes-like types [np.int64(1), np.int64(2)], # numpy scalars in a list are not Python numbers ], ) diff --git a/tests/test_native_columnar_temporal.py b/tests/test_native_columnar_temporal.py index a97d0b61..7ae3b545 100644 --- a/tests/test_native_columnar_temporal.py +++ b/tests/test_native_columnar_temporal.py @@ -174,17 +174,21 @@ def test_pandas_timestamps_keep_microseconds_as_the_json_path_does( @pytest.mark.parametrize( - "column", + "column, colp_runs", [ - [dt.datetime(2020, 1, 1), dt.date(2020, 1, 1)], # a date-time and a date - [decimal.Decimal("1.5"), decimal.Decimal("NaN")], # a non-finite Decimal - [b"a", "b"], # bytes and text + ([dt.datetime(2020, 1, 1), dt.date(2020, 1, 1)], 0), # a date-time and a date + # A non-finite Decimal: the column is refused (BigDecimal("NaN") would throw mid-batch in the image), so the NaN row goes to the + # statement on its own, as before. The row around it is a run of carriable rows: it goes in bulk, and the columnar entry + # carries a finite Decimal now, so that one run crosses on arcadedb_n_insert_colp (before, it was the JSON entry). + ([decimal.Decimal("1.5"), decimal.Decimal("NaN")], 1), + ([b"a", "b"], 0), # bytes and text ], ids=["datetime-and-date", "nan-decimal", "bytes-and-text"], ) def test_a_column_the_buffers_cannot_carry_unchanged_still_falls_back( - temp_db, spy, monkeypatch, column + temp_db, spy, monkeypatch, column, colp_runs ): + assert columnar.pack({"x": column}, columnar.Declared(known={})) is None _type(temp_db, "Ref", declared=False) _type(temp_db, "Col", declared=False) rows = [{"id": i, "x": v} for i, v in enumerate(column)] @@ -195,11 +199,11 @@ def test_a_column_the_buffers_cannot_carry_unchanged_still_falls_back( ): # the statement path may refuse it; then the new path must refuse it the same way with pytest.raises(Exception): temp_db.insert_many("Col", rows) - assert spy.count(COLP) == 0 + assert spy.count(COLP) == colp_runs return spy.calls.clear() temp_db.insert_many("Col", rows) - assert spy.count(COLP) == 0 + assert spy.count(COLP) == colp_runs assert _stored(temp_db, "Col") == _stored(temp_db, "Ref") diff --git a/tests/test_native_columnar_temporal_review_fixes.py b/tests/test_native_columnar_temporal_review_fixes.py index 31c8430e..34898f9c 100644 --- a/tests/test_native_columnar_temporal_review_fixes.py +++ b/tests/test_native_columnar_temporal_review_fixes.py @@ -10,8 +10,10 @@ """ import datetime as dt +import decimal import math +import numpy as np import pytest from arcadedb._native import columnar, lib # noqa: E402 @@ -52,7 +54,7 @@ def test_a_date64_past_int32_days_is_not_laid_out_and_never_wraps(): assert columnar.pack({"d": wraps}, NO_TYPES) is None below = pa.array([DAY_MS * (-(2**31) - 1)], pa.int64()).cast(pa.date64()) assert columnar.pack({"d": below}, NO_TYPES) is None - one_bad = pa.array([0, DAY_MS, DAY_MS * 2**40], pa.int64()).cast(pa.date64()) + one_bad = pa.array([0, DAY_MS, DAY_MS * 2**33], pa.int64()).cast(pa.date64()) assert columnar.pack({"d": one_bad}, NO_TYPES) is None # one value is enough @@ -90,6 +92,18 @@ def test_a_timestamp_at_the_limits_stays_columnar(): assert _kinds({"t": big}) == ["T"], unit +@pytest.mark.parametrize("text", ["NaN", "-NaN", "sNaN", "Infinity", "-Infinity"]) +def test_a_non_finite_decimal_is_never_laid_out(text): + # new BigDecimal("NaN") throws in the image, in the middle of a batch: the column must stay in Python, in every container + bad = decimal.Decimal(text) + assert columnar.pack({"x": [decimal.Decimal("1.5"), bad]}, NO_TYPES) is None + assert columnar.pack({"x": [None, bad, None]}, NO_TYPES) is None + as_object = np.array([decimal.Decimal("1"), bad], dtype=object) + assert columnar.pack({"x": as_object}, NO_TYPES) is None + finite = columnar.pack({"x": [decimal.Decimal("1.5"), None]}, NO_TYPES) + assert finite is not None and finite[1][0].kind == "n" # the finite column crosses + + def _type(db, name): db.command("sql", f"CREATE DOCUMENT TYPE {name}") db.command("sql", f"CREATE PROPERTY {name}.id INTEGER") diff --git a/tests/test_native_mixed_chunk.py b/tests/test_native_mixed_chunk.py index 0c746bdf..cf409cb6 100644 --- a/tests/test_native_mixed_chunk.py +++ b/tests/test_native_mixed_chunk.py @@ -250,8 +250,10 @@ def test_an_uncarriable_value_sends_its_own_row_to_the_statement_not_the_chunk( spy.clear() db.insert_many("Bulk", rows) assert spy.count(MANY) == 1, "one statement run for the one uncarriable row" + # the rows before and after it stay on the bulk entry; its columnar form carries the date-time column now, so each run + # crosses on arcadedb_n_insert_colp (before it was the JSON entry, arcadedb_n_insert_rows) assert ( - spy.count(ROWS) == 2 + spy.count(COLP) == 2 and spy.count(ROWS) == 0 ), "the rows before and after it stay on the bulk entry" assert _stored(db, "Bulk") == _stored(db, "Ref") From 9f054a9264c408b9776abb0d28592b0dcd71d2b4 Mon Sep 17 00:00:00 2001 From: Taewoon Kim Date: Mon, 9 Nov 2026 19:07:55 +0900 Subject: [PATCH 29/49] insert_dataframe and insert_arrow: a vertex type loads through the columnar insert (refs #319) Both wheels loaded a vertex type through GraphBatch.create_vertices with one dict per row and a JSON text per chunk, and discarded the RIDs. The Java column inserters already create vertices (BulkOps.insertColumns, DocumentBatcher.insertColumns): newVertex, set per property, save, in a transaction, the loop GraphBatch.createVertices runs, so the stored records, the index entries and the bucket of each vertex are the same. insert_table now sends a vertex type the way it sends a document type; the GraphBatch is no longer opened for it. The rule kept from GraphBatch: a vertex load refuses to run inside a transaction the caller opened (a document load still joins it), with the engine's wording, before anything is written. parallel=True on a vertex type is still refused as before. Co-Authored-By: Claude Sonnet 5.5 --- src/arcadedb/_native/database.py | 2 +- src/arcadedb/dataframe.py | 141 ++++++++++----------- tests/conftest.py | 2 + tests/test_native_vertex_columnar.py | 55 +++++++++ tests/test_vertex_load_columnar.py | 178 +++++++++++++++++++++++++++ 5 files changed, 298 insertions(+), 80 deletions(-) create mode 100644 tests/test_native_vertex_columnar.py create mode 100644 tests/test_vertex_load_columnar.py diff --git a/src/arcadedb/_native/database.py b/src/arcadedb/_native/database.py index 3ee0f51b..c52495df 100644 --- a/src/arcadedb/_native/database.py +++ b/src/arcadedb/_native/database.py @@ -1394,7 +1394,7 @@ def insert_dataframe( ) -> int: """A pandas or Polars DataFrame into a type, in batches: the strict Arrow loading of ``Database.insert_dataframe`` (dtype refusals, declared types, NaN rules, the committed-rows message), batches through :meth:`insert_many` for a - document type and ``GraphBatch.create_vertices`` for a vertex type (``arcadedb/dataframe.py``). + document type and a vertex type alike (a vertex load refuses to run inside a transaction the caller opened; ``arcadedb/dataframe.py``). """ from ..dataframe import insert_table, to_arrow_table diff --git a/src/arcadedb/dataframe.py b/src/arcadedb/dataframe.py index 37c67a29..9b52dd56 100644 --- a/src/arcadedb/dataframe.py +++ b/src/arcadedb/dataframe.py @@ -5,17 +5,18 @@ (``Database.insert_dataframe`` and ``Database.insert_arrow``). Arrow is the one internal form: a pandas frame becomes an Arrow table, a Polars frame is already one, and each Arrow column becomes the typed array ``Database.insert_columns`` -sends across the bridge (a document type), or the property rows -``GraphBatch.create_vertices`` takes (a vertex type). No Java was added: both -paths use the existing glue (``DocumentBatcher.insertColumns`` and the -``GraphBatch`` vertex batcher). +sends across the bridge, for a document type and for a vertex type alike (the +vertex type gets vertices from the same ``DocumentBatcher.insertColumns``; it +refuses to run inside a transaction the caller opened, as ``GraphBatch`` does). +No Java was added. The native backend (the ``arcadedb`` wheel) shares everything but the last step: the checks, the type mapping, the batching and the error wording are -the same code. A document batch goes over as one JSON text through -``Database.insert_many`` (one crossing per 10,000 rows), a vertex batch through -``GraphBatch.create_vertices``, and a value JSON cannot carry (a date-time, a -decimal, bytes) travels as a tagged value the image decodes. +the same code. A batch of documents or vertices goes over by address through +the columnar entry (``arcadedb_n_insert_colp``) when every column can cross as +memory, else as one JSON text through ``Database.insert_many`` (one crossing per +10,000 rows), where a value JSON cannot carry (a date-time, a decimal, bytes) +travels as a tagged value the image decodes. """ import re @@ -425,81 +426,63 @@ def insert_table( native = _is_native(db) written = 0 # a document load inside the caller's own transaction joins it (insert_columns commits - # nothing then); a vertex load refuses to run inside one. parallel=True never joins it: the - # async writers commit the records themselves, so a rollback of the caller's transaction does - # not remove them - in_caller_transaction = ( - (not is_vertex) and (not parallel) and bool(db.is_transaction_active()) - ) - graph_batch = db.graph_batch() if is_vertex else None - try: - for batch in table.to_batches(max_chunksize=batch_size): - if batch.num_rows == 0: - continue - columns = { - name: _column_values( - name, - batch.column(i), - pa, - native, - arrow=native and not is_vertex, + # nothing then); a vertex load refuses to run inside one, as GraphBatch.createVertices does. + # parallel=True never joins it: the async writers commit the records themselves, so a rollback + # of the caller's transaction does not remove them + in_transaction = bool(db.is_transaction_active()) + in_caller_transaction = in_transaction and not parallel + if is_vertex and in_transaction: + raise ArcadeDBError( + f"Failed to create batch vertices for type '{type_name}': a vertex load commits its own batches " + f"and cannot run inside a transaction the caller opened: commit or roll back that transaction first" + ) + for batch in table.to_batches(max_chunksize=batch_size): + if batch.num_rows == 0: + continue + columns = { + name: _column_values( + name, + batch.column(i), + pa, + native, + arrow=native, + ) + for i, name in enumerate(names) + } + try: + if native: + _insert_documents_native( + db, type_name, names, columns, batch_size, parallel, declared + ) + else: + db.insert_columns( + type_name, columns, commit_every=batch_size, parallel=parallel ) - for i, name in enumerate(names) - } - try: - if is_vertex: - _insert_vertices( - graph_batch, type_name, names, columns, batch.num_rows - ) - elif native: - _insert_documents_native( - db, type_name, names, columns, batch_size, parallel, declared - ) - else: - db.insert_columns( - type_name, columns, commit_every=batch_size, parallel=parallel - ) - except ArcadeDBError as exc: - if written and in_caller_transaction: - # the batches joined the caller's transaction: nothing is committed - # until the caller commits, and a rollback removes them - raise ArcadeDBError( - f"{exc}; the {written} row(s) of the batches before this one " - f"were written in your transaction, not committed" - ) from exc - if written and parallel: - # the async writers commit on their own, inside the caller's transaction or not - raise ArcadeDBError( - f"{exc}; the {written} row(s) of the batches before this one " - f"were handed to the async writers, which commit on their own: " - f"a rollback of your transaction does not remove them" - ) from exc - if written: - raise ArcadeDBError( - f"{exc}; the {written} row(s) of the batches before this one " - f"were committed" - ) from exc - raise - written += batch.num_rows - finally: - if graph_batch is not None: - graph_batch.close() + except ArcadeDBError as exc: + if written and in_caller_transaction: + # the batches joined the caller's transaction: nothing is committed + # until the caller commits, and a rollback removes them + raise ArcadeDBError( + f"{exc}; the {written} row(s) of the batches before this one " + f"were written in your transaction, not committed" + ) from exc + if written and parallel: + # the async writers commit on their own, inside the caller's transaction or not + raise ArcadeDBError( + f"{exc}; the {written} row(s) of the batches before this one " + f"were handed to the async writers, which commit on their own: " + f"a rollback of your transaction does not remove them" + ) from exc + if written: + raise ArcadeDBError( + f"{exc}; the {written} row(s) of the batches before this one " + f"were committed" + ) from exc + raise + written += batch.num_rows return written -def _insert_vertices( - graph_batch, type_name: str, names: List[str], columns: Dict[str, Any], n: int -): - """One batch of vertices: the columns as one property dict per row, through ``GraphBatch.create_vertices``.""" - lists = [] - for name in names: - values = columns[name] - lists.append(values.tolist() if hasattr(values, "tolist") else values) - graph_batch.create_vertices( - type_name, [dict(zip(names, row)) for row in zip(*lists)] - ) - - def _arrow_row_values(col) -> list: """A column kept as an Arrow array, as the values the row path takes: Python values, or tagged values for a timestamp, date, decimal or binary column (the form it had before the columnar entry took them, so the stored value is unchanged). diff --git a/tests/conftest.py b/tests/conftest.py index d7d95ce1..11c528e4 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -55,6 +55,8 @@ collect_ignore.append("test_native_columnar_temporal.py") # the review of the temporal kinds: a value the image cannot take is refused in Python, never wrapped or thrown mid-batch collect_ignore.append("test_native_columnar_temporal_review_fixes.py") + # a vertex load by insert_dataframe or insert_arrow crosses as columns, one crossing per batch, observed on the native library + collect_ignore.append("test_native_vertex_columnar.py") # the rows of an insert_many chunk with date-times, Decimals, bytes and numpy scalars are told apart by the entry points they cross collect_ignore.append("test_native_mixed_chunk.py") # the native build's ingest defaults (insert_many and graph_batch commit size), observed on the native library diff --git a/tests/test_native_vertex_columnar.py b/tests/test_native_vertex_columnar.py new file mode 100644 index 00000000..154a1fe2 --- /dev/null +++ b/tests/test_native_vertex_columnar.py @@ -0,0 +1,55 @@ +"""Native: a vertex type loaded by ``insert_dataframe`` or ``insert_arrow`` crosses as columns (``arcadedb_n_insert_colp``), one crossing +per batch, with no GraphBatch crossing and no JSON text (ArcadeData/arcadedb-python #319). The stored vertices are compared with the old +route in ``tests/test_vertex_load_columnar.py``, which runs on both wheels. Needs the native library, so conftest.py collects this file on +the native backend only. +""" + +import numpy as np +import pandas as pd +import pyarrow as pa + +from arcadedb._native import lib # noqa: E402 + +COLP, ROWS, MANY = ( + "arcadedb_n_insert_colp", + "arcadedb_n_insert_rows", + "arcadedb_n_exec_many", +) + + +def _spy(monkeypatch): + calls = [] + real = lib.call + + def call(name, *args): + calls.append(name) + return real(name, *args) + + monkeypatch.setattr(lib, "call", call) + return calls + + +def _vertex_type(db, name): + db.command("sql", f"CREATE VERTEX TYPE {name}") + db.command("sql", f"CREATE PROPERTY {name}.id LONG") + + +def test_every_batch_of_a_vertex_load_is_one_columnar_crossing(temp_db, monkeypatch): + _vertex_type(temp_db, "V") + calls = _spy(monkeypatch) + frame = pd.DataFrame( + { + "id": np.arange(25, dtype=np.int64), + "name": [f"p{i}" for i in range(25)], + "score": [None if i % 5 == 0 else i * 0.5 for i in range(25)], + "born": pd.to_datetime(["2000-01-01"] * 25), + } + ) + assert temp_db.insert_dataframe("V", frame, batch_size=10) == 25 + assert calls.count(COLP) == 3 # batches of 10, 10 and 5 + assert calls.count(ROWS) == 0 and calls.count(MANY) == 0 + assert not [c for c in calls if "gb" in c or "graph" in c], calls + assert temp_db.query("sql", "SELECT count(*) AS n FROM V").first().get("n") == 25 + calls.clear() + assert temp_db.insert_arrow("V", pa.Table.from_pandas(frame)) == 25 + assert calls.count(COLP) == 1 diff --git a/tests/test_vertex_load_columnar.py b/tests/test_vertex_load_columnar.py new file mode 100644 index 00000000..e5a8d87c --- /dev/null +++ b/tests/test_vertex_load_columnar.py @@ -0,0 +1,178 @@ +"""``insert_dataframe`` and ``insert_arrow`` load a vertex type through the columnar insert, and the stored vertices, their indexes and +their buckets are those of the old route, ``GraphBatch.create_vertices`` (ArcadeData/arcadedb-python #319). + +Both routes are ``newVertex``, ``set`` per property and ``save`` on the caller's thread inside a transaction, so the records, the +index entries and the bucket each vertex lands in come out the same; the one difference the engine code shows is that a GraphBatch +turns the write-ahead log off for its load (its ``useWAL`` default) while the columnar insert uses the database's own settings, as the +document loads of the same calls always did. The old route is the private reference here: a type loaded through +``graph_batch().create_vertices`` and a type loaded through ``insert_dataframe`` are compared value by value and type by type, with the +index lookups and the per-bucket record counts. A spy on ``create_vertices`` proves which route ran (the old code calls it, the new code +must not). Runs on both wheels. +""" + +import datetime as dt + +import numpy as np +import pandas as pd +import pyarrow as pa +import pytest + +from arcadedb.exceptions import ArcadeDBError + +N = 37 # not a multiple of the batch size or of the bucket count + + +def _type(db, name, buckets=3): + db.command("sql", f"CREATE VERTEX TYPE {name} BUCKETS {buckets}") + db.command("sql", f"CREATE PROPERTY {name}.id LONG") + db.command("sql", f"CREATE PROPERTY {name}.grp INTEGER") + db.command("sql", f"CREATE PROPERTY {name}.emb ARRAY_OF_FLOATS") + db.command("sql", f"CREATE INDEX ON {name} (id) UNIQUE") + db.command("sql", f"CREATE INDEX ON {name} (grp) NOTUNIQUE") + + +def _frame(n=N): + ids = np.arange(n, dtype=np.int64) + return pd.DataFrame( + { + "id": ids, + "grp": (ids % 5).astype(np.int32), + "name": [None if i % 9 == 0 else f"n{i}-é" for i in range(n)], + "score": [None if i % 4 == 0 else i * 0.25 for i in range(n)], + "born": pd.to_datetime( + [ + dt.datetime(1969, 12, 31, 23, 59, 59, 999999) + dt.timedelta(days=i) + for i in range(n) + ] + ), + "emb": [np.array([i, i + 0.5, -i], dtype=np.float32) for i in range(n)], + } + ) + + +def _rows_of(frame): + out = [] + for rec in frame.to_dict("records"): + row = {} + for k, v in rec.items(): + if v is None or (isinstance(v, float) and v != v): + v = None # an Arrow null, as the old route of insert_dataframe handed it over + elif isinstance(v, pd.Timestamp): + v = v.to_pydatetime() + elif isinstance(v, np.ndarray): + v = v.tolist() + elif isinstance(v, np.generic): + v = v.item() + row[k] = v + out.append(row) + return out + + +def _old_route(db, name, frame): + with db.graph_batch() as batch: + batch.create_vertices(name, _rows_of(frame)) + + +def _stored(db, name): + sql = f"SELECT FROM {name} ORDER BY id" # nosec B608 - test-owned type name + return [ + {k: (type(v).__name__, v) for k, v in rec.items() if not k.startswith("@")} + for rec in db.query("sql", sql).to_list() + ] + + +def _bucket_counts(db, name): + counts = [] + for i in range(3): + sql = f"SELECT count(*) AS n FROM bucket:{name}_{i}" # nosec B608 - test-owned names + counts.append(db.query("sql", sql).first().get("n")) + return sorted(counts) + + +def _index_view(db, name): + by_id, by_grp = {}, {} + for i in (0, 7, N - 1): + sql = f"SELECT id, name FROM {name} WHERE id = {i}" # nosec B608 - test-owned names + by_id[i] = [ + (r.get("id"), r.get("name")) for r in db.query("sql", sql).to_list() + ] + for g in range(5): + sql = f"SELECT id FROM {name} WHERE grp = {g}" # nosec B608 - test-owned names + by_grp[g] = sorted(r.get("id") for r in db.query("sql", sql).to_list()) + return by_id, by_grp + + +@pytest.fixture +def old_and_new(temp_db): + _type(temp_db, "Old") + _type(temp_db, "New") + _old_route(temp_db, "Old", _frame()) + return temp_db + + +def test_the_vertices_indexes_and_buckets_equal_those_of_graph_batch( + old_and_new, monkeypatch +): + db = old_and_new + with db.graph_batch() as batch: + graph_batch_class = type(batch) + + def refused(*args, **kwargs): + raise AssertionError( + "a vertex load through insert_dataframe went through GraphBatch.create_vertices" + ) + + monkeypatch.setattr(graph_batch_class, "create_vertices", refused) + assert db.insert_dataframe("New", _frame(), batch_size=10) == N + expected = _stored(db, "Old") + assert len(expected) == N and expected[7]["emb"][0] == "list" + assert expected[0].get("name", ("NoneType", None))[1] is None + assert sum(_bucket_counts(db, "Old")) == N and max(_bucket_counts(db, "Old")) < N + assert _stored(db, "New") == expected + assert _bucket_counts(db, "New") == _bucket_counts(db, "Old") + assert _index_view(db, "New") == _index_view(db, "Old") + assert db.lookup_by_key("New", ["id"], [7]).get("name") == "n7-é" + assert ( + db.query("sql", "SELECT outE().size() AS e FROM New WHERE id = 7") + .first() + .get("e") + == 0 + ) # a vertex + + +def test_insert_arrow_loads_a_vertex_type_the_same_way(old_and_new): + db = old_and_new + assert db.insert_arrow("New", pa.Table.from_pandas(_frame()), batch_size=16) == N + assert _stored(db, "New") == _stored(db, "Old") + + +def test_a_duplicate_key_fails_and_rolls_the_batch_back(temp_db): + _type(temp_db, "Dup") + frame = _frame(20) + frame.loc[15, "id"] = 3 # the unique index refuses it + with pytest.raises(ArcadeDBError, match="Duplicat"): + temp_db.insert_dataframe("Dup", frame, batch_size=50) + assert temp_db.is_transaction_active() is False + assert temp_db.query("sql", "SELECT count(*) AS n FROM Dup").first().get("n") == 0 + + +def test_a_vertex_load_refuses_to_run_inside_a_transaction_the_caller_opened(temp_db): + _type(temp_db, "Mine") + with temp_db.transaction(): + with pytest.raises( + ArcadeDBError, match="cannot run inside a transaction the caller opened" + ): + temp_db.insert_dataframe("Mine", _frame(5)) + assert temp_db.query("sql", "SELECT count(*) AS n FROM Mine").first().get("n") == 0 + assert temp_db.insert_dataframe("Mine", _frame(5)) == 5 # outside one it loads + + +def test_a_document_load_still_joins_the_callers_transaction(temp_db): + temp_db.command("sql", "CREATE DOCUMENT TYPE Doc") + with pytest.raises(RuntimeError, match="stop"): + with temp_db.transaction(): + temp_db.insert_dataframe( + "Doc", pd.DataFrame({"id": [1, 2, 3]}), batch_size=2 + ) + raise RuntimeError("stop") + assert temp_db.query("sql", "SELECT count(*) AS n FROM Doc").first().get("n") == 0 From 848a304be54c929c206fc3b072e345f42f6e7088 Mon Sep 17 00:00:00 2001 From: Taewoon Kim Date: Mon, 9 Nov 2026 21:28:51 +0900 Subject: [PATCH 30/49] Columnar insert: retry a chunk whose commit raises a NeedRetryException, on both wheels (review of #335) GraphBatch.createVerticesWithRetryInternal rolled a vertex chunk back and wrote it again (back-off from 1 s doubling to 10 s, 10 retries) on any NeedRetryException: an MVCC ConcurrentModificationException, a lock timeout. The vertex load through the columnar insert is one plain transaction per chunk, so two threads on one vertex type with a NOTUNIQUE index would get the error with earlier chunks committed. BulkOps.insertColumns (native wheel) and DocumentBatcher.insertColumns and insertManyJson (JVM wheel) now write a chunk in a transaction of their own, roll it back on a NeedRetryException and write it again from its first row, bounded like the engine's and rethrowing the error unchanged after the last attempt. A transaction the caller opened is untouched: rows join it, nothing is retried. Tests (both wheels): 2 and 4 threads, 20 batches each on one vertex type, count == total; a duplicate in the third of five batches keeps the first two, leaves no index entry of its own, states the committed count, and the fixed tail loads. Co-Authored-By: Claude Sonnet 5.5 --- .../java/org/humemai/nativelib/BulkOps.java | 94 +++++++++---- .../com/arcadedb/python/DocumentBatcher.java | 124 +++++++++++------- tests/test_vertex_load_review_fixes.py | 92 +++++++++++++ 3 files changed, 241 insertions(+), 69 deletions(-) create mode 100644 tests/test_vertex_load_review_fixes.py diff --git a/native/src/main/java/org/humemai/nativelib/BulkOps.java b/native/src/main/java/org/humemai/nativelib/BulkOps.java index 00798947..c703d872 100644 --- a/native/src/main/java/org/humemai/nativelib/BulkOps.java +++ b/native/src/main/java/org/humemai/nativelib/BulkOps.java @@ -12,6 +12,7 @@ import com.arcadedb.database.MutableDocument; import com.arcadedb.database.async.DatabaseAsyncExecutor; import com.arcadedb.database.async.ErrorCallback; +import com.arcadedb.exception.NeedRetryException; import com.arcadedb.schema.EdgeType; import com.arcadedb.schema.VertexType; @@ -27,6 +28,7 @@ import java.time.ZoneOffset; import java.util.List; import java.util.Map; +import java.util.function.IntConsumer; final class BulkOps { private BulkOps() { @@ -188,32 +190,80 @@ private static long insertColumns(final Database db, final String type, final Ma } return n; } - final boolean wasActive = db.isTransactionActive(); - if (!wasActive) - db.begin(); - try { - for (int i = 0; i < n; i++) { - final MutableDocument doc = vertex ? db.newVertex(type) : db.newDocument(type); - setRow(doc, i, names, kinds, dims, data, nulls, blobs, scratch); - doc.save(); - if (!wasActive && commitEvery > 0 && (i + 1) % commitEvery == 0) { - db.commit(); - db.begin(); - } - } - if (!wasActive) - db.commit(); - } catch (final Throwable e) { - if (!wasActive && db.isTransactionActive()) { + writeChunks(db, n, commitEvery, i -> { + final MutableDocument doc = vertex ? db.newVertex(type) : db.newDocument(type); + setRow(doc, i, names, kinds, dims, data, nulls, blobs, scratch); + doc.save(); + }); + return n; + } + + /** + * The retries of a chunk and the back-off before them are the engine's GraphBatch defaults (commitRetries 10, commitRetryDelayMs 1000, + * doubling up to 10 s), so a load that went through createVertices before keeps the same patience. + */ + private static final int COMMIT_RETRIES = 10; + private static final long COMMIT_RETRY_DELAY_MS = 1_000L; + private static final long MAX_COMMIT_RETRY_DELAY_MS = 10_000L; + + /** + * Writes rows [0, n) with writeRow (build and save row i), in transactions opened here: one per commitEvery rows (0: one for all). A + * chunk whose save or commit raises a NeedRetryException (an MVCC conflict with another writer, a lock timeout) is rolled back and + * written again from its first row after a back-off, up to COMMIT_RETRIES times; after the last attempt the error is thrown unchanged. + * Any other failure rolls the open chunk back and is thrown; the chunks committed before it stay. A transaction the caller opened is the + * caller's: the rows join it, nothing is committed, rolled back or retried here. + */ + private static void writeChunks(final Database db, final int n, final int commitEvery, final IntConsumer writeRow) { + if (db.isTransactionActive()) { + for (int i = 0; i < n; i++) + writeRow.accept(i); + return; + } + for (int from = 0; from < n;) { + final int to = commitEvery > 0 ? (int) Math.min((long) n, (long) from + commitEvery) : n; + for (int attempt = 0;;) { try { - db.rollback(); - } catch (final Throwable rollbackError) { - e.addSuppressed(rollbackError); + db.begin(); + for (int i = from; i < to; i++) + writeRow.accept(i); + db.commit(); + break; + } catch (final NeedRetryException e) { + if (!rollbackQuietly(db, e) || ++attempt > COMMIT_RETRIES) + throw e; + backoffBeforeRetry(attempt); + } catch (final Throwable e) { + rollbackQuietly(db, e); + throw e; } } - throw e; + from = to; + } + } + + /** Rolls back the open transaction, if any; false when the rollback itself failed (the failure is added to e as suppressed). */ + private static boolean rollbackQuietly(final Database db, final Throwable e) { + if (!db.isTransactionActive()) + return true; + try { + db.rollback(); + return true; + } catch (final Throwable rollbackError) { + e.addSuppressed(rollbackError); + return false; + } + } + + private static void backoffBeforeRetry(final int attempt) { + long delay = COMMIT_RETRY_DELAY_MS; + for (int i = 1; i < attempt && delay < MAX_COMMIT_RETRY_DELAY_MS; i++) + delay = Math.min(delay * 2, MAX_COMMIT_RETRY_DELAY_MS); + try { + Thread.sleep(delay); + } catch (final InterruptedException ie) { + Thread.currentThread().interrupt(); + throw new NeedRetryException("Interrupted while waiting to retry the commit of a bulk-insert chunk"); } - return n; } /** Sets the properties of row i on doc from the columns: the one place that reads a value out of a buffer. */ diff --git a/src/java/com/arcadedb/python/DocumentBatcher.java b/src/java/com/arcadedb/python/DocumentBatcher.java index bbcae4f8..278361b5 100644 --- a/src/java/com/arcadedb/python/DocumentBatcher.java +++ b/src/java/com/arcadedb/python/DocumentBatcher.java @@ -31,6 +31,7 @@ import com.arcadedb.database.Database; import com.arcadedb.database.MutableDocument; import com.arcadedb.database.async.ErrorCallback; +import com.arcadedb.exception.NeedRetryException; import com.arcadedb.schema.DocumentType; import com.arcadedb.schema.EdgeType; import com.arcadedb.schema.VertexType; @@ -38,6 +39,7 @@ import com.arcadedb.serializer.json.JSONObject; import java.util.Map; +import java.util.function.IntConsumer; import java.util.concurrent.atomic.AtomicLong; import java.util.concurrent.atomic.AtomicReference; @@ -69,31 +71,11 @@ public static long insertManyJson(final Database db, final String typeName, fina // commits and the failure rollback apply only to the transactions opened // here, matching the Python per-row fallback (#7882). final boolean vertex = n > 0 && isVertexType(db, typeName); - final boolean wasActive = db.isTransactionActive(); - if (!wasActive) - db.begin(); - try { - for (int i = 0; i < n; i++) { - final MutableDocument doc = newRecord(db, typeName, vertex); - fill(doc, rows.getJSONObject(i), i, vectorNames, vectors); - doc.save(); - if (!wasActive && commitEvery > 0 && (i + 1) % commitEvery == 0) { - db.commit(); - db.begin(); - } - } - if (!wasActive) - db.commit(); - } catch (final Throwable e) { - if (!wasActive && db.isTransactionActive()) { - try { - db.rollback(); - } catch (final Throwable rollbackError) { - e.addSuppressed(rollbackError); - } - } - throw e; - } + writeChunks(db, n, commitEvery, i -> { + final MutableDocument doc = newRecord(db, typeName, vertex); + fill(doc, rows.getJSONObject(i), i, vectorNames, vectors); + doc.save(); + }); return n; } @@ -113,32 +95,80 @@ public static long insertColumns(final Database db, final String typeName, final final int n, final int commitEvery) { checkColumns(names, columns, n); final boolean vertex = n > 0 && isVertexType(db, typeName); - final boolean wasActive = db.isTransactionActive(); - if (!wasActive) - db.begin(); - try { - for (int i = 0; i < n; i++) { - final MutableDocument doc = newRecord(db, typeName, vertex); - fillRow(doc, names, columns, i); - doc.save(); - if (!wasActive && commitEvery > 0 && (i + 1) % commitEvery == 0) { - db.commit(); - db.begin(); - } - } - if (!wasActive) - db.commit(); - } catch (final Throwable e) { - if (!wasActive && db.isTransactionActive()) { + writeChunks(db, n, commitEvery, i -> { + final MutableDocument doc = newRecord(db, typeName, vertex); + fillRow(doc, names, columns, i); + doc.save(); + }); + return n; + } + + /** + * The retries of a chunk and the back-off before them are the engine's GraphBatch defaults (commitRetries 10, commitRetryDelayMs 1000, + * doubling up to 10 s), so a load that went through createVertices before keeps the same patience. + */ + private static final int COMMIT_RETRIES = 10; + private static final long COMMIT_RETRY_DELAY_MS = 1_000L; + private static final long MAX_COMMIT_RETRY_DELAY_MS = 10_000L; + + /** + * Writes rows [0, n) with writeRow (build and save row i), in transactions opened here: one per commitEvery rows (0: one for all). A + * chunk whose save or commit raises a NeedRetryException (an MVCC conflict with another writer, a lock timeout) is rolled back and + * written again from its first row after a back-off, up to COMMIT_RETRIES times; after the last attempt the error is thrown unchanged. + * Any other failure rolls the open chunk back and is thrown; the chunks committed before it stay. A transaction the caller opened is the + * caller's: the rows join it, nothing is committed, rolled back or retried here. + */ + private static void writeChunks(final Database db, final int n, final int commitEvery, final IntConsumer writeRow) { + if (db.isTransactionActive()) { + for (int i = 0; i < n; i++) + writeRow.accept(i); + return; + } + for (int from = 0; from < n;) { + final int to = commitEvery > 0 ? (int) Math.min((long) n, (long) from + commitEvery) : n; + for (int attempt = 0;;) { try { - db.rollback(); - } catch (final Throwable rollbackError) { - e.addSuppressed(rollbackError); + db.begin(); + for (int i = from; i < to; i++) + writeRow.accept(i); + db.commit(); + break; + } catch (final NeedRetryException e) { + if (!rollbackQuietly(db, e) || ++attempt > COMMIT_RETRIES) + throw e; + backoffBeforeRetry(attempt); + } catch (final Throwable e) { + rollbackQuietly(db, e); + throw e; } } - throw e; + from = to; + } + } + + /** Rolls back the open transaction, if any; false when the rollback itself failed (the failure is added to e as suppressed). */ + private static boolean rollbackQuietly(final Database db, final Throwable e) { + if (!db.isTransactionActive()) + return true; + try { + db.rollback(); + return true; + } catch (final Throwable rollbackError) { + e.addSuppressed(rollbackError); + return false; + } + } + + private static void backoffBeforeRetry(final int attempt) { + long delay = COMMIT_RETRY_DELAY_MS; + for (int i = 1; i < attempt && delay < MAX_COMMIT_RETRY_DELAY_MS; i++) + delay = Math.min(delay * 2, MAX_COMMIT_RETRY_DELAY_MS); + try { + Thread.sleep(delay); + } catch (final InterruptedException ie) { + Thread.currentThread().interrupt(); + throw new NeedRetryException("Interrupted while waiting to retry the commit of a bulk-insert chunk"); } - return n; } /** diff --git a/tests/test_vertex_load_review_fixes.py b/tests/test_vertex_load_review_fixes.py new file mode 100644 index 00000000..a0eea0c4 --- /dev/null +++ b/tests/test_vertex_load_review_fixes.py @@ -0,0 +1,92 @@ +"""Review of the vertex loads through the columnar insert (#335): a commit conflict is retried, a failed later batch is exact. + +The old route, ``GraphBatch.create_vertices``, rolled a vertex chunk back and wrote it again (with a back-off, up to ``commitRetries``) +when its commit raised a ``NeedRetryException``: an MVCC conflict with another writer, a lock timeout. The columnar insert is one plain +transaction per chunk, so the same conflict would have surfaced as an error with earlier chunks already committed. The chunk loop of +``BulkOps.insertColumns`` (native wheel) and ``DocumentBatcher.insertColumns`` (JVM wheel) now does the retry, only in a transaction it +opened itself. Runs on both wheels. + +* several threads load one vertex type with a NOTUNIQUE index on a handful of key values, in many small batches: every row is stored; +* a duplicate key in a later batch: the earlier batches stay committed, the failing batch leaves no row and no index entry, the message + states the committed count, and the fixed tail loads afterwards. +""" + +import threading + +import numpy as np +import pyarrow as pa +import pytest + +from arcadedb.exceptions import ArcadeDBError + + +def _count(db, name): + sql = f"SELECT count(*) AS n FROM {name}" # nosec B608 - test-owned type name + return db.query("sql", sql).first().get("n") + + +def _vertex_type(db, name, unique=False): + db.command("sql", f"CREATE VERTEX TYPE {name} BUCKETS 3") + db.command("sql", f"CREATE PROPERTY {name}.id LONG") + db.command("sql", f"CREATE PROPERTY {name}.grp INTEGER") + db.command("sql", f"CREATE INDEX ON {name} (grp) NOTUNIQUE") + if unique: + db.command("sql", f"CREATE INDEX ON {name} (id) UNIQUE") + + +def _table(first, n, grps=5): + ids = np.arange(first, first + n, dtype=np.int64) + return pa.table({"id": ids, "grp": pa.array(ids % grps, pa.int32())}) + + +@pytest.mark.parametrize("threads", [2, 4]) +def test_threads_loading_one_vertex_type_in_small_batches_lose_no_row(temp_db, threads): + _vertex_type(temp_db, "V") + batches, batch_size = 20, 25 + errors = [] + + def load(worker): + try: + table = _table(worker * 1_000_000, batches * batch_size) + n = temp_db.insert_arrow("V", table, batch_size=batch_size) + assert n == batches * batch_size + except BaseException as e: # noqa: BLE001 - reported below, in the test thread + errors.append(repr(e)) + + pool = [threading.Thread(target=load, args=(w,)) for w in range(threads)] + for t in pool: + t.start() + for t in pool: + t.join(timeout=300) + assert not errors, errors + total = threads * batches * batch_size + assert _count(temp_db, "V") == total + distinct = temp_db.query("sql", "SELECT count(DISTINCT id) AS n FROM V") + assert distinct.first().get("n") == total # no row written twice by a retry + by_group = temp_db.query("sql", "SELECT grp, count(*) AS n FROM V GROUP BY grp") + assert sum(r.get("n") for r in by_group.to_list()) == total # the index agrees + + +def test_a_duplicate_in_a_later_batch_keeps_the_earlier_batches_and_nothing_of_its_own( + temp_db, +): + _vertex_type(temp_db, "Dup", unique=True) + ids = np.arange(500, dtype=np.int64) + ids[250] = 7 # the third batch of five repeats a key that the first batch committed + table = pa.table({"id": ids, "grp": pa.array(ids % 5, pa.int32())}) + with pytest.raises(ArcadeDBError, match="Duplicat") as err: + temp_db.insert_arrow("Dup", table, batch_size=100) + assert "the 200 row(s) of the batches before this one were committed" in str( + err.value + ) + assert temp_db.is_transaction_active() is False + assert _count(temp_db, "Dup") == 200 # batches 1 and 2, whole + for key in (200, 249, 251, 299): # the failing batch left no index entry behind + assert temp_db.lookup_by_key("Dup", ["id"], [key]) is None + assert temp_db.lookup_by_key("Dup", ["id"], [7]).get("id") == 7 # the committed one + + ids[250] = 250 # the fixed tail loads + tail = pa.table({"id": ids[200:], "grp": pa.array(ids[200:] % 5, pa.int32())}) + assert temp_db.insert_arrow("Dup", tail, batch_size=100) == 300 + assert _count(temp_db, "Dup") == 500 + assert temp_db.lookup_by_key("Dup", ["id"], [250]).get("id") == 250 From 7e811708439fbb2d0db3e160d99f49865554bd99 Mon Sep 17 00:00:00 2001 From: Taewoon Kim Date: Mon, 9 Nov 2026 22:25:12 +0900 Subject: [PATCH 31/49] create_vertices: decide the whole input before any chunk is sent when it has several chunks (refs #318) A bad scalar in a later chunk used to send only that chunk through the property-matrix path, which stores a Python int as a Long while the JSON chunks store an Integer: one undeclared property got two Java types in one call. An input of more than one chunk now has its columns of exact scalars screened in C over the whole input first (type census, joined text of a str column, min and max of an int column, one float64 array for a float column, in blocks of 100,000 rows); a column that may hold a bad value is judged value by value as before, so a bad scalar anywhere sends the whole input the matrix way, as before #330. One chunk is decided by its own dumps before anything is sent. Co-Authored-By: Claude Sonnet 5.5 --- src/arcadedb/graph_batch.py | 2 +- src/arcadedb/type_conversion.py | 86 +++++++++++++++++++--- tests/test_bulk_single_pass_validation.py | 30 ++++++++ tests/test_graph_batch_single_pass.py | 87 ++++++++++++++++------- 4 files changed, 168 insertions(+), 37 deletions(-) diff --git a/src/arcadedb/graph_batch.py b/src/arcadedb/graph_batch.py index efc1aff2..4fad9342 100644 --- a/src/arcadedb/graph_batch.py +++ b/src/arcadedb/graph_batch.py @@ -272,7 +272,7 @@ def _create_vertices_json_bulk(self, type_name, rows): except Exception: return None - tagged = classify_vertex_rows(rows, declared) + tagged = classify_vertex_rows(rows, declared, len(rows) > self._BULK_CHUNK) if tagged is None: # a property name that is not a str (json.dumps would write 1 as "1" where the matrix path refuses it), or a type the # tagged JSON does not carry (a set, a numpy or pandas value, any subclass, a list of only numbers: the matrix path diff --git a/src/arcadedb/type_conversion.py b/src/arcadedb/type_conversion.py index 2ec8f424..68e3857a 100644 --- a/src/arcadedb/type_conversion.py +++ b/src/arcadedb/type_conversion.py @@ -1062,14 +1062,70 @@ def _column(rows: Any, key: str): return map(dict.get, filter(None, rows), itertools.repeat(key)) -def classify_vertex_rows(rows: Any, declared: Any): +_NONE_KIND = type(None) +_SCREEN_BLOCK = 100_000 # rows per block of the scalar screen, so its transient lists and joined texts stay bounded + + +def _scalar_column_suspect(column: list) -> bool: + """Whether one block of a column of exact scalar values may hold a value the JSON text would change. + + Conservative (a false alarm only sends the column to the exact per-value judgement) and run in C: the type census, the joined + text of a str column (a lone surrogate has no UTF-8 form), min and max of an int column (the 64-bit range), and a float column as + one float64 array (NaN, Infinity, a subnormal). Columns of mixed kinds are suspect. + """ + kinds = set(map(type, column)) + if _NONE_KIND in kinds: + column = [value for value in column if value is not None] + kinds.discard(_NONE_KIND) + if kinds <= {bool}: + return False + if len(kinds) != 1: + return True + (kind,) = kinds + if kind is str: + text = "".join(column) + if text.isascii(): + return False + try: + text.encode("utf-8") + except UnicodeEncodeError: + return True + return False + if kind is int: + return not (_INT64_MIN <= min(column) and max(column) <= _INT64_MAX) + if kind is float and _np is not None: + values = _np.fromiter(column, dtype=_np.float64, count=len(column)) + if not _np.isfinite(values).all(): + return True + values = values[values != 0.0] + return bool((_np.abs(values) < _FLOAT_NORMAL_MIN).any()) + return True + + +def _scalar_screen(rows: Any, key: str) -> bool: + """Whether the column `key` of all `rows`, which holds exact scalars only, may hold a value the JSON text would change.""" + for start in range(0, len(rows), _SCREEN_BLOCK): + block = list(_column(rows[start : start + _SCREEN_BLOCK], key)) + if _scalar_column_suspect(block): + return True + return False + + +def classify_vertex_rows(rows: Any, declared: Any, screen_scalars: bool = False): """What GraphBatch.create_vertices must know about its rows before it writes any chunk, decided by types. + `screen_scalars` is for an input of more than one chunk. One chunk is decided by its own dumps before anything is sent, so a + value the text would change sends all of it the matrix way. Several chunks are sent one after the other, so such a value in + a late chunk would leave the earlier ones stored from the JSON text (an int as an Integer) and the late one from the matrix path + (a Long): the stored type of a property would depend on which chunk held a bad value. With `screen_scalars` the columns of exact + scalars are screened in C over the whole input first (`_scalar_screen`), and a column that may hold a bad value is judged value + by value as the code before did, so a bad scalar anywhere gives None for the whole input, as it always did. + Returns None when the rows must all go the property-matrix way (a property name that is not a str, or a value of a type the tagged JSON does not carry: a set, a numpy or pandas value, a list or tuple of only numbers, any other object); otherwise True when some value needs the tagged text (json_bulk_dumps_tagged) and False when json_bulk_dumps carries every row. A value of an exact - scalar type is not judged here: whether the text carries it (a NaN, an integer beyond 64 bits, a lone surrogate) is decided in - the dumps of its chunk. The property of `declared` (ARRAY_OF_FLOATS) that holds a list or a numpy array is left to + scalar type is not judged here unless `screen_scalars`: whether the text carries it (a NaN, an integer beyond 64 bits, a lone + surrogate) is decided in the dumps of its chunk. The property of `declared` (ARRAY_OF_FLOATS) that holds a list or a numpy array is left to split_float_vectors. The census over the keys and the values runs in C; a Python-level loop runs only over a column that holds a type other than the exact scalars. """ @@ -1083,18 +1139,28 @@ def classify_vertex_rows(rows: Any, declared: Any): tagged = False for key in keys: odd = set(map(type, _column(rows, key))) - _BULK_SCALAR_TYPES - if not odd: - continue - if key in declared and all( - kind is list or (_np is not None and issubclass(kind, _np.ndarray)) - for kind in odd + if screen_scalars: + if not odd and not _scalar_screen(rows, key): + continue + elif not odd or ( + key in declared + and all( + kind is list or (_np is not None and issubclass(kind, _np.ndarray)) + for kind in odd + ) ): - continue # the vectors of a declared column + continue # exact scalars, or the vectors of a declared column for value in _column(rows, key): - if type(value) in _BULK_SCALAR_TYPES or ( + if (type(value) in _BULK_SCALAR_TYPES and not screen_scalars) or ( key in declared and is_vector_candidate(value) ): continue + if ( + screen_scalars + and type(value) is float + and 0.0 < abs(value) < _FLOAT_NORMAL_MIN + ): + return None # a subnormal: the text is read back as a DECIMAL if not json_bulk_scalar_ok(value): if not json_bulk_shape_ok(value): return None diff --git a/tests/test_bulk_single_pass_validation.py b/tests/test_bulk_single_pass_validation.py index 7fa56ed6..c830597b 100644 --- a/tests/test_bulk_single_pass_validation.py +++ b/tests/test_bulk_single_pass_validation.py @@ -432,3 +432,33 @@ def test_a_bad_row_inside_a_large_batch_is_refused(temp_db_path, entry, bad): else: with db.graph_batch(parallel_flush=False) as batch: batch.create_vertices("PartV", rows) + + +def test_the_scalar_screen_flags_what_the_text_would_change_and_passes_the_rest(): + # the whole-input screen of create_vertices: conservative, so a false alarm is allowed, a miss is not + clean = { + "str": ["a", "é中", "\U0001f600", "", None], + "int": [1, -1, INT64_MAX, INT64_MIN, None], + "float": [0.5, -0.0, 0.0, 1e-300, 1.7e308, None], + "bool": [True, False, None], + "nones": [None, None], + } + for name, column in clean.items(): + assert tc._scalar_column_suspect(list(column)) is False, name + bad = { + "surrogate": ["ok", "a\ud800b"], + "above_max": [1, INT64_MAX + 1], + "below_min": [INT64_MIN - 1, 1], + "nan": [1.5, math.nan], + "inf": [math.inf], + "subnormal": [1.5, 5e-324], + "mixed": [1, "a"], + "int_float": [1, 2.5], + } + for name, column in bad.items(): + assert tc._scalar_column_suspect(list(column)) is True, name + # the screen runs over blocks: a bad value in the last block of a large column is found + column = [{"x": i} for i in range(250_000)] + column[249_999] = {"x": INT64_MAX + 1} + assert tc.classify_vertex_rows(column, set(), True) is None + assert tc.classify_vertex_rows(column, set(), False) is False diff --git a/tests/test_graph_batch_single_pass.py b/tests/test_graph_batch_single_pass.py index 3b21bfdd..2a35a768 100644 --- a/tests/test_graph_batch_single_pass.py +++ b/tests/test_graph_batch_single_pass.py @@ -3,9 +3,11 @@ Before, `_create_vertices_json_bulk` called `json_bulk_scalar_ok` on every value of every row before the first chunk was written, and the chunk was then walked again by json.dumps. Now `type_conversion.classify_vertex_rows` takes a census of the types in C, and the value checks come from the one json.dumps pass of each chunk (`json_bulk_dumps`, as `json_bulk_dumps_tagged` already did for the tagged -text). What stays: the whole input goes the property-matrix way for a property name that is not a str or a type the text does not carry -(`_create_vertices_json_bulk` returns None); what changes: a scalar the text would change (a subnormal, a NaN, an integer beyond 64 bits, -a lone surrogate) sends its own chunk, not the whole input, through the matrix path, as it already did for a tagged chunk. +text). The rule that keeps the stored type of a property from depending on which chunk held a bad value: an input of one chunk is decided +by its own dumps before anything is sent; an input of several chunks first has its columns of exact scalars screened in C over the whole +input (`classify_vertex_rows(..., screen_scalars=True)`), and a bad scalar (a NaN, an integer beyond 64 bits, a lone surrogate, a +subnormal) anywhere sends the WHOLE input the property-matrix way, as it did before. A name that is not a str or a type the text does not +carry does the same (`_create_vertices_json_bulk` returns None). """ import pytest @@ -23,10 +25,12 @@ def _rows(n): ] -def test_scalar_rows_are_not_judged_value_by_value(temp_db_path, monkeypatch): +@pytest.mark.parametrize("chunk", [100_000, 500], ids=["one_chunk", "screened"]) +def test_scalar_rows_are_not_judged_value_by_value(temp_db_path, monkeypatch, chunk): def refuse(value): raise AssertionError("a per-value gate ran over a scalar table") + monkeypatch.setattr(GraphBatch, "_BULK_CHUNK", chunk) monkeypatch.setattr(tc, "json_bulk_scalar_ok", refuse) monkeypatch.setattr(gbm, "json_bulk_scalar_ok", refuse) monkeypatch.setattr(tc, "json_bulk_shape_ok", refuse) @@ -53,16 +57,17 @@ def test_the_whole_input_still_goes_the_matrix_way_for_a_name_or_type_the_text_d @pytest.mark.parametrize("value", [5e-324, float("nan"), 2**70, "a\ud800b"]) -def test_a_scalar_the_text_changes_sends_its_chunk_through_the_matrix_path( - temp_db_path, monkeypatch, value +@pytest.mark.parametrize("chunks", [1, 3], ids=["one_chunk", "three_chunks"]) +def test_a_scalar_the_text_changes_sends_the_whole_input_through_the_matrix_path( + temp_db_path, monkeypatch, value, chunks ): - monkeypatch.setattr(GraphBatch, "_BULK_CHUNK", 100) + monkeypatch.setattr(GraphBatch, "_BULK_CHUNK", 100 if chunks == 3 else 1_000) calls = [] real = GraphBatch._to_java_property_matrix - def spy(self, chunk): - calls.append(len(chunk)) - return real(self, chunk) + def spy(self, rows): + calls.append(len(rows)) + return real(self, rows) monkeypatch.setattr(GraphBatch, "_to_java_property_matrix", spy) rows = [{"id": i, "x": 1.5} for i in range(250)] @@ -76,7 +81,7 @@ def spy(self, chunk): Exception ): # noqa: BLE001 - the matrix path refuses a value that no entry point stores rids = None - assert calls == [100], "only the chunk of row 150 takes the matrix path" + assert calls == [250], "the whole input takes the matrix path, once" if value == 5e-324: # stored as the double every other entry point stores; the text of a subnormal was read as a DECIMAL assert len(rids) == 250 @@ -84,16 +89,20 @@ def spy(self, chunk): assert stored == 5e-324 and type(stored) is float +@pytest.mark.parametrize( + "bad_row", [5, 150_000, 249_999], ids=["first_chunk", "middle_chunk", "last_chunk"] +) def test_the_stored_type_does_not_depend_on_which_chunk_held_a_bad_value( - temp_db_path, monkeypatch + temp_db_path, monkeypatch, bad_row ): - """An undeclared int property is stored as the same type in the chunks that crossed as JSON text and in the chunk that took the - property-matrix path because a NaN sat in another property of one of its rows (the 100,000-row chunking of the real class). + """An undeclared int property is stored as the same Java type in every chunk, whichever chunk holds a NaN in another property + (the 100,000-row chunking of the real class): the NaN sends the whole input the matrix way, so no chunk is stored from the JSON text. """ n = 250_000 rows = [{"id": i, "k": i % 1000} for i in range(n)] - rows[150_000] = {"id": 150_000, "k": 1, "x": float("nan")} # chunk 2 of 3 only - probes = {5: "chunk 1", 150_001: "chunk 2", 249_999: "chunk 3"} + rows[bad_row] = {"id": bad_row, "k": 1, "x": float("nan")} + probes = {7: "chunk 1", 150_001: "chunk 2", 249_999: "chunk 3"} + probes.pop(bad_row, None) calls = [] real = GraphBatch._to_java_property_matrix @@ -106,16 +115,42 @@ def spy(self, chunk): db.command("sql", "CREATE VERTEX TYPE V") with db.graph_batch(parallel_flush=False) as batch: assert len(batch.create_vertices("V", rows)) == n - assert calls == [100_000], "only the chunk with the NaN takes the matrix path" - ids = ", ".join(str(i) for i in probes) - query = ( - f"SELECT id, k, k.javaType() AS t FROM V WHERE id IN [{ids}]" # nosec B608 - ) - got = db.query("sql", query).to_list() - by_id = {r["id"]: (r["k"], str(r["t"])) for r in got} - assert set(by_id) == set(probes) - assert {i: by_id[i][0] for i in probes} == {i: i % 1000 for i in probes} - types = {probes[i]: by_id[i][1] for i in probes} + assert calls == [n], "the whole input takes the matrix path, once" + types = _stored_types(db, probes) assert ( len(set(types.values())) == 1 ), f"the stored type depends on the chunk: {types}" + + +def test_a_clean_input_of_several_chunks_is_stored_alike_in_every_chunk( + temp_db_path, monkeypatch +): + n = 250_000 + rows = [{"id": i, "k": i % 1000} for i in range(n)] + probes = {7: "chunk 1", 150_001: "chunk 2", 249_999: "chunk 3"} + calls = [] + real = GraphBatch._to_java_property_matrix + monkeypatch.setattr( + GraphBatch, + "_to_java_property_matrix", + lambda self, chunk: calls.append(len(chunk)) or real(self, chunk), + ) + with arcadedb.create_database(temp_db_path) as db: + db.command("sql", "CREATE VERTEX TYPE V") + with db.graph_batch(parallel_flush=False) as batch: + assert len(batch.create_vertices("V", rows)) == n + assert calls == [], "a clean input never takes the matrix path" + types = _stored_types(db, probes) + assert ( + len(set(types.values())) == 1 + ), f"the stored type depends on the chunk: {types}" + + +def _stored_types(db, probes): + ids = ", ".join(str(i) for i in probes) + query = f"SELECT id, k, k.javaType() AS t FROM V WHERE id IN [{ids}]" # nosec B608 + got = db.query("sql", query).to_list() + by_id = {r["id"]: (r["k"], str(r["t"])) for r in got} + assert set(by_id) == set(probes) + assert {i: by_id[i][0] for i in probes} == {i: i % 1000 for i in probes} + return {probes[i]: by_id[i][1] for i in probes} From 1d5b32e4730652ddde00e9d3736a36f90e6076e9 Mon Sep 17 00:00:00 2001 From: Taewoon Kim Date: Mon, 9 Nov 2026 22:26:21 +0900 Subject: [PATCH 32/49] Tests: a lone surrogate keeps the column off the blob and is refused by the per-element path as before (refs #324) The blob builder returns None for a column with a lone surrogate; the UnicodeEncodeError of the first CI run came from JPype in the per-element path that ran before, which is the refusal every bulk entry point has. The test now checks the builder at the first, middle and last row, the refusal of _column_to_java and insert_columns with nothing written, and that a non-BMP character still takes the blob. Co-Authored-By: Claude Sonnet 5.5 --- tests/test_insert_columns_string_blob.py | 48 +++++++++++++----------- 1 file changed, 27 insertions(+), 21 deletions(-) diff --git a/tests/test_insert_columns_string_blob.py b/tests/test_insert_columns_string_blob.py index 497e755b..40e9b306 100644 --- a/tests/test_insert_columns_string_blob.py +++ b/tests/test_insert_columns_string_blob.py @@ -69,7 +69,6 @@ def test_a_column_of_str_and_none_is_one_string_column(values): ["a", 1.5, None], [1, 2, 3], [None, None], - ["ok", "lone \ud800 surrogate"], [], ], ids=[ @@ -78,7 +77,6 @@ def test_a_column_of_str_and_none_is_one_string_column(values): "str_float", "ints", "only_none", - "lone_surrogate", "empty", ], ) @@ -123,25 +121,33 @@ def test_the_blob_stores_the_table_the_per_element_path_stores( assert _stored(temp_db, "Blob") == [(i, label[i]) for i in range(n)] -def test_a_lone_surrogate_column_is_handled_as_before(temp_db, monkeypatch): - # no UTF-8 form: the column takes the per-element path whatever that path does with it, so the outcome equals the old one - values = ["ok", "lone \ud800 surrogate", "fine"] - cols = {"id": np.arange(3, dtype=np.int64), "label": values} - _make_type(temp_db, "Blob") - _make_type(temp_db, "Ref") - outcomes = [] - for name, force_old in (("Blob", False), ("Ref", True)): - with monkeypatch.context() as mp: - if force_old: - mp.setattr(core, "_string_column", lambda values: None) - try: - temp_db.insert_columns(name, cols) - outcomes.append(("stored", _stored(temp_db, name))) - except ( - Exception - ) as e: # noqa: BLE001 - the type and message are what is compared - outcomes.append((type(e).__name__, str(e).replace(name, "T"))) - assert outcomes[0] == outcomes[1] +@pytest.mark.parametrize("position", [0, 500, 999], ids=["first", "middle", "last"]) +def test_a_lone_surrogate_keeps_the_column_off_the_blob_and_is_refused_as_before( + temp_db, position +): + # a lone surrogate has no UTF-8 form: the blob builder gives up (it never raises), and the per-element path, which is what ran + # before, refuses the column (JPype cannot encode the str) before anything is written. The refusal is the contract of every bulk + # entry point: a lone surrogate raises instead of becoming a question mark (test_bulk_json_paths.py). + values = [f"row {i} \U0001f600" if i % 3 else f"row {i}" for i in range(1_000)] + values[position] = "lone \ud800 surrogate" + assert core._string_column(values) is None + with pytest.raises(UnicodeEncodeError): + core._column_to_java("label", values) + _make_type(temp_db, "S") + with pytest.raises(UnicodeEncodeError): + temp_db.insert_columns( + "S", {"id": np.arange(1_000, dtype=np.int64), "label": values} + ) + count = temp_db.query("sql", "SELECT count(*) AS c FROM S").first().get("c") + assert count == 0 + + +def test_a_non_bmp_character_still_takes_the_blob(): + # U+1F600 and U+10000 are one code point each and a surrogate PAIR in Java, but valid UTF-8: the blob carries them + values = ["a", "\U0001f600", "\U00010000\U0010ffff", None, "\ud7ff\ue000"] + assert core._string_column(values) is not None + array, n = core._column_to_java("label", values) + assert n == 5 and isinstance(array, _string_column_class()) @pytest.mark.parametrize( From 98d0bab76f43e33ee0533528053ca598f6e16de4 Mon Sep 17 00:00:00 2001 From: Taewoon Kim Date: Mon, 9 Nov 2026 22:31:36 +0900 Subject: [PATCH 33/49] integration: consistency fixes Docstrings that still described the 26.11.1 routes after the combined PRs: - core.py Database.insert_arrow: a vertex type no longer loads through GraphBatch (#335); it loads through the columnar insert like a document type. - _native/database.py Database.insert_dataframe: batches go through the columnar insert (insert_many only when a column cannot cross as memory) for a document and a vertex type alike (#333, #335). - dataframe.py: module docstring dropped 'No Java was added' (this candidate changes BulkOps.java and DocumentBatcher.java for this path) and lists the kinds that now cross as memory (#333); _insert_documents_native docstring covers vertices and the temporal, decimal and bytes columns. Co-Authored-By: Claude Sonnet 5.5 --- src/arcadedb/_native/database.py | 5 +++-- src/arcadedb/core.py | 7 ++++--- src/arcadedb/dataframe.py | 14 +++++++------- 3 files changed, 14 insertions(+), 12 deletions(-) diff --git a/src/arcadedb/_native/database.py b/src/arcadedb/_native/database.py index adb7e95c..19446dbc 100644 --- a/src/arcadedb/_native/database.py +++ b/src/arcadedb/_native/database.py @@ -1503,8 +1503,9 @@ def insert_dataframe( self, type_name, df, *, batch_size: int = 10_000, parallel: bool = False ) -> int: """A pandas or Polars DataFrame into a type, in batches: the strict Arrow loading of ``Database.insert_dataframe`` - (dtype refusals, declared types, NaN rules, the committed-rows message), batches through :meth:`insert_many` for a - document type and a vertex type alike (a vertex load refuses to run inside a transaction the caller opened; ``arcadedb/dataframe.py``). + (dtype refusals, declared types, NaN rules, the committed-rows message), batches through the columnar insert + (:meth:`insert_columns`; :meth:`insert_many` for a batch whose columns cannot cross as memory) for a document type and a + vertex type alike (a vertex load refuses to run inside a transaction the caller opened; ``arcadedb/dataframe.py``). """ from ..dataframe import insert_table, to_arrow_table diff --git a/src/arcadedb/core.py b/src/arcadedb/core.py index a1024e91..a050b72f 100644 --- a/src/arcadedb/core.py +++ b/src/arcadedb/core.py @@ -1343,9 +1343,10 @@ def insert_arrow( The Arrow counterpart of :meth:`insert_columns`: each Arrow column is converted to the typed array ``insert_columns`` sends across the bridge - (``DocumentBatcher.insertColumns``), so a document type loads at that - path's speed; a vertex type loads through ``GraphBatch``. Documents and - vertices only: an edge type is refused. + (``DocumentBatcher.insertColumns``), so a document type and a vertex + type load at that path's speed (a vertex type is not loaded through + ``GraphBatch``; a vertex load refuses to run inside a transaction the + caller opened). Documents and vertices only: an edge type is refused. A property the type does not declare takes the stored type below; a declared property converts to its declared type. diff --git a/src/arcadedb/dataframe.py b/src/arcadedb/dataframe.py index 9b52dd56..d9f42013 100644 --- a/src/arcadedb/dataframe.py +++ b/src/arcadedb/dataframe.py @@ -8,15 +8,15 @@ sends across the bridge, for a document type and for a vertex type alike (the vertex type gets vertices from the same ``DocumentBatcher.insertColumns``; it refuses to run inside a transaction the caller opened, as ``GraphBatch`` does). -No Java was added. The native backend (the ``arcadedb`` wheel) shares everything but the last step: the checks, the type mapping, the batching and the error wording are the same code. A batch of documents or vertices goes over by address through the columnar entry (``arcadedb_n_insert_colp``) when every column can cross as -memory, else as one JSON text through ``Database.insert_many`` (one crossing per -10,000 rows), where a value JSON cannot carry (a date-time, a decimal, bytes) -travels as a tagged value the image decodes. +memory (numbers, bools, strings, float vectors, date-times, dates, decimals and bytes), +else as one JSON text through ``Database.insert_many`` (one crossing per 10,000 rows), +where a value JSON cannot carry (a date-time, a decimal, bytes) travels as a tagged +value the image decodes. """ import re @@ -506,9 +506,9 @@ def _insert_documents_native( parallel: bool, declared: Optional[Dict[str, str]] = None, ): - """One batch of documents on the native backend. When every column can cross as memory (numbers, bools, strings, nulls, - float vectors for an ``ARRAY_OF_FLOATS`` property) the batch goes through the columnar entry, no Python values and no JSON - text; otherwise the columns become one dict per row and go to ``insert_many`` (one crossing per 10,000 rows). Either way + """One batch of documents or vertices on the native backend. When every column can cross as memory (numbers, bools, strings, + nulls, date-times, dates, decimals, bytes, float vectors for an ``ARRAY_OF_FLOATS`` property) the batch goes through the + columnar entry, no Python values and no JSON text; otherwise the columns become one dict per row and go to ``insert_many`` (one crossing per 10,000 rows). Either way the batch is one transaction, or joins the caller's. With ``parallel`` the same columnar entry hands the records to the async writers (no transaction), and the call waits for them and raises when a record failed, as ``insert_many(parallel=True)`` does. """ From ea6dd0cd08f5d8347b5d99e67962a1c52b2b9127 Mon Sep 17 00:00:00 2001 From: Taewoon Kim Date: Mon, 9 Nov 2026 23:01:07 +0900 Subject: [PATCH 34/49] Native frames: a null vector is a NaN row in the to_columns matrix whatever the writer put there The decoder set no value for the null rows of a vector column and read what the frame held, which is zeros in a hand-built frame and NaN only where the writer chose NaN; a zero row reads as a real zero vector. decode_columns_frame now sets the null rows to NaN itself, so the single-frame matrix, the matrix after a merge with other frames and the first-reply frame agree, and the array stays writable. Co-Authored-By: Claude Sonnet 5.5 --- src/arcadedb/_columnar.py | 6 ++++-- tests/test_column_frames.py | 13 ++++++++++++- 2 files changed, 16 insertions(+), 3 deletions(-) diff --git a/src/arcadedb/_columnar.py b/src/arcadedb/_columnar.py index 75dd8f5e..c33ff0e8 100644 --- a/src/arcadedb/_columnar.py +++ b/src/arcadedb/_columnar.py @@ -287,14 +287,16 @@ def decode_columns_frame( else: values = arr elif ctype in ("f4v", "f8v"): - # fixed-dimension vector column -> 2-D array (count, dim); the writer puts NaN rows into null slots; - # an empty vector has no "dim" in the header + # fixed-dimension vector column -> 2-D array (count, dim); an empty vector has no "dim" in the header. The null rows are + # made NaN here, whatever the writer put in the slot: a row of zeros would read as a real zero vector dim = col.get("dim", 0) dt = " Date: Tue, 10 Nov 2026 04:17:34 +0900 Subject: [PATCH 35/49] insert_arrow and insert_dataframe on the native wheel: a vector column stays an Arrow array, and a batch is dropped before the next is built (refs #319, #316) Measured on the post1 candidate-3 native wheel (cores 12-15, 3 clean cells per arm, tracemalloc around the whole insert_arrow call): a 10,000-row batch with a 64-dimension vector column held 22.9 MiB of Python objects for the whole crossing, 96 dimensions 34.8 MiB, because _column_values turned every Arrow list column into Python values (to_pylist: one float object per element) and the columnar entry then packed that list into a float32 matrix. The Arrow column goes to columnar.pack as it is now (it was already able to lay out a fixed-size or list column of numbers for an ARRAY_OF_FLOATS property); a column that cannot cross as memory (a null list, a LIST property, a batch with another column the entry refuses) still gets its Python values at the fallback, in _arrow_row_values, as before. The Python-side peak of those calls is 0.0 MiB now and the call is about 2 times faster (product vertices 1.16 -> 0.56 s for 150,000 rows, article vertices 1.03 -> 0.41 s). insert_table also drops the batch and its columns at the end of each iteration, so two batches are never alive together. What this does not change: the anonymous memory at the end of the window (RssAnon) of those loads. It is the image's heap, not Python: packed buffers are 2.6 MiB per 10,000 rows (zero-copy views of the Arrow buffers) and the date column of the LineItem load packs in 0.03 MiB; see .notes/bench/repros/bindings-native-memory-20261110/README.md. Tests: tests/test_native_columnar_memory.py (native only, registered in conftest.py) pins that the vector column stays Arrow on the native load and a list on the JVM load, that Arrow and Python lists lay out the same bytes (also for a sliced table), that the Python-side peak of a vector batch and of a whole load is a fraction of the Python-float copy (ratios, no MiB limits), that columnar.pack runs once per batch with at most batch_size rows and its arrays are gone before the next batch is packed, and that nulls and LIST properties still take the row path with the same stored values. Co-Authored-By: Claude Sonnet 5.5 --- src/arcadedb/dataframe.py | 28 ++- tests/conftest.py | 2 + tests/test_native_columnar_memory.py | 300 +++++++++++++++++++++++++++ 3 files changed, 326 insertions(+), 4 deletions(-) create mode 100644 tests/test_native_columnar_memory.py diff --git a/src/arcadedb/dataframe.py b/src/arcadedb/dataframe.py index 9b52dd56..6fffe536 100644 --- a/src/arcadedb/dataframe.py +++ b/src/arcadedb/dataframe.py @@ -322,14 +322,28 @@ def _scalars_to_tagged(kind: str, chunk): ] +def _is_number_list(pa, arrow_type) -> bool: + """A list, large list or fixed-size list of integers or floats: the column the columnar entry lays out as a float32 matrix.""" + types = pa.types + if not ( + types.is_list(arrow_type) + or types.is_large_list(arrow_type) + or types.is_fixed_size_list(arrow_type) + ): + return False + return types.is_floating(arrow_type.value_type) or types.is_integer( + arrow_type.value_type + ) + + def _column_values(name: str, chunk, pa, native: bool = False, arrow: bool = False): """One Arrow array as what ``Database.insert_columns`` takes: a numpy array when the column crosses as one buffer (integers, floats and booleans without nulls), else a list of values (``None`` is a null). - ``arrow`` (the native backend's document load): a string, binary, timestamp, date or decimal column, and a numeric one with - nulls, stay Arrow arrays, which the columnar entry reads in place (the offsets, the bytes, the integer ticks and the validity - bitmap) instead of as Python values; ``_insert_documents_native`` turns them into tagged values for the row path when a batch - cannot go columnar. + ``arrow`` (the native backend's document and vertex load): a string, binary, timestamp, date or decimal column, a list of + numbers (a vector) and a numeric column with nulls, stay Arrow arrays, which the columnar entry reads in place (the offsets, + the bytes, the integer ticks, the vector values and the validity bitmap) instead of as Python values; ``_insert_documents_native`` + turns them into Python or tagged values for the row path when a batch cannot go columnar. """ types = pa.types if types.is_dictionary(chunk.type): # a pandas categorical or a Polars Categorical @@ -366,6 +380,8 @@ def _column_values(name: str, chunk, pa, native: bool = False, arrow: bool = Fal return chunk.to_pylist() if arrow and kind in ("timestamp", "date", "decimal", "binary"): return chunk + if arrow and kind == "list" and _is_number_list(pa, chunk.type): + return chunk # a vector column: the columnar entry reads the values in place (to_pylist makes one Python float per element) if kind in ("timestamp", "date"): return ( _temporal_to_tagged(chunk, pa) if native else _temporal_to_java(chunk, pa) @@ -480,6 +496,10 @@ def insert_table( ) from exc raise written += batch.num_rows + del ( + columns, + batch, + ) # sent: drop this batch's columns before the next batch is built, so two batches are never alive together return written diff --git a/tests/conftest.py b/tests/conftest.py index 11c528e4..af18bb6f 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -57,6 +57,8 @@ collect_ignore.append("test_native_columnar_temporal_review_fixes.py") # a vertex load by insert_dataframe or insert_arrow crosses as columns, one crossing per batch, observed on the native library collect_ignore.append("test_native_vertex_columnar.py") + # a document or vertex load keeps its transient Python memory to one batch and a vector column is never turned into Python floats + collect_ignore.append("test_native_columnar_memory.py") # the rows of an insert_many chunk with date-times, Decimals, bytes and numpy scalars are told apart by the entry points they cross collect_ignore.append("test_native_mixed_chunk.py") # the native build's ingest defaults (insert_many and graph_batch commit size), observed on the native library diff --git a/tests/test_native_columnar_memory.py b/tests/test_native_columnar_memory.py new file mode 100644 index 00000000..b3ed404c --- /dev/null +++ b/tests/test_native_columnar_memory.py @@ -0,0 +1,300 @@ +"""Native: a document or vertex load by ``insert_arrow`` or ``insert_dataframe`` keeps its transient memory to one batch, and a vector column +is never turned into Python floats on its way to the columnar entry (ArcadeData/arcadedb-python #333 and #335, the extra anonymous memory of +the post1 candidate's native wheel). + +Three properties are pinned, none of them with an absolute size: + +* ``_column_values(..., arrow=True)`` returns an Arrow list of numbers (a vector column) as the Arrow array, which ``columnar.pack`` reads in + place; the JVM build (``arrow=False``) still gets Python lists. One ``to_pylist()`` of a 64-dimensional vector column makes 64 Python + floats per row (about 2 KB a row), which a 10,000-row batch held for the whole crossing. +* the Python-side peak of a whole load is a small fraction of what one batch of Python floats costs (``tracemalloc`` sees numpy and Python + objects, not Arrow's memory or the image's heap, which is why the ratio is generous). +* ``columnar.pack`` is called once per batch and never with more than ``batch_size`` rows, and what it laid out is released when the batch + is sent (the arrays of batch k are gone before batch k + 1 is packed). + +Needs the native library, so conftest.py collects this file on the native backend only. +""" + +import ctypes +import gc +import tracemalloc +import weakref + +import numpy as np +import pyarrow as pa +import pytest + +from arcadedb import dataframe +from arcadedb._native import columnar + +DIM = 64 + + +def _vectors(n, start=0, dim=DIM): + r = np.arange(start, start + n, dtype=np.int64) + j = np.arange(dim, dtype=np.int64) + return ( + ((r[:, None] * 131 + j[None, :] * 17) % 1000).astype(np.float32) + ) / np.float32(1000.0) + + +def _fixed(mat): + return pa.FixedSizeListArray.from_arrays( + pa.array(mat.reshape(-1), type=pa.float32()), mat.shape[1] + ) + + +def _bytes(addr_len): + addr, length = addr_len + return ctypes.string_at(addr, length) if length else b"" + + +def _peak(fn): + gc.collect() + tracemalloc.start() + try: + tracemalloc.reset_peak() + base = tracemalloc.get_traced_memory()[0] + fn() + return tracemalloc.get_traced_memory()[1] - base + finally: + tracemalloc.stop() + + +@pytest.mark.parametrize( + "make", + [ + lambda m: _fixed(m), + lambda m: pa.array(m.astype(np.float64).tolist(), type=pa.list_(pa.float64())), + lambda m: pa.array(m.astype(np.int32).tolist(), type=pa.large_list(pa.int32())), + ], + ids=["fixed_size_list_f32", "list_f64", "large_list_i32"], +) +def test_a_vector_column_stays_an_arrow_array_on_the_native_load(make): + chunk = make(_vectors(10)) + kept = dataframe._column_values("embedding", chunk, pa, native=True, arrow=True) + assert isinstance(kept, pa.Array) and kept.type == chunk.type + assert ( + dataframe._column_values("embedding", chunk, pa, native=True, arrow=False) + == chunk.to_pylist() + ) + assert isinstance( + dataframe._column_values("embedding", chunk, pa, native=False, arrow=False), + list, + ) + + +def test_a_list_that_is_not_a_list_of_numbers_still_becomes_python_values(): + for chunk in ( + pa.array([["a", "b"], ["c"]]), + pa.array([[{"x": 1}], [{"x": 2}]]), + pa.array([[[1.0], [2.0]], [[3.0]]]), + ): + got = dataframe._column_values("c", chunk, pa, native=True, arrow=True) + assert isinstance(got, list) and got == chunk.to_pylist() + + +def test_the_vector_layout_from_arrow_is_the_layout_from_python_lists(): + mat = _vectors(300) + table = pa.table({"embedding": _fixed(mat)}).slice( + 7, 211 + ) # a slice: the offset of the array is not zero + declared = columnar.Declared(known={"embedding": "ARRAY_OF_FLOATS"}) + kept = dataframe._column_values( + "embedding", table.column(0).combine_chunks(), pa, True, True + ) + listed = dataframe._column_values( + "embedding", table.column(0).combine_chunks(), pa, True, False + ) + rows_a, (col_a,) = columnar.pack({"embedding": kept}, declared) + rows_b, (col_b,) = columnar.pack({"embedding": listed}, declared) + assert ( + rows_a == rows_b == 211 + and col_a.kind == col_b.kind == "f" + and col_a.dim == col_b.dim == DIM + ) + assert _bytes(col_a.data) == _bytes(col_b.data) == mat[7:218].tobytes() + + +def test_the_python_side_peak_of_a_vector_batch_is_a_fraction_of_the_python_float_copy(): + batch = _fixed(_vectors(10_000)) + declared = columnar.Declared(known={"embedding": "ARRAY_OF_FLOATS"}) + as_python = _peak( + lambda: columnar.pack( + { + "embedding": dataframe._column_values( + "embedding", batch, pa, True, False + ) + }, + declared, + ) + ) + in_place = _peak( + lambda: columnar.pack( + {"embedding": dataframe._column_values("embedding", batch, pa, True, True)}, + declared, + ) + ) + assert ( + as_python > 10 * 1024 * 1024 + ), as_python # 640,000 Python floats: the copy this change removes (about 20 MiB) + assert in_place * 8 < as_python, ( + in_place, + as_python, + ) # measured: about 100 times less + + +class _PackSpy: + """Wraps ``columnar.pack``: the row count of every call and a weak reference to every array it laid out.""" + + def __init__(self, monkeypatch): + self.rows, self.arrays, self.alive_at_next_call = [], [], [] + real = columnar.pack + + def pack(columns, declared): + gc.collect() + self.alive_at_next_call.append( + sum(ref() is not None for refs in self.arrays for ref in refs) + ) + out = real(columns, declared) + self.rows.append(None if out is None else out[0]) + self.arrays.append( + [weakref.ref(c.keep[0]) for c in out[1]] if out is not None else [] + ) + return out + + monkeypatch.setattr(columnar, "pack", pack) + + +def _vector_type(db, kind="VERTEX"): + db.command("sql", f"CREATE {kind} TYPE V") + db.command("sql", "CREATE PROPERTY V.id LONG") + db.command("sql", "CREATE PROPERTY V.embedding ARRAY_OF_FLOATS") + + +@pytest.mark.parametrize("kind", ["VERTEX", "DOCUMENT"]) +def test_every_batch_is_packed_once_and_released_before_the_next( + temp_db, monkeypatch, kind +): + _vector_type(temp_db, kind) + spy = _PackSpy(monkeypatch) + n, batch = 2_505, 1_000 + table = pa.table( + {"id": pa.array(np.arange(n, dtype=np.int64)), "embedding": _fixed(_vectors(n))} + ) + assert temp_db.insert_arrow("V", table, batch_size=batch) == n + assert spy.rows == [ + 1_000, + 1_000, + 505, + ] # one pack per batch, never more than batch_size rows + assert spy.alive_at_next_call == [ + 0, + 0, + 0, + ], spy.alive_at_next_call # batch k is gone when batch k + 1 is packed + gc.collect() + assert not any( + ref() is not None for refs in spy.arrays for ref in refs + ) # and the last one when the call returns + + +def test_a_dataframe_load_is_packed_per_batch_too(temp_db, monkeypatch): + import pandas as pd + + temp_db.command("sql", "CREATE VERTEX TYPE P") + temp_db.command("sql", "CREATE PROPERTY P.id LONG") + temp_db.command("sql", "CREATE PROPERTY P.name STRING") + spy = _PackSpy(monkeypatch) + frame = pd.DataFrame( + {"id": np.arange(250, dtype=np.int64), "name": [f"p{i}" for i in range(250)]} + ) + assert temp_db.insert_dataframe("P", frame, batch_size=100) == 250 + assert spy.rows == [100, 100, 50] and spy.alive_at_next_call == [0, 0, 0] + + +def test_the_python_side_peak_of_a_whole_vector_load_stays_below_one_batch_of_python_floats( + temp_db, +): + _vector_type(temp_db) + n, batch = 40_000, 10_000 + table = pa.table( + {"id": pa.array(np.arange(n, dtype=np.int64)), "embedding": _fixed(_vectors(n))} + ) + one_batch_of_python_floats = _peak( + lambda: table.slice(0, batch).column("embedding").to_pylist() + ) + whole_load = _peak(lambda: temp_db.insert_arrow("V", table, batch_size=batch)) + assert one_batch_of_python_floats > 10 * 1024 * 1024 + assert whole_load * 2 < one_batch_of_python_floats, ( + whole_load, + one_batch_of_python_floats, + ) # measured: about 7 times less + assert temp_db.query("sql", "SELECT count(*) AS n FROM V").first().get("n") == n + + +def _stored(db, key): + return [ + float(x) + for x in db.query("sql", "SELECT embedding FROM V WHERE id = ?", key) + .first() + .get("embedding") + ] + + +@pytest.mark.parametrize("kind", ["VERTEX", "DOCUMENT"]) +def test_the_stored_vectors_are_the_ones_the_python_lists_stored(temp_db, kind): + _vector_type(temp_db, kind) + mat = _vectors(120) + table = pa.table( + {"id": pa.array(np.arange(120, dtype=np.int64)), "embedding": _fixed(mat)} + ) + assert ( + temp_db.insert_arrow("V", table.slice(5, 100), batch_size=30) == 100 + ) # a sliced table, four batches + for key in (5, 6, 34, 35, 64, 104): + assert ( + max(abs(a - b) for a, b in zip(_stored(temp_db, key), mat[key].tolist())) + < 1e-6 + ) + assert temp_db.query("sql", "SELECT count(*) AS n FROM V").first().get("n") == 100 + + +def test_a_vector_column_with_a_null_or_a_list_property_takes_the_row_path_with_the_same_values( + temp_db, +): + _vector_type(temp_db) + temp_db.command("sql", "CREATE VERTEX TYPE L") + temp_db.command("sql", "CREATE PROPERTY L.id LONG") + temp_db.command("sql", "CREATE PROPERTY L.tags LIST") + with_null = pa.table( + { + "id": pa.array([1, 2, 3], type=pa.int64()), + "embedding": pa.array( + [[0.5, 0.25], None, [1.0, 2.0]], type=pa.list_(pa.float32()) + ), + } + ) + assert temp_db.insert_arrow("V", with_null) == 3 + assert [ + float(x) + for x in temp_db.query("sql", "SELECT embedding FROM V WHERE id = 1") + .first() + .get("embedding") + ] == [0.5, 0.25] + assert ( + temp_db.query("sql", "SELECT embedding FROM V WHERE id = 2") + .first() + .get("embedding") + is None + ) + tags = pa.table( + { + "id": pa.array([1, 2], type=pa.int64()), + "tags": pa.array([[1, 2, 3], [4]], type=pa.list_(pa.int64())), + } + ) + assert temp_db.insert_arrow("L", tags) == 2 + assert list( + temp_db.query("sql", "SELECT tags FROM L WHERE id = 1").first().get("tags") + ) == [1, 2, 3] From 9fd4334f2a50ac77390d9fe38118502acebdbac8 Mon Sep 17 00:00:00 2001 From: Taewoon Kim Date: Tue, 10 Nov 2026 04:28:57 +0900 Subject: [PATCH 36/49] create_vertices: no text scan for a table of scalars (g1 regression of candidate 3), vector rows cross in small blocks (refs #337, #318) Measured cause of the 31 percent g1 regression (5,000-row Person load, JVM wheel, candidate 3 against 26.11.1): _dumps_once ran _INT64_OVERFLOW.search over the 330 KB JSON text of every chunk, 13 ms of a 43 ms call (2.5 us a row; the regex starts with a lookbehind and an alternation at every byte, no literal prefix). cProfile: 60 re.Pattern.search calls, 0.29 s per 20 calls. The 26.11.1 per-value loop it replaced cost 1.7 ms. Fix: the columns decide, in C, and the text is not scanned. - _rows_need_walk takes the type census and screens each column of exact scalars (int range by min and max, UTF-8 form of the joined strs); both regexes are gone. - GraphBatch.create_vertices screens the whole input once (screen_vertex_rows, for one chunk or several). When every column is exact scalars and clean the verdict is vetted and the chunks are written by json_bulk_dumps(vetted=True): json.dumps, nothing else (also no subnormal text search). A bad scalar anywhere still sends the whole input the property-matrix way, and the first bad row still gives the old error text (the walk is unchanged and still the reference). - classify_vertex_rows keeps its signature and result; the census and the screen share one list per column. Memory of the vector vertex load: rows with vector columns cross in blocks of about 1 MiB of floats (_VECTOR_BLOCK_FLOATS, at least 512 rows), and the previous chunk's rest rows, text and matrices are released before the next chunk is built (two float[] buffers were alive at once). Python only; the Java copies each row's vector as before. Tests: the regex pattern test becomes a column-screen test over the same corpus; spies that a vetted table runs neither the census nor the walk per chunk and that other columns still do; the block sizes of a vector load; vector_width; the release of the previous chunk. Differential check against the 26.11.1 validation loop (36,000 cases) is in the .notes harness. Co-Authored-By: Claude Sonnet 5.5 --- src/arcadedb/graph_batch.py | 58 +++-- src/arcadedb/type_conversion.py | 231 +++++++++++------- tests/test_bulk_single_pass_validation.py | 120 ++++++++- tests/test_graph_batch_single_pass.py | 55 ++++- .../test_graph_batch_vector_crossings_jvm.py | 81 ++++++ 5 files changed, 427 insertions(+), 118 deletions(-) diff --git a/src/arcadedb/graph_batch.py b/src/arcadedb/graph_batch.py index 75313d08..dbd9ba7c 100644 --- a/src/arcadedb/graph_batch.py +++ b/src/arcadedb/graph_batch.py @@ -12,7 +12,6 @@ from .exceptions import ArcadeDBError from .graph import Document, Vertex from .type_conversion import ( - classify_vertex_rows, convert_python_to_java, declared_float_vectors, declared_scalars_ok, @@ -20,7 +19,9 @@ json_bulk_dumps_tagged, json_bulk_scalar_ok, json_bulk_shape_ok, + screen_vertex_rows, split_float_vectors, + vector_width, ) _LOGGER = get_logger(__name__) @@ -251,6 +252,12 @@ def create_vertices( ) from e _BULK_CHUNK = 100_000 # rows per boundary crossing in bulk paths + # Rows with vector columns cross in blocks of about this many floats (1 MiB of float32) over all their vector columns, at least + # _VECTOR_BLOCK_MIN_ROWS rows: 10,000 rows of 96 floats cross in 4 blocks, not in one. The engine builds the vertices of a crossing + # from the JSON text, one float[] buffer and one copy of every row's vector, and a smaller crossing keeps less of that alive when + # a collection runs (the peak RSS of a load of Java float[] vectors was 1.1 to 1.3 times the 26.11.1 wheel's, which never copies them). + _VECTOR_BLOCK_FLOATS = 262_144 + _VECTOR_BLOCK_MIN_ROWS = 512 def _create_vertices_json_bulk(self, type_name, rows): """Bulk path: rows cross as one JSON string, RIDs return as one joined @@ -263,31 +270,52 @@ def _create_vertices_json_bulk(self, type_name, rows): json_bulk_dumps_tagged writes and VertexBatcher decodes (the stored value is what the property-matrix path stores). An input whose values are all scalars is written by json_bulk_dumps. The values are checked - in the one json.dumps pass of each chunk (see type_conversion._dumps_once), - not in a loop over the rows before it: a chunk holding a value the text - cannot carry (a NaN, an integer beyond 64 bits, a lone surrogate, a - subnormal, or a date-time inside a list) goes through the property-matrix - path on its own. Returns None when unavailable/unsuitable (a property - name that is not a str, a type the text does not carry) so the caller - falls back to the property-matrix path for the whole input.""" + by their types and columns, in C (type_conversion.screen_vertex_rows), + not in a loop over the rows: a scalar the text would change (a NaN, an + integer beyond 64 bits, a lone surrogate, a subnormal) anywhere in the + input sends the whole input through the property-matrix path, and + rows of exact scalars only are then written with no further check. + Other rows are checked in the one json.dumps pass of each chunk (see + type_conversion._dumps_once); a chunk holding a date-time inside a + list, or a vector column the matrix cannot carry, goes through the + property-matrix path on its own. Returns None when unavailable/unsuitable + (a property name that is not a str, a type the text does not carry) so + the caller falls back to the property-matrix path for the whole input.""" try: vertex_batcher = jpype.JClass("com.arcadedb.python.VertexBatcher") declared = set(declared_float_vectors(self._java_db, type_name)) except Exception: return None - tagged = classify_vertex_rows(rows, declared, len(rows) > self._BULK_CHUNK) - if tagged is None: + verdict = screen_vertex_rows(rows, declared) + if verdict is None: # a property name that is not a str (json.dumps would write 1 as "1" where the matrix path refuses it), or a type the # tagged JSON does not carry (a set, a numpy or pandas value, any subclass, a list of only numbers: the matrix path # converts those) return None - - dumps = json_bulk_dumps_tagged if tagged else json_bulk_dumps - # chunked so huge ingests never materialize one giant JSON string + tagged, vetted = verdict + + if vetted: + # every value is an exact scalar the text carries unchanged (screened in C above): the chunks are written without a check + def dumps(chunk_rows): + return json_bulk_dumps(chunk_rows, vetted=True) + + else: + dumps = json_bulk_dumps_tagged if tagged else json_bulk_dumps + # chunked so huge ingests never materialize one giant JSON string; rows that hold vectors cross in blocks of about + # _VECTOR_BLOCK_FLOATS floats per crossing, so the float[] buffer of a matrix and what the engine builds from it stay small + step = self._BULK_CHUNK + width = vector_width(rows, declared) + if width: + step = min( + step, + max(self._VECTOR_BLOCK_MIN_ROWS, self._VECTOR_BLOCK_FLOATS // width), + ) rids = [] - for start in range(0, len(rows), self._BULK_CHUNK): - chunk = rows[start : start + self._BULK_CHUNK] + for start in range(0, len(rows), step): + # the previous chunk's rest rows, text and matrices go before this chunk's are built: two float[] buffers were alive at once + split = rest = text = None + chunk = rows[start : start + step] split = split_float_vectors( self._java_db, type_name, chunk, declared, undeclared=True ) diff --git a/src/arcadedb/type_conversion.py b/src/arcadedb/type_conversion.py index 84dd0fa4..557100f5 100644 --- a/src/arcadedb/type_conversion.py +++ b/src/arcadedb/type_conversion.py @@ -806,26 +806,9 @@ def _dumps_checked(rows: Any, **kwargs: Any) -> str: return text -def _int64_overflow_pattern() -> str: - """The digit runs of 2**63 and above, which a JSON integer token beyond 64 bits has. - - A run of 20 digits or more, or of 19 digits that is not below 9223372036854775808. A run that follows a digit or a "." is not an - integer token (the digits after the point of a float such as 0.00012345678901234568 are no token), so only a run start counts. - """ - limit = str(2**63) - parts = [r"[0-9]{20,}"] - for i, digit in enumerate(limit): - low = int(digit) if i == len(limit) - 1 else int(digit) + 1 - if low <= 9: - parts.append(f"{limit[:i]}[{low}-9][0-9]{{{len(limit) - 1 - i}}}") - return r"(? bool: def _rows_need_walk(rows: Any, tagged: bool) -> bool: - """Whether the types of the rows leave a check that the JSON text cannot show: only exact dict rows with str keys and values of - the exact scalar types (and, tagged, the kinds bulk_tag_default checks) are decided by the text. The census runs in C: one pass - over the keys and one over the values, no Python-level loop per row. + """Whether the rows may hold a value the JSON text would change, decided from their types and their columns, in C. + + Only exact dict rows with str keys can be decided here. Each column is one list of its values and one census of their types; + a column of the exact scalar types is screened (`_scalar_column_suspect`: the 64-bit range of its ints, the UTF-8 form of its + strs), a column of the kinds bulk_tag_default writes (tagged) holds nothing the walk has to see, and any other type, or a + column of mixed kinds, sends the rows to the walk. A false alarm costs the walk and changes nothing else. This replaces two + scans of the JSON text with regular expressions (a lone surrogate, an integer token of 2**63 and above): the one for the + integer token cost 2.5 microseconds a row of a three-column table (12 ms of a 5,000-row call), a column screen about 0.2. """ if not set(map(type, rows)) <= _DICT_ONLY: return True - if not set(map(type, itertools.chain.from_iterable(rows))) <= _STR_ONLY: + keys = set(itertools.chain.from_iterable(rows)) + if not set(map(type, keys)) <= _STR_ONLY: return True - extra = ( - set(map(type, itertools.chain.from_iterable(map(dict.values, rows)))) - - _BULK_SCALAR_TYPES - ) - if not extra: - return False - return not (tagged and all(map(_kind_is_tagged, extra))) + for key in keys: + column = list(map(dict.get, rows, itertools.repeat(key))) + kinds = set(map(type, column)) + extra = kinds - _BULK_SCALAR_TYPES + if extra: + if not (tagged and all(map(_kind_is_tagged, extra))): + return True + if kinds - extra - {_NONE_KIND}: + return True # tagged kinds beside scalars in one column: the walk judges each value + continue # bulk_tag_default checks these as json.dumps writes them + if not kinds <= _TEXT_SAFE and _scalar_column_suspect(column, kinds): + return True + return False + +def _dumps_once(rows: Any, tagged: bool, vetted: bool = False) -> str: + """One json.dumps pass for the bulk paths, with the checks of `_check_rows` decided from the types and columns of the rows. -def _dumps_once(rows: Any, tagged: bool) -> str: - """One json.dumps pass for the bulk paths, with the checks of `_check_rows` decided from what that pass shows. + A str or int the text would change cannot be seen by a json.dumps hook (the encoder writes both itself), so the rows decide: + `_rows_need_walk` screens their columns in C and sends the rows to the walk when a value may be bad or its type is one it cannot + vouch for. When json.dumps itself refuses the rows (NaN, an unknown type, a subnormal, a hook error) the walk runs first, as the + validation loop used to run before json.dumps, so the error of the first bad row in row order is still the walk's; if the walk + finds nothing, json.dumps's own error stands. A false alarm costs the walk and changes nothing else. - A str or int the text would change cannot be seen by a json.dumps hook (the encoder writes both itself), so the text decides: - a surrogate escape (a lone surrogate) or an integer token of 2**63 and above sends the rows to the walk, and so do rows whose - types the text cannot vouch for (`_rows_need_walk`). When json.dumps itself refuses the rows (NaN, an unknown type, a - subnormal, a hook error) the walk runs first, as the validation loop used to run before json.dumps, so the error of the first - bad row in row order is still the walk's; if the walk finds nothing, json.dumps's own error stands. A false alarm costs the walk - and changes nothing else. + `vetted` is for a caller that has already screened these rows with the same census (`screen_vertex_rows`: every value an exact + scalar, no NaN, Infinity, subnormal, integer beyond 64 bits or lone surrogate): the text is written without a second look. """ try: - text = ( - _dumps_checked(rows, default=bulk_tag_default) - if tagged - else _dumps_checked(rows) - ) + if vetted: + text = json.dumps(rows, allow_nan=False) + elif tagged: + text = _dumps_checked(rows, default=bulk_tag_default) + else: + text = _dumps_checked(rows) except (TypeError, ValueError) as error: refused = error else: @@ -929,22 +927,19 @@ def _dumps_once(rows: Any, tagged: bool) -> str: raise refused finally: del refused - if ( - _rows_need_walk(rows, tagged) - or _SURROGATE_ESCAPE.search(text) - or _INT64_OVERFLOW.search(text) - ): + if not vetted and _rows_need_walk(rows, tagged): _check_rows(rows, tagged) return text -def json_bulk_dumps(rows: Any) -> str: +def json_bulk_dumps(rows: Any, vetted: bool = False) -> str: """json.dumps for the bulk paths: raises ValueError (or TypeError) when any value in `rows` would change on the way to the engine. - `rows` is a list of dicts (one per record). The rows are written once; see `_dumps_once` for how the values are checked. + `rows` is a list of dicts (one per record). The rows are written once; see `_dumps_once` for how the values are checked and + for `vetted`. """ - return _dumps_once(rows, False) + return _dumps_once(rows, False, vetted) # --------------------------------------------------------------------------- @@ -1067,14 +1062,15 @@ def _column(rows: Any, key: str): _SCREEN_BLOCK = 100_000 # rows per block of the scalar screen, so its transient lists and joined texts stay bounded -def _scalar_column_suspect(column: list) -> bool: +def _scalar_column_suspect(column: list, kinds: Any = None) -> bool: """Whether one block of a column of exact scalar values may hold a value the JSON text would change. Conservative (a false alarm only sends the column to the exact per-value judgement) and run in C: the type census, the joined text of a str column (a lone surrogate has no UTF-8 form), min and max of an int column (the 64-bit range), and a float column as - one float64 array (NaN, Infinity, a subnormal). Columns of mixed kinds are suspect. + one float64 array (NaN, Infinity, a subnormal). Columns of mixed kinds are suspect. `kinds` is the set of the types of `column` + when the caller has taken that census already (it is not changed). """ - kinds = set(map(type, column)) + kinds = set(map(type, column)) if kinds is None else set(kinds) if _NONE_KIND in kinds: column = [value for value in column if value is not None] kinds.discard(_NONE_KIND) @@ -1103,50 +1099,52 @@ def _scalar_column_suspect(column: list) -> bool: return True -def _scalar_screen(rows: Any, key: str) -> bool: - """Whether the column `key` of all `rows`, which holds exact scalars only, may hold a value the JSON text would change.""" - for start in range(0, len(rows), _SCREEN_BLOCK): - block = list(_column(rows[start : start + _SCREEN_BLOCK], key)) - if _scalar_column_suspect(block): - return True - return False - - -def classify_vertex_rows(rows: Any, declared: Any, screen_scalars: bool = False): - """What GraphBatch.create_vertices must know about its rows before it writes any chunk, decided by types. - - `screen_scalars` is for an input of more than one chunk. One chunk is decided by its own dumps before anything is sent, so a - value the text would change sends all of it the matrix way. Several chunks are sent one after the other, so such a value in - a late chunk would leave the earlier ones stored from the JSON text (an int as an Integer) and the late one from the matrix path - (a Long): the stored type of a property would depend on which chunk held a bad value. With `screen_scalars` the columns of exact - scalars are screened in C over the whole input first (`_scalar_screen`), and a column that may hold a bad value is judged value - by value as the code before did, so a bad scalar anywhere gives None for the whole input, as it always did. - - Returns None when the rows must all go the property-matrix way (a property name that is not a str, or a value of a type the - tagged JSON does not carry: a set, a numpy or pandas value, a list or tuple of only numbers, any other object); otherwise True when - some value needs the tagged text (json_bulk_dumps_tagged) and False when json_bulk_dumps carries every row. A value of an exact - scalar type is not judged here unless `screen_scalars`: whether the text carries it (a NaN, an integer beyond 64 bits, a lone - surrogate) is decided in the dumps of its chunk. The property of `declared` (ARRAY_OF_FLOATS) that holds a list or a numpy array is left to - split_float_vectors. The census over the keys and the values runs in C; a Python-level loop runs only over a column that - holds a type other than the exact scalars. +def _screen_key(rows: Any, key: str, screen: bool): + """(odd, suspect) for the column `key` of all `rows`: the types in it beyond the exact scalars, and, with `screen`, whether a column + of exact scalars only may hold a value the JSON text would change. One list and one census per block of `_SCREEN_BLOCK` rows. """ + odd = set() + suspect = False + for start in range(0, len(rows), _SCREEN_BLOCK): + block = ( + rows if len(rows) <= _SCREEN_BLOCK else rows[start : start + _SCREEN_BLOCK] + ) + column = list(_column(block, key)) + kinds = set(map(type, column)) + extra = kinds - _BULK_SCALAR_TYPES + if extra: + odd |= extra + elif screen and not suspect: + suspect = _scalar_column_suspect(column, kinds) + return odd, suspect + + +def _classify_rows(rows: Any, declared: Any, screen_scalars: bool): + """The verdict of classify_vertex_rows and screen_vertex_rows: None, or ``(tagged, vetted)``. + + `vetted` is True when every column of the rows holds exact scalars only and `screen_scalars` found nothing the text would change + in it (a column the screen flagged counts when every value passed the judgement by value): then no chunk of these rows needs a + check of its own.""" if not set(map(type, rows)) <= _ROWS_OR_NONE: - return _classify_walk( - rows, declared + tagged = _classify_walk(rows, declared) + return ( + None if tagged is None else (tagged, False) ) # a row that is no dict: the walk raises what it always raised keys = set(itertools.chain.from_iterable(filter(None, rows))) if not set(map(type, keys)) <= _STR_ONLY: return None tagged = False + vetted = screen_scalars for key in keys: - odd = set(map(type, _column(rows, key))) - _BULK_SCALAR_TYPES - if not odd: - if not screen_scalars or not _scalar_screen(rows, key): - continue # exact scalars; with screen_scalars (several chunks) only a column that passes the screen - elif key in declared and all(_is_vector_kind(kind) for kind in odd): - continue # the vectors of a declared column - elif all(_is_float_array_kind(kind) for kind in odd): - continue # a column of arrays: take_vector_columns takes the float32 ones; a chunk it leaves goes the matrix way + odd, suspect = _screen_key(rows, key, screen_scalars) + if odd: + vetted = False + if key in declared and all(_is_vector_kind(kind) for kind in odd): + continue # the vectors of a declared column + if all(_is_float_array_kind(kind) for kind in odd): + continue # a column of arrays: take_vector_columns takes the float32 ones; a chunk it leaves goes the matrix way + elif not suspect: + continue # exact scalars; with screen_scalars a column that passed the screen for value in _column(rows, key): if (type(value) in _BULK_SCALAR_TYPES and not screen_scalars) or ( key in declared and is_vector_candidate(value) @@ -1162,7 +1160,39 @@ def classify_vertex_rows(rows: Any, declared: Any, screen_scalars: bool = False) if not json_bulk_shape_ok(value): return None tagged = True - return tagged + return tagged, vetted and not tagged + + +def classify_vertex_rows(rows: Any, declared: Any, screen_scalars: bool = False): + """What GraphBatch.create_vertices must know about its rows before it writes any chunk, decided by types. + + `screen_scalars` is for an input of several chunks, which are sent one after the other: a value the text would change in a late + chunk would leave the earlier ones stored from the JSON text (an int as an Integer) and the late one from the matrix path (a Long), + so the stored type of a property would depend on which chunk held it. With `screen_scalars` the columns of exact scalars are + screened in C over the whole input first (`_screen_key`), and a column that may hold a bad value is judged value by value, so a + bad scalar anywhere gives None for the whole input. + + Returns None when the rows must all go the property-matrix way (a property name that is not a str, or a value of a type the + tagged JSON does not carry: a set, a numpy or pandas value, a list or tuple of only numbers, any other object); otherwise True when + some value needs the tagged text (json_bulk_dumps_tagged) and False when json_bulk_dumps carries every row. A value of an exact + scalar type is not judged here unless `screen_scalars`: whether the text carries it (a NaN, an integer beyond 64 bits, a lone + surrogate) is then decided in the dumps of its chunk. The property of `declared` (ARRAY_OF_FLOATS) that holds a list or a numpy + array is left to split_float_vectors. The census over the keys and the values runs in C; a Python-level loop runs only over a + column that holds a type other than the exact scalars, or that the screen flagged. + """ + verdict = _classify_rows(rows, declared, screen_scalars) + return None if verdict is None else verdict[0] + + +def screen_vertex_rows(rows: Any, declared: Any): + """classify_vertex_rows with the whole-input screen, for any number of chunks: None, or ``(tagged, vetted)``. + + `vetted` says that every value of every row is an exact scalar the JSON text carries unchanged (a column holding any other type, + a vector or a tagged kind, is not vetted). The chunks of such rows are then written by ``json_bulk_dumps(chunk, vetted=True)`` + with no check of their own. One chunk is screened here too, so its values are looked at once, as columns in C, and not again in + the text: a value the text would change sends all of it the property-matrix way, which is what its own dumps did before. + """ + return _classify_rows(rows, declared, True) def _classify_walk(rows: Any, declared: Any): @@ -1333,6 +1363,25 @@ def take_vector_columns(rows, declared, undeclared=False): return taken, matrices, _without_keys(rows, taken) +def vector_width(rows, declared): + """How many floats the vector columns of the first row hold (0 when it holds none): a list, array, Java ``float[]`` or ``double[]`` + in a declared property, or a float32 array or Java ``float[]`` in any property. GraphBatch.create_vertices sizes its crossings by it. + """ + first = rows[0] if rows else None + if not isinstance(first, dict): + return 0 + width = 0 + for key, value in first.items(): + if (key in declared and is_vector_candidate(value)) or _is_float32_vector( + value + ): + try: + width += len(value) + except TypeError: # a 0-dimensional array + pass + return width + + def split_float_vectors(java_db, type_name, rows, declared=None, undeclared=False): """Take the float vectors of `rows` out of the JSON text of a bulk load (the JVM wheel). diff --git a/tests/test_bulk_single_pass_validation.py b/tests/test_bulk_single_pass_validation.py index c830597b..2970e2a7 100644 --- a/tests/test_bulk_single_pass_validation.py +++ b/tests/test_bulk_single_pass_validation.py @@ -327,7 +327,9 @@ def test_a_value_that_may_be_bad_runs_the_walk(monkeypatch, row): assert walks, "the walk must decide a row the text cannot vouch for" -def test_int64_overflow_pattern_matches_exactly_the_tokens_from_2_to_the_63(): +def test_the_column_screen_flags_exactly_the_ints_beyond_64_bits(): + # two regular expressions over the JSON text used to decide this; the text scan of the integer tokens cost 2.5 microseconds a row + # (12 ms of a 5,000-row create_vertices call of the paper's l2 shape), so the columns decide it now, in C, and the text is not scanned rnd = random.Random(63) # nosec B311 - a seeded corpus, not security values = [ 0, @@ -345,12 +347,116 @@ def test_int64_overflow_pattern_matches_exactly_the_tokens_from_2_to_the_63(): ] values += [rnd.randint(10**18, 2**65) for _ in range(5000)] # fmt: skip for value in values: - for text in (str(value), str(-value), f"[{value},1]", f'{{"a":{value}}}'): - hit = tc._INT64_OVERFLOW.search(text) is not None - assert hit == (value >= 2**63), text - # digits after the point of a float and inside an exponent are no integer token - assert tc._INT64_OVERFLOW.search("[0.00012345678901234568]") is None - assert tc._INT64_OVERFLOW.search("[1.2345678901234567e+19]") is None + for number in (value, -value): + outside = not INT64_MIN <= number <= INT64_MAX + assert tc._rows_need_walk([{"a": number}], False) is outside, number + assert tc._rows_need_walk([{"a": None}, {"a": number}], False) is outside + assert tc._scalar_column_suspect([number, None]) is outside, number + # digits after the point of a float and inside an exponent are no integer token, and a float column is not screened at all + assert tc._rows_need_walk([{"f": 0.00012345678901234568}], False) is False + assert tc._rows_need_walk([{"f": 1.2345678901234567e19}], False) is False + assert tc._rows_need_walk([{"s": "id-123456789012345678901"}], False) is False + + +@pytest.mark.parametrize("tagged", [False, True], ids=["untagged", "tagged"]) +def test_a_bad_value_in_a_ragged_or_nullable_column_runs_the_walk_and_gives_the_old_error( + tagged, +): + new = tc.json_bulk_dumps_tagged if tagged else tc.json_bulk_dumps + for rows in ( + [{"a": 1}, {"b": "x"}, {"a": 2**70}], # the key is missing in the middle row + [{"a": None}, {"a": INT64_MIN - 1}], + [{"s": "ok"}, {"s": None}, {"s": "x\ud800"}], + [{"a": 1}, {"a": "text"}, {"a": 2**63}], # a column of two kinds + ): + assert tc._rows_need_walk(rows, tagged) is True, rows + expected = _outcome(lambda r: _old_dumps(r, tagged), rows) + assert expected[0] == "ValueError", expected + assert _outcome(new, rows) == expected + + +def test_a_nullable_column_of_a_tagged_kind_and_the_scalar_columns_are_not_walked( + monkeypatch, +): + walks = [] + real = tc._check_rows + monkeypatch.setattr( + tc, + "_check_rows", + lambda rows, tagged: walks.append(tagged) or real(rows, tagged), + ) + rows = [ + { + "id": i, + "born": None if i % 3 == 0 else dt.datetime(2026, 1, 2, 3, 4, i % 60), + "name": None if i % 5 == 0 else f"n{i}", + "w": i / 7, + } + for i in range(300) + ] + assert tc._rows_need_walk(rows, True) is False + assert tc.json_bulk_dumps_tagged(rows) + assert walks == [] + # a column of a tagged kind beside an int: the walk judges each value + assert tc._rows_need_walk([{"a": dt.date(2026, 1, 2)}, {"a": 5}], True) is True + + +def test_screen_vertex_rows_vets_a_table_of_exact_scalars(monkeypatch): + rows = _batch(2_000, {}) + rows += [ + { + "ts": 1_700_000_000_123_456_789, + "hi": INT64_MAX, + "lo": INT64_MIN, + "f": 0.00012345678901234568, + }, + {"han": "한글퟿中文", "nul": "a\x00b", "e": "", "n": None, "b": True}, + {"mixed": 1}, + {"mixed": 2.5}, # a column of two kinds: judged value by value, and it passes + {}, + ] + assert tc.screen_vertex_rows(rows, set()) == (False, True) + # a vetted table is written with no check of its own: neither the type census nor the walk runs, and the text is the plain text + calls = [] + for name in ("_rows_need_walk", "_check_rows"): + real = getattr(tc, name) + monkeypatch.setattr( + tc, name, lambda *a, _n=name, _r=real: calls.append(_n) or _r(*a) + ) + assert tc.json_bulk_dumps(rows, vetted=True) == json.dumps(rows) + assert calls == [] + plain = rows[ + :2_000 + ] # no column of two kinds: the type census and the column screen decide, no walk + assert tc.json_bulk_dumps(plain) == json.dumps(plain) + assert calls == ["_rows_need_walk"], "an unvetted call still looks at its rows" + + +def test_screen_vertex_rows_does_not_vet_a_column_of_another_type(): + x = np.arange(12, dtype=np.float32).reshape(4, 3) + born = dt.datetime(2026, 1, 2) + assert tc.screen_vertex_rows([{"a": 1, "d": born}] * 3, set()) == (True, False) + assert tc.screen_vertex_rows([{"a": 1, "l": ["x", 2]}] * 3, set()) == (True, False) + # float32 arrays of an undeclared column, and the list of a declared one, are left to split_float_vectors + assert tc.screen_vertex_rows([{"v": row} for row in x], set()) == (False, False) + assert tc.screen_vertex_rows([{"v": row.tolist()} for row in x], {"v"}) == ( + False, + False, + ) + # an empty row (None counts as one) holds nothing to judge + assert tc.screen_vertex_rows([{"a": 1}, None, {}], set()) == (False, True) + + +@pytest.mark.parametrize( + "bad", [2**63, INT64_MIN - 1, math.nan, math.inf, 5e-324, "a\ud800", {1: 2}, {1, 2}] +) +def test_screen_vertex_rows_gives_none_for_what_the_text_would_change(bad): + rows = [{"id": i, "x": 1.5} for i in range(300)] + assert tc.screen_vertex_rows(rows, set()) == (False, True) + rows[150]["x"] = bad + assert tc.screen_vertex_rows(rows, set()) is None + assert tc.classify_vertex_rows(rows, set(), True) is None + assert tc.screen_vertex_rows([{"id": 1}, {1: "a"}], set()) is None # --------------------------------------------------------------------------- diff --git a/tests/test_graph_batch_single_pass.py b/tests/test_graph_batch_single_pass.py index 2a35a768..a6aa6a98 100644 --- a/tests/test_graph_batch_single_pass.py +++ b/tests/test_graph_batch_single_pass.py @@ -3,11 +3,12 @@ Before, `_create_vertices_json_bulk` called `json_bulk_scalar_ok` on every value of every row before the first chunk was written, and the chunk was then walked again by json.dumps. Now `type_conversion.classify_vertex_rows` takes a census of the types in C, and the value checks come from the one json.dumps pass of each chunk (`json_bulk_dumps`, as `json_bulk_dumps_tagged` already did for the tagged -text). The rule that keeps the stored type of a property from depending on which chunk held a bad value: an input of one chunk is decided -by its own dumps before anything is sent; an input of several chunks first has its columns of exact scalars screened in C over the whole -input (`classify_vertex_rows(..., screen_scalars=True)`), and a bad scalar (a NaN, an integer beyond 64 bits, a lone surrogate, a -subnormal) anywhere sends the WHOLE input the property-matrix way, as it did before. A name that is not a str or a type the text does not -carry does the same (`_create_vertices_json_bulk` returns None). +text). The rule that keeps the stored type of a property from depending on which chunk held a bad value: the columns of exact scalars are +screened in C over the whole input, one chunk or several (`type_conversion.screen_vertex_rows`), and a bad scalar (a NaN, an integer beyond +64 bits, a lone surrogate, a subnormal) anywhere sends the WHOLE input the property-matrix way, as it did before. A name that is not a str +or a type the text does not carry does the same (`_create_vertices_json_bulk` returns None). An input of exact scalars only has then been +looked at once, as columns, so its chunks are written with no check of their own (no text scan, no walk): the 5,000-row call of the l2 +Person load spent 12 ms of 43 in two regular expressions over the JSON text before this (#318 follow-up, the g1 regression of candidate 3). """ import pytest @@ -154,3 +155,47 @@ def _stored_types(db, probes): assert set(by_id) == set(probes) assert {i: by_id[i][0] for i in probes} == {i: i % 1000 for i in probes} return {probes[i]: by_id[i][1] for i in probes} + + +@pytest.mark.parametrize("chunk", [100_000, 500], ids=["one_chunk", "screened"]) +def test_a_screened_scalar_table_is_written_with_no_check_per_chunk( + temp_db_path, monkeypatch, chunk +): + # the l2 Person shape: the input was screened once as columns, so no chunk runs the census again or the walk (candidate 3 ran two + # regular expressions over every chunk's text here: 31 percent more wall for the 5,000-row call) + calls = [] + for name in ("_rows_need_walk", "_check_rows"): + real = getattr(tc, name) + monkeypatch.setattr( + tc, name, lambda *a, _n=name, _r=real: calls.append(_n) or _r(*a) + ) + monkeypatch.setattr(GraphBatch, "_BULK_CHUNK", chunk) + rows = _rows(2_000) + with arcadedb.create_database(temp_db_path) as db: + db.command("sql", "CREATE VERTEX TYPE V") + with db.graph_batch(parallel_flush=False) as batch: + assert len(batch.create_vertices("V", rows)) == len(rows) + assert db.query("sql", "SELECT count(*) AS c FROM V").first().get("c") == 2_000 + assert calls == [] + + +def test_rows_with_another_kind_of_column_are_still_checked_chunk_by_chunk( + temp_db_path, monkeypatch +): + from datetime import datetime + + walks = [] + real = tc._rows_need_walk + monkeypatch.setattr( + tc, + "_rows_need_walk", + lambda rows, tagged: walks.append(len(rows)) or real(rows, tagged), + ) + monkeypatch.setattr(GraphBatch, "_BULK_CHUNK", 100) + rows = [{"id": i, "born": datetime(2026, 1, 2, 3, 4, 5)} for i in range(250)] + with arcadedb.create_database(temp_db_path) as db: + db.command("sql", "CREATE VERTEX TYPE V") + with db.graph_batch(parallel_flush=False) as batch: + assert len(batch.create_vertices("V", rows)) == 250 + assert db.query("sql", "SELECT count(*) AS c FROM V").first().get("c") == 250 + assert walks == [100, 100, 50] diff --git a/tests/test_graph_batch_vector_crossings_jvm.py b/tests/test_graph_batch_vector_crossings_jvm.py index a16d36c3..a86b89b9 100644 --- a/tests/test_graph_batch_vector_crossings_jvm.py +++ b/tests/test_graph_batch_vector_crossings_jvm.py @@ -239,3 +239,84 @@ def test_take_vector_columns_reads_java_arrays_through_one_stack(db): doubles[0]["v"] ) assert isinstance(arcadedb.to_java_float_array(x[0]), float_array) + + +def test_rows_with_vectors_cross_in_blocks_sized_by_their_width( + db, crossings, monkeypatch +): + # about _VECTOR_BLOCK_FLOATS floats per crossing over all the vector columns, never fewer than _VECTOR_BLOCK_MIN_ROWS rows + monkeypatch.setattr(GraphBatch, "_BULK_CHUNK", 100_000) + monkeypatch.setattr(GraphBatch, "_VECTOR_BLOCK_FLOATS", 640) # 40 rows of 16 floats + monkeypatch.setattr(GraphBatch, "_VECTOR_BLOCK_MIN_ROWS", 10) + x = _x() + arrays = _floats(x) + rows = [{"id": i, "embedding": arrays[i]} for i in range(ROWS)] + with db.graph_batch(parallel_flush=False) as gb: + assert len(gb.create_vertices("Declared", rows)) == ROWS + assert crossings["fallback"] == [] and crossings["convert"] == [] + assert ( + crossings["bridge"] == ["createVerticesJson"] * 7 + ) # 6 blocks of 40, one of 10 + assert crossings["matrix"].count("Matrix") == 7 + sql = "SELECT id, embedding FROM Declared" + stored = {r["id"]: r["embedding"] for r in db.query("sql", sql).to_list()} + assert len(stored) == ROWS + for i in range(ROWS): + assert np.array(stored[i], dtype=np.float32).tobytes() == x[i].tobytes() + # the floor: a block is never smaller than _VECTOR_BLOCK_MIN_ROWS rows + crossings["bridge"].clear() + monkeypatch.setattr(GraphBatch, "_VECTOR_BLOCK_FLOATS", 16) + rows = [{"id": 1000 + i, "embedding": arrays[i]} for i in range(25)] + with db.graph_batch(parallel_flush=False) as gb: + gb.create_vertices("Free", rows) + assert crossings["bridge"] == ["createVerticesJson"] * 3 # 10, 10, 5 + # rows without a vector keep the whole _BULK_CHUNK + crossings["bridge"].clear() + with db.graph_batch(parallel_flush=False) as gb: + gb.create_vertices("Free", [{"id": 2000 + i} for i in range(ROWS)]) + assert crossings["bridge"] == ["createVerticesJson"] + + +def test_vector_width_counts_the_vector_columns_of_the_first_row(db): + x = _x(3, 16) + arrays = _floats(x) + both = [{"a": 1, "v": arrays[0], "w": x[1], "d": x[2].astype(np.float64)}] + assert ( + tc.vector_width(both, set()) == 16 + 16 + ) # a float32 array and a Java float[] of any property; the float64 array is not one + assert tc.vector_width(both, {"d"}) == 16 + 16 + 16 # on a declared property it is + assert tc.vector_width([{"a": 1, "l": [1.0, 2.0]}], {"l"}) == 2 + assert tc.vector_width([{"a": 1, "l": [1.0, 2.0]}], set()) == 0 + assert tc.vector_width([], {"l"}) == 0 and tc.vector_width([None], {"l"}) == 0 + assert tc.vector_width([{"v": np.float32(1.5)}], set()) == 0 + + +def test_the_previous_chunks_rows_are_released_before_the_next_chunk_is_split( + db, crossings, monkeypatch +): + """Two chunks' float[] buffers and rest rows were alive at once: `split` still held the previous chunk's while the next was built.""" + import weakref + + class Rest(list): # a list that can be weakly referenced + pass + + alive = ( + [] + ) # for every call of the split: whether the rest rows of each earlier call are still alive + refs = [] + real = gbm.split_float_vectors + + def spy(*args, **kwargs): + alive.append([ref() is not None for ref in refs]) + out = real(*args, **kwargs) + rest = Rest(out[0]) + refs.append(weakref.ref(rest)) + return (rest,) + tuple(out[1:]) + + monkeypatch.setattr(gbm, "split_float_vectors", spy) + x = _x() + arrays = _floats(x) + rows = [{"id": i, "embedding": arrays[i]} for i in range(ROWS)] + with db.graph_batch(parallel_flush=False) as gb: + assert len(gb.create_vertices("Declared", rows)) == ROWS + assert alive == [[], [False], [False, False]] From 4cc08cf02fc229869fee22079e5cb335466ed82e Mon Sep 17 00:00:00 2001 From: Taewoon Kim Date: Tue, 10 Nov 2026 04:29:40 +0900 Subject: [PATCH 37/49] Test: a map with a non-str key is judged by the walk, not refused by the screen (the verdict is tagged, not None) screen_vertex_rows returns None for a set (a type the tagged JSON does not carry) but a dict with an int key is a tagged shape whose bad key the chunk's walk refuses, as before; the parametrized case was wrong. Co-Authored-By: Claude Sonnet 5.5 --- tests/test_bulk_single_pass_validation.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_bulk_single_pass_validation.py b/tests/test_bulk_single_pass_validation.py index 2970e2a7..ea0660f6 100644 --- a/tests/test_bulk_single_pass_validation.py +++ b/tests/test_bulk_single_pass_validation.py @@ -448,7 +448,7 @@ def test_screen_vertex_rows_does_not_vet_a_column_of_another_type(): @pytest.mark.parametrize( - "bad", [2**63, INT64_MIN - 1, math.nan, math.inf, 5e-324, "a\ud800", {1: 2}, {1, 2}] + "bad", [2**63, INT64_MIN - 1, math.nan, math.inf, 5e-324, "a\ud800", {1, 2}] ) def test_screen_vertex_rows_gives_none_for_what_the_text_would_change(bad): rows = [{"id": i, "x": 1.5} for i in range(300)] From cf945e33c6bced8a46e93a37e9f025286bc7ee63 Mon Sep 17 00:00:00 2001 From: Taewoon Kim Date: Tue, 10 Nov 2026 05:09:54 +0900 Subject: [PATCH 38/49] Test: the emoji row is no bad value; the old text scan sent it to the walk, the column screen does not (fixes CI row4 of test_a_value_that_may_be_bad_runs_the_walk) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Row 4 of the parameter list was {"a": "\U0001f600"}. json.dumps writes a non-BMP character as a surrogate PAIR (😀); the removed _SURROGATE_ESCAPE regex took that for a lone surrogate and sent the rows to the walk, which found nothing: a false alarm that the test had recorded as "may be bad". The 26.11.1 loop accepts the value (UTF-8 encodes it) and writes the same text, and the column screen passes it, so no walk runs. Not an unsound screen: on the old and the new tree the end-to-end text is the 26.11.1 text. The row leaves the "runs the walk" list. New: test_a_non_bmp_character_is_no_bad_value_and_is_not_walked (emoji, U+10000, U+10FFFF, mixed text, untagged and tagged: same text as the 26.11.1 loop, no walk; two adjacent lone surrogates are no pair and still refused by the walk), and non-BMP text in the rows of the not-walked table test and in the vetted-table test. Co-Authored-By: Claude Sonnet 5.5 --- tests/test_bulk_single_pass_validation.py | 39 +++++++++++++++++++++-- 1 file changed, 37 insertions(+), 2 deletions(-) diff --git a/tests/test_bulk_single_pass_validation.py b/tests/test_bulk_single_pass_validation.py index ea0660f6..3d12c8b8 100644 --- a/tests/test_bulk_single_pass_validation.py +++ b/tests/test_bulk_single_pass_validation.py @@ -256,7 +256,9 @@ def test_a_clean_large_batch_is_the_plain_json_text(tagged): def test_scalar_rows_are_not_walked_when_the_text_carries_them(monkeypatch): # the insert_many shape of the paper's l1tpc Part load and the create_vertices shape of l2, e2 and l3d: dict rows of str, int, float, # bool and None. Numbers that look alike but fit: 19-digit ints below 2**63, floats whose text has a long run of digits after the - # point, text with digits, Hangul and CJK (BMP characters that json.dumps writes as \uXXXX, some of them as \ud...). + # point, text with digits, Hangul and CJK (BMP characters that json.dumps writes as \uXXXX, some of them as \ud...), and non-BMP + # characters (an emoji): json.dumps writes each as a surrogate PAIR (\ud83d\ude00), which the old text scan took for a lone surrogate + # and sent to the walk, a false alarm; UTF-8 encodes them, so the column screen passes them and the old loop accepted them too. walks = [] real = tc._check_rows monkeypatch.setattr( @@ -278,6 +280,11 @@ def test_scalar_rows_are_not_walked_when_the_text_carries_them(monkeypatch): "han": "한글퟿中文", "nul": "a\x00b", }, + { + "emoji": "\U0001f600", + "plane1": "\U00010000x\U0010ffff", + "mixed": "a\U0001f600b", + }, {"e": "", "n": None, "b": True, "neg": -123456789012345678}, ] assert tc.json_bulk_dumps(rows) == json.dumps(rows) @@ -308,7 +315,6 @@ def test_tagged_kinds_are_not_walked_either(monkeypatch): {"a": 10**20}, {"a": INT64_MIN - 1}, {"a": "a\ud800"}, - {"a": "\U0001f600"}, {7: "x"}, {"a": [1]}, {"a": 5e-324}, @@ -375,6 +381,34 @@ def test_a_bad_value_in_a_ragged_or_nullable_column_runs_the_walk_and_gives_the_ assert _outcome(new, rows) == expected +@pytest.mark.parametrize("tagged", [False, True], ids=["untagged", "tagged"]) +@pytest.mark.parametrize( + "text", + ["\U0001f600", "\U00010000", "\U0010ffff", "a\U0001f600b", "\ue000\U0001f600"], +) +def test_a_non_bmp_character_is_no_bad_value_and_is_not_walked( + monkeypatch, text, tagged +): + # the row the old text scan sent to the walk (a surrogate pair in the text) and the old loop accepted: same text, no walk + walks = [] + real = tc._check_rows + monkeypatch.setattr( + tc, + "_check_rows", + lambda rows, tagged: walks.append(tagged) or real(rows, tagged), + ) + rows = [{"id": 1}, {"a": text}] + new = tc.json_bulk_dumps_tagged if tagged else tc.json_bulk_dumps + assert _outcome(new, rows) == ("text", json.dumps(rows)) + assert _outcome(lambda r: _old_dumps(r, tagged), rows) == ("text", json.dumps(rows)) + assert walks == [] + # two lone surrogates next to each other are no pair: UTF-8 cannot encode them, the walk refuses them + bad = [{"id": 1}, {"a": "\ud83d\ude00"}] + got = _outcome(new, bad) + assert got == _outcome(lambda r: _old_dumps(r, tagged), bad) + assert got[0] == "ValueError" and walks == [tagged] + + def test_a_nullable_column_of_a_tagged_kind_and_the_scalar_columns_are_not_walked( monkeypatch, ): @@ -416,6 +450,7 @@ def test_screen_vertex_rows_vets_a_table_of_exact_scalars(monkeypatch): {}, ] assert tc.screen_vertex_rows(rows, set()) == (False, True) + assert tc.screen_vertex_rows([{"a": "\U0001f600"}], set()) == (False, True) # a vetted table is written with no check of its own: neither the type census nor the walk runs, and the text is the plain text calls = [] for name in ("_rows_need_walk", "_check_rows"): From c6c3999cb299ec7f0609168725a8871d3a067830 Mon Sep 17 00:00:00 2001 From: Taewoon Kim Date: Tue, 10 Nov 2026 10:10:19 +0900 Subject: [PATCH 39/49] Test: 300 unread cursors stream instead of holding 300 sorted buffers (the engine's 2 GB query heap budget failed the ORDER BY version on macOS arm64 CI and on a laptop) test_a_cursor_with_nothing_read_closed_many_times_over opened 300 ORDER BY cursors at once; each holds its sorted buffer in the in-heap budget from the start, together about 2 GB, so QueryHeapBudgetExceededException on macOS arm64 (all four Python versions) and in a local run on both wheels. The test is about cursors that were never read; it now uses the unsorted SELECT. 51 passed on both wheels (test_column_frames.py). Co-Authored-By: Claude Sonnet 5.5 --- tests/test_column_frames.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/tests/test_column_frames.py b/tests/test_column_frames.py index ff07ed59..f94a810e 100644 --- a/tests/test_column_frames.py +++ b/tests/test_column_frames.py @@ -423,7 +423,11 @@ def test_two_cursors_open_at_once_and_a_cursor_closed_early(table): def test_a_cursor_with_nothing_read_closed_many_times_over(table): db, _ = table - cursors = [db.query("sql", _sel()) for _ in range(300)] + # No ORDER BY: a sorted query holds its whole sorted buffer in the engine's query heap budget (2 GB by default) from the moment + # it starts, so 300 of them open at once exhausted it (macOS arm64 CI and a laptop, 2026-11-10). This test is about cursors that + # were never read, so the query streams. + unsorted = "SELECT n, big, d, f, s, ok, ts, v3, v9, v128 FROM T" + cursors = [db.query("sql", unsorted) for _ in range(300)] for cursor in cursors: cursor.close() assert db.query("sql", _sel() + " LIMIT 3").to_arrow().num_rows == 3 From 4d2db173bfd31bf9e4d951b3c12ec1e8c69f00b0 Mon Sep 17 00:00:00 2001 From: Taewoon Kim Date: Tue, 10 Nov 2026 14:14:03 +0900 Subject: [PATCH 40/49] to_dataframe: keep an integer beyond 2**53 next to a float exact (refs #340) A result column that holds an integer outside float64's exact +-2**53 range next to a floating value is a list of exact Python values in the shared decoder (the writer's json column, or the merge of frames). to_columns() returns that list and to_arrow() the exact text, but both wheels' to_dataframe() handed the list to pd.DataFrame(columns), and pandas infers float64 for a list of ints and floats: the integer came out rounded (2**60 + 1 -> 1152921504606846976.0). Same data, exact in three readers, rounded in the fourth. The fix is in the one place both wheels share (arcadedb._columnar): - dataframe_from_columns(columns) builds the frame for both wheels (results.py on the JVM wheel, _native/database.py on the native wheel). A list column that holds an int beyond +-2**53 and a float becomes an object Series of the exact Python values (a null stays None, an int stays an int). Every other column is passed to pandas as before, so no other dtype changes: ints and floats within 2**53 are still float64, an int column is still int64, text is still object. The check is one set(map(type, ...)) pass per list column, so a text column pays a C-speed scan and nothing else. - merge_columns no longer np.concatenate()s an int64 frame with a value beyond +-2**53 and a float64 frame: that rounded the integers across a frame boundary (a result of more than 25,000 rows), so the column's type depended on where the frames end. It takes the generic list path (exact Python values), which to_dataframe then makes an object column, and to_columns() returns as the list it already returns for one frame. Tests first (tests/test_column_frames.py, 8 functions, 15 cases, 21 asserts), run on the CI-built post1 wheels of dbe1670, both the JVM and the native wheel: on the unmodified wheels 11 of the 15 fail (the other 4 pin what must not change: text next to the integer, integers alone stay int64, up to 2**53 next to floats stay float64, an empty result), on the patched wheels all pass. They cover the integer beside a float in alternating and block layouts at 10 and 300 rows, the ends of int64 (+-2**53 + 1, 2**63 - 1, -2**63 + 1, -2**63), a null in the mixed column, and two frames (int64 beyond 2**53, then float64) merged. Not changed: to_dataframe(convert_types=False) on the JVM wheel and the no-bridge fallback still build the frame from to_list(); an integer column with a null stays float64 with NaN (the documented pandas rule shared with to_columns()). Co-Authored-By: Claude Sonnet 5.5 --- src/arcadedb/_columnar.py | 53 +++++++++++++++- src/arcadedb/_native/database.py | 11 ++-- src/arcadedb/results.py | 8 ++- tests/test_column_frames.py | 102 +++++++++++++++++++++++++++++++ 4 files changed, 165 insertions(+), 9 deletions(-) diff --git a/src/arcadedb/_columnar.py b/src/arcadedb/_columnar.py index c33ff0e8..fe1ed34b 100644 --- a/src/arcadedb/_columnar.py +++ b/src/arcadedb/_columnar.py @@ -319,12 +319,33 @@ def _is_all_none(part, count: int) -> bool: ) +# float64 holds every integer in +-2**53 exactly, and not every one beyond it +_TWO53 = 1 << 53 + + +def _int64_beyond_2_53_next_to_float64(parts) -> bool: + """True when the parts of one column hold an int64 array with a value beyond +-2**53 and a float64 array: ``np.concatenate`` + would make the integers doubles and round them. One frame of such a column is a list of exact Python values (the writer's + ``json`` column); the merge of frames does the same, so the column does not depend on where the frames end. + """ + import numpy as np + + arrays = [p for p in parts if isinstance(p, np.ndarray) and p.ndim == 1] + if not any(a.dtype.kind == "f" for a in arrays): + return False + return any( + a.dtype.kind == "i" and a.size and (a.max() > _TWO53 or a.min() < -_TWO53) + for a in arrays + ) + + def merge_columns( names: List[str], batches: List[Tuple[int, Dict[str, Any]]], columns=None ) -> Dict[str, Any]: """The ``to_columns`` dict of the parts of every frame. The type of a column belongs to the whole result, not to where the frames end: a frame that lacks the column, or holds only nulls for it, takes the type of the frames that have values (NaN, NaT or None - in its rows). With no rows at all the result is :func:`empty_columns`.""" + in its rows). An int64 frame with a value beyond +-2**53 next to a float64 frame makes a list of exact Python values, not a + float64 array. With no rows at all the result is :func:`empty_columns`.""" import numpy as np if not batches or not names: @@ -344,7 +365,11 @@ def merge_columns( for c, p in batches ] np_parts = [p for p in parts if not isinstance(p, list)] - if parts and len(np_parts) == len(parts): + if ( + parts + and len(np_parts) == len(parts) + and not _int64_beyond_2_53_next_to_float64(typed) + ): try: out[name] = np.concatenate(parts) if len(parts) > 1 else parts[0] continue @@ -357,6 +382,30 @@ def merge_columns( return out +def _pandas_would_round(values) -> bool: + """True for a list that holds an integer beyond +-2**53 next to a float: pandas infers float64 for a list of ints and floats, + and an integer that is not an exact double comes out rounded.""" + kinds = set(map(type, values)) # one C-speed pass: a text column ends here + if int not in kinds or float not in kinds: + return False + return any(type(v) is int and not -_TWO53 <= v <= _TWO53 for v in values) + + +def dataframe_from_columns(columns): + """The ``pandas.DataFrame`` of the ``to_columns`` dict ``columns`` (what both wheels' ``to_dataframe`` builds). A column that is a + list holding an integer beyond +-2**53 next to a float is an ``object`` column of the exact Python values, as ``to_columns()`` + returns them: pandas would make it float64 and round the integer (#340). Every other column is handed to pandas as it was. + """ + import pandas as pd + + exact = { + name: pd.Series(values, dtype=object) + for name, values in columns.items() + if isinstance(values, list) and _pandas_would_round(values) + } + return pd.DataFrame({**columns, **exact} if exact else columns) + + def decode_arrow_frame( buf, side, json_values: Callable ) -> Tuple[int, Dict[str, Any], Dict[str, Any]]: diff --git a/src/arcadedb/_native/database.py b/src/arcadedb/_native/database.py index 19446dbc..c02ac669 100644 --- a/src/arcadedb/_native/database.py +++ b/src/arcadedb/_native/database.py @@ -485,12 +485,13 @@ def to_dataframe(self, convert_types: bool = True): with two exceptions that are the same on both builds. A DATETIME keeps its microseconds (a ``datetime64[us]`` column, where ``to_columns()`` and ``to_arrow()`` carry milliseconds), a DATE is a ``datetime.date`` in an object column, and a ``Decimal`` stays one. A column of ``float[]`` vectors holds one numpy ``float32`` array per cell (a row of the matrix ``to_columns()`` - returns, not a Python list per row) and None for a null; ``to_list()`` gives lists of floats. + returns, not a Python list per row) and None for a null; ``to_list()`` gives lists of floats. A column that holds an integer + beyond +-2**53 next to a float is an ``object`` column of the exact Python values (``float64`` would round the integer), as + ``to_columns()`` returns them; ``to_arrow()`` gives the exact text. """ - import pandas as pd - - columns = self._columns(25_000, None, True) - return pd.DataFrame(columns) + # availability check: a missing pandas raises before the cursor is read + __import__("pandas") + return _columnar.dataframe_from_columns(self._columns(25_000, None, True)) def _columns(self, batch_size, columns, micros): from . import colframes diff --git a/src/arcadedb/results.py b/src/arcadedb/results.py index 369c1dfa..a17f8bc6 100644 --- a/src/arcadedb/results.py +++ b/src/arcadedb/results.py @@ -13,6 +13,7 @@ from jpype import JException from ._columnar import ( + dataframe_from_columns, decode_arrow_frame, decode_columns_frame, empty_arrow, @@ -556,7 +557,10 @@ def to_dataframe(self, convert_types: bool = True): and a ``Decimal`` stays one. A column of ``float[]`` vectors holds one numpy ``float32`` array per cell (a row of the matrix ``to_columns()`` returns, no copy into a Python list per row) and None for a null; - ``to_list()`` gives lists of floats. + ``to_list()`` gives lists of floats. A column that holds an integer + beyond +-2**53 next to a float is an ``object`` column of the exact + Python values (``float64`` would round the integer), as ``to_columns()`` + returns them; ``to_arrow()`` gives the exact text. Args: convert_types: Convert Java types to Python (default: True) @@ -589,7 +593,7 @@ def to_dataframe(self, convert_types: bool = True): columns = self._read_columns(25_000, None, micros=True) if columns is not None: # a vector column is already one numpy array per cell, None for a null - return pd.DataFrame(columns) + return dataframe_from_columns(columns) return pd.DataFrame(self.to_list(convert_types=convert_types)) diff --git a/tests/test_column_frames.py b/tests/test_column_frames.py index f94a810e..98de3150 100644 --- a/tests/test_column_frames.py +++ b/tests/test_column_frames.py @@ -523,6 +523,108 @@ def test_an_integer_beyond_2_53_next_to_a_float_stays_exact(temp_db, rows): ] # no common numeric type: exact text, as across batches +# --- to_dataframe and an integer beyond 2**53 (#340) ----------------------------------------------------------------------- +# to_columns() gives the exact Python list and to_arrow() the exact text; to_dataframe() handed that list to pandas, which infers +# float64 for ints and floats and rounded the integer. Each test reads the frame, so the same literal holds on both wheels. + + +def _mix_frame(db, values): + """The ``to_dataframe()`` of ``values`` stored in property ``x`` of one document each, in order (None = the row lacks ``x``).""" + db.command("sql", "CREATE DOCUMENT TYPE Mix") + db.command("sql", "CREATE PROPERTY Mix.i INTEGER") + db.insert_many( + "Mix", + [{"i": i} if v is None else {"i": i, "x": v} for i, v in enumerate(values)], + ) + return db.query("sql", "SELECT i, x FROM Mix ORDER BY i").to_dataframe() + + +def _same_values_and_types(have, want): + return list(have) == list(want) and all( + type(a) is type(b) for a, b in zip(have, want) + ) + + +@pytest.mark.parametrize("layout", ["alternating", "blocks"]) +@pytest.mark.parametrize("rows", [10, 300]) +def test_to_dataframe_keeps_an_integer_beyond_2_53_next_to_a_float( + temp_db, rows, layout +): + big = 2**60 + 1 + if layout == "alternating": + want = [big if i % 2 == 0 else 1.5 for i in range(rows)] + else: + want = [big if i < rows // 2 else 1.5 for i in range(rows)] + df = _mix_frame(temp_db, want) + assert df["x"].dtype == object # float64 would round the integer + assert _same_values_and_types(df["x"].tolist(), want) # an int stays an int + assert df["i"].dtype == np.int64 and df["i"].tolist() == list(range(rows)) + + +@pytest.mark.parametrize( + "big", [2**53 + 1, -(2**53) - 1, 2**63 - 1, -(2**63) + 1, -(2**63)] +) +def test_to_dataframe_keeps_the_ends_of_int64_next_to_a_float(temp_db, big): + want = [big, 1.5, big, -0.25] + df = _mix_frame(temp_db, want) + assert df["x"].dtype == object + assert _same_values_and_types(df["x"].tolist(), want) + + +def test_to_dataframe_mixed_column_with_a_null_keeps_the_null_as_none(temp_db): + big = 2**60 + 1 + df = _mix_frame(temp_db, [big, None, 1.5, -(2**63)]) + assert df["x"].dtype == object + assert df["x"].tolist() == [big, None, 1.5, -(2**63)] + assert df["x"][1] is None # not NaN: a missing value is None as in a text column + assert type(df["x"][0]) is int and type(df["x"][2]) is float + + +def test_to_dataframe_integer_beyond_2_53_next_to_text_and_a_float(temp_db): + want = [2**60 + 1, "text", 1.5, -(2**63)] + df = _mix_frame(temp_db, want) + assert df["x"].dtype == object + assert _same_values_and_types(df["x"].tolist(), want) + + +def test_to_dataframe_integers_alone_stay_int64_even_beyond_2_53(temp_db): + want = [2**60 + 1, -(2**63), 2**63 - 1, 7] + df = _mix_frame(temp_db, want) + assert df["x"].dtype == np.int64 # unchanged: exact, no float in the column + assert df["x"].tolist() == want + + +def test_to_dataframe_integers_up_to_2_53_next_to_floats_stay_float64(temp_db): + # every integer here is an exact double, so the usual numeric column is kept (the dtype does not change for this shape) + df = _mix_frame(temp_db, [1, 1.5, 2**53, -(2**53)]) + assert df["x"].dtype == np.float64 + assert df["x"].tolist() == [1.0, 1.5, 2.0**53, -(2.0**53)] + + +def test_to_dataframe_of_an_empty_result_is_an_empty_frame(temp_db): + _mix_frame(temp_db, [2**60 + 1, 1.5]) + df = temp_db.query("sql", "SELECT i, x FROM Mix WHERE i < 0").to_dataframe() + assert len(df) == 0 and list(df.columns) == [] + + +def test_frames_of_an_integer_beyond_2_53_and_frames_of_floats_merge_exactly(): + # the column's type belongs to the whole result: two frames, one of int64 (beyond 2**53) and one of float64, must not go + # through float64 when they merge (np.concatenate would round the integers), and to_dataframe must keep the exact values + ints = _frame(2, [("n", "i8", [], struct.pack("<2q", 2**60 + 1, -(2**63)))]) + floats = _frame(2, [("n", "f8", [], struct.pack("<2d", 0.5, 1.5))]) + columns = _columnar.merge_columns(["n"], [_columns(ints), _columns(floats)]) + want = [2**60 + 1, -(2**63), 0.5, 1.5] + assert _same_values_and_types(columns["n"], want) + df = _columnar.dataframe_from_columns(columns) + assert df["n"].dtype == object + assert _same_values_and_types(df["n"].tolist(), want) + # integers within 2**53 next to floats keep the numeric column of the merge (float64), and so does the frame + small = _frame(2, [("n", "i8", [], struct.pack("<2q", 1, -(2**53)))]) + columns = _columnar.merge_columns(["n"], [_columns(small), _columns(floats)]) + assert columns["n"].dtype == np.float64 + assert _columnar.dataframe_from_columns(columns)["n"].dtype == np.float64 + + def test_vectors_whose_dimension_changes_between_frames_are_one_list_column(temp_db): db = temp_db db.command("sql", "CREATE DOCUMENT TYPE Dim") From 9914e9350ed3e578039499ce4e41b1099ed5789a Mon Sep 17 00:00:00 2001 From: Taewoon Kim Date: Tue, 10 Nov 2026 14:16:21 +0900 Subject: [PATCH 41/49] perf: a float32 vector column of insert_arrow and insert_dataframe crosses as one matrix on the JVM wheel A FixedSizeList (or list, or large list) column into an ARRAY_OF_FLOATS property went chunk.to_pylist() (one Python float per element), then core._column_to_java converted every row with convert_python_to_java (an ArrayList of 96 boxed doubles per row, 100,000 conversions for ten batches of 10,000 rows) and the engine narrowed each list back to a float[]. That was the cost the released route did not have (GraphBatch took the rows differently): in the untimed profile of the l3d shape the conversion was the largest part of the call, the Python heap peaked at 31.5 MiB for a 3.7 MiB buffer, and the boxed doubles are what the JVM heap held. _column_values now takes a float32 vector column of one length, with no null row and no null element, as one 2-D float32 numpy matrix (a view of Arrow's buffer; flatten follows a slice's offset), and _column_to_java already sent a 2-D float32 numpy column as one float[] inside the bridge's FloatVectors.Matrix, so the documents get a row each Java-side. Taken only for a property declared ARRAY_OF_FLOATS and only on the JVM backend (float_vectors=not native and declared == ARRAY_OF_FLOATS): the native wheel keeps Arrow arrays and is not touched. Everything else keeps to_pylist and so keeps what that route stored or refused: a null row stays null, a null element is refused, vectors of different length are stored as given or refused by a vector index, float64 and integer lists are narrowed by the engine (and refused past the float range) which a cast here would not do, undeclared and LIST properties keep a list. tests/test_arrow_vector_column_jvm.py (JVM only, registered in conftest.py): 18 tests pin what the old route stored (count and an order-independent digest through SQL and through the result API, both again after close and reopen, special float32 values bit for bit, a null row, a null element, different lengths, a wrong dimension against a vector index, float64 and integer lists, undeclared and LIST properties, empty vectors, chunked tables and slices); 5 count what the load does and fail on the code before (per-row conversions, Matrix crossings, Python heap of one batch). Co-Authored-By: Claude Sonnet 5.5 --- src/arcadedb/dataframe.py | 54 ++++- tests/conftest.py | 1 + tests/test_arrow_vector_column_jvm.py | 309 ++++++++++++++++++++++++++ 3 files changed, 363 insertions(+), 1 deletion(-) create mode 100644 tests/test_arrow_vector_column_jvm.py diff --git a/src/arcadedb/dataframe.py b/src/arcadedb/dataframe.py index e91a02f7..4c7beb22 100644 --- a/src/arcadedb/dataframe.py +++ b/src/arcadedb/dataframe.py @@ -336,10 +336,57 @@ def _is_number_list(pa, arrow_type) -> bool: ) -def _column_values(name: str, chunk, pa, native: bool = False, arrow: bool = False): +_JAVA_INT_MAX = 2**31 - 1 + + +def _float32_matrix(pa, chunk): + """A list, large list or fixed-size list column of float32 with no null row, no null element and one length, as one 2-D float32 + numpy matrix (a view of Arrow's value buffer: ``flatten`` follows a slice's offset), else None. None means "not this shape" and + leaves the column to ``to_pylist``, which keeps what that route stored or refused: a null row stays null, a null element is + refused, vectors of different length are stored as given or refused by a vector index. Only float32 is taken: a float64 or integer + value is narrowed by the engine (and refused past the float range), which a cast here would not do. + """ + types = pa.types + arrow_type = chunk.type + if not ( + types.is_list(arrow_type) + or types.is_large_list(arrow_type) + or types.is_fixed_size_list(arrow_type) + ): + return None + n = len(chunk) + if not types.is_float32(arrow_type.value_type) or not n or chunk.null_count: + return None + flat = chunk.flatten() + if flat.null_count or len(flat) % n or len(flat) > _JAVA_INT_MAX: + return None + dim = len(flat) // n + if dim == 0: + return None + if not types.is_fixed_size_list(arrow_type): + import pyarrow.compute as pc + + lengths = pc.list_value_length(chunk) + if pc.min(lengths).as_py() != dim or pc.max(lengths).as_py() != dim: + return None + return flat.to_numpy(zero_copy_only=False).reshape(n, dim) + + +def _column_values( + name: str, + chunk, + pa, + native: bool = False, + arrow: bool = False, + float_vectors: bool = False, +): """One Arrow array as what ``Database.insert_columns`` takes: a numpy array when the column crosses as one buffer (integers, floats and booleans without nulls), else a list of values (``None`` is a null). + ``float_vectors`` (the JVM backend, a property declared ``ARRAY_OF_FLOATS``): a float32 vector column of one length and no null is + one 2-D float32 matrix, which ``insert_columns`` sends as one ``float[]``; ``to_pylist`` made one Python float per element and + ``insert_columns`` one ``ArrayList`` of boxed doubles per row (see ``_float32_matrix`` for what is left to it). + ``arrow`` (the native backend's document and vertex load): a string, binary, timestamp, date or decimal column, a list of numbers (a vector) and a numeric column with nulls, stay Arrow arrays, which the columnar entry reads in place (the offsets, the bytes, the integer ticks, the vector values and the validity bitmap) instead of as Python values; ``_insert_documents_native`` @@ -382,6 +429,10 @@ def _column_values(name: str, chunk, pa, native: bool = False, arrow: bool = Fal return chunk if arrow and kind == "list" and _is_number_list(pa, chunk.type): return chunk # a vector column: the columnar entry reads the values in place (to_pylist makes one Python float per element) + if float_vectors and kind == "list": + matrix = _float32_matrix(pa, chunk) + if matrix is not None: + return matrix if kind in ("timestamp", "date"): return ( _temporal_to_tagged(chunk, pa) if native else _temporal_to_java(chunk, pa) @@ -462,6 +513,7 @@ def insert_table( pa, native, arrow=native, + float_vectors=not native and declared.get(name) == "ARRAY_OF_FLOATS", ) for i, name in enumerate(names) } diff --git a/tests/conftest.py b/tests/conftest.py index 819fb71e..8b6cd23e 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -147,6 +147,7 @@ "test_graph_batch_single_pass.py": "the JVM wheel's graph batch bulk path (GraphBatch._create_vertices_json_bulk, the property-matrix fallback)", "test_graph_batch_vector_crossings_jvm.py": "the JPype crossings of the JVM wheel's create_vertices vector path (the bridge classes, the property-matrix fallback)", "test_insert_columns_string_blob.py": "the bridge's StringColumn and the JVM wheel's core._column_to_java (JPype arrays, a spy on the per-element conversion)", + "test_arrow_vector_column_jvm.py": "the JPype crossings of the JVM wheel's Arrow vector column (core.convert_python_to_java, the bridge's FloatVectors.Matrix) and its Python heap", "test_async_proxy_cache.py": "the JPype proxy classes of the JVM build's async executor", "test_read_crossings.py": "crossing counts of the JVM build's read path (JPype proxies of the Java objects and the bridge classes)", "test_reader_crossings.py": "crossing counts of the JVM build's record and row readers (JPype proxies of the Java objects and the bridge classes)", diff --git a/tests/test_arrow_vector_column_jvm.py b/tests/test_arrow_vector_column_jvm.py new file mode 100644 index 00000000..be7ba0cc --- /dev/null +++ b/tests/test_arrow_vector_column_jvm.py @@ -0,0 +1,309 @@ +"""insert_arrow and insert_dataframe with a vector column on the JVM wheel: the column crosses as one float32 matrix, not as one Python list of +Python floats per row, and what is stored is unchanged. + +A ``FixedSizeList`` (or list, or large list) column into an ``ARRAY_OF_FLOATS`` property went ``chunk.to_pylist()`` (one Python float +per element, 31.5 MiB of Python heap for one 10,000 x 96 batch against a 3.7 MiB flat buffer), then ``_column_to_java`` converted every row +with ``convert_python_to_java`` (an ``ArrayList`` of 96 boxed doubles per row, 100,000 conversions for the 10 x 10,000 rows of the +study) and the engine narrowed each list back to a ``float[]``. In the untimed profile that conversion was about 63 percent of the call, +and the boxed doubles are what the JVM heap held (``heap used`` 183 to 195 MiB at the end against 69 MiB live after a collection). Now a +float32 column with one length and no null stays an Arrow buffer, ``_float32_matrix`` lays it out as one 2-D float32 numpy matrix, and +``_column_to_java`` sends it as one ``float[]`` inside the bridge's ``FloatVectors.Matrix`` (the route a 2-D numpy column of +``insert_columns`` already took); the documents get a row each Java-side. + +Three tests fail on the code before (they count the per-row conversions, the matrix crossings and the Python heap of one batch); the others +pin what the old route stored, so the new one must store the same: count, an order-independent digest of every row's vector bytes read back +through SQL and through the result API, both again after close and reopen, the special float32 values, a null row, a null element, vectors +of different length, a wrong dimension against a vector index, float64 and integer lists (which are NOT taken: the old route refuses a +value past the float range and rounds the rest, a float32 cast would store an infinity), and the properties that are not declared +``ARRAY_OF_FLOATS`` (they keep a list). The JVM build only: it counts JPype crossings and reads Java-side values, so conftest.py does not +collect this file on the native wheel (whose vector columns stay Arrow arrays, test_native_columnar_memory.py). +""" + +import hashlib +import tracemalloc + +import numpy as np +import pyarrow as pa +import pytest + +import arcadedb +from arcadedb import core +from arcadedb.exceptions import ArcadeDBError + +F32 = np.float32 +SHAPES = ["fixed_size", "list", "large_list"] +TYPES = ["VERTEX", "DOCUMENT"] + + +def _matrix(n, dim, start=0, specials=True): + """float32 vectors, a pure function of the row number, with the values a decimal text of 6 digits would not hold in rows 0 and 1.""" + r = np.arange(start, start + n, dtype=np.int64) + j = np.arange(dim, dtype=np.int64) + m = (((r[:, None] * 257 + j[None, :] * 17) % 1000).astype(F32)) / F32(1000.0) + if specials and start == 0 and n > 1 and dim > 3: + m[0, 0] = F32(-0.0) + m[1, 0] = np.finfo(F32).max + m[1, 1] = np.finfo(F32).tiny + m[1, 2] = np.finfo(F32).smallest_subnormal + return m + + +def _column(matrix, shape): + dim = matrix.shape[1] + flat = pa.array(matrix.reshape(-1), type=pa.float32()) + if shape == "fixed_size": + return pa.FixedSizeListArray.from_arrays(flat, dim) + kind = pa.list_ if shape == "list" else pa.large_list + rows = [row.tolist() for row in matrix] + return pa.array(rows, type=kind(pa.float32())) + + +def _table(matrix, shape, start=0): + n = matrix.shape[0] + return pa.table({"k": pa.array(np.arange(start, start + n), type=pa.int32()), "embedding": _column(matrix, shape)}) + + +def _make(db, name, kind="VERTEX", embedding="ARRAY_OF_FLOATS", index=None): + db.command("sql", f"CREATE {kind} TYPE {name}") + db.command("sql", f"CREATE PROPERTY {name}.k INTEGER") + if embedding: + db.command("sql", f"CREATE PROPERTY {name}.embedding {embedding}") + if index: + db.command("sql", f"CREATE INDEX ON {name} (embedding) LSM_VECTOR METADATA {{dimensions: {index}, similarity: 'COSINE'}}") + + +def _count(db, name): + return db.query("sql", f"SELECT count(*) AS c FROM {name}").first().get("c") # nosec B608 - test-owned name + + +def _key(k, vector_bytes): + return int.from_bytes(hashlib.sha256(int(k).to_bytes(8, "little") + vector_bytes).digest()[:16], "little") + + +def _expected_digest(matrix, start=0): + """The digest the stored table must have, from the generator, never from the database.""" + h = 0 + for i, row in enumerate(matrix): + h ^= _key(start + i, row.tobytes()) + return h + + +def _sql_digest(db, name): + h, n = 0, 0 + for row in db.query("sql", f"SELECT k, embedding FROM {name}").to_list(): # nosec B608 - test-owned name + h ^= _key(row["k"], np.asarray(row["embedding"], dtype=np.float64).astype(F32).tobytes()) + n += 1 + return n, h + + +def _api_digest(db, name): + """The same digest through the result API (one Result per record, ``get`` per property), not through ``to_list``.""" + h, n = 0, 0 + result_set = db.query("sql", f"SELECT FROM {name}") # nosec B608 - test-owned name + for result in result_set: + h ^= _key(result.get("k"), np.asarray(result.get("embedding"), dtype=np.float64).astype(F32).tobytes()) + n += 1 + return n, h + + +def _bits(values): + return np.asarray(values, dtype=np.float64).astype(F32).view(np.uint32).tolist() + + +@pytest.fixture +def crossings(monkeypatch): + """What one load does on the JPype side: the per-value conversions and the bridge's Matrix constructions.""" + log = {"convert": [], "matrix": []} + real_convert = core.convert_python_to_java + real_class = core._java_class + + def convert(value): + log["convert"].append(type(value).__name__) + return real_convert(value) + + class _Counting: + def __init__(self, target): + self._target = target + + def __call__(self, *args): + log["matrix"].append(len(args[0])) + return self._target(*args) + + def __getattr__(self, name): + return getattr(self._target, name) + + def java_class(name): + cls = real_class(name) + return _Counting(cls) if name == "com.arcadedb.python.FloatVectors$Matrix" else cls + + monkeypatch.setattr(core, "convert_python_to_java", convert) + monkeypatch.setattr(core, "_java_class", java_class) + return log + + +@pytest.mark.parametrize("kind", TYPES) +@pytest.mark.parametrize("shape", SHAPES) +def test_count_digest_and_reopen_of_a_batched_vector_load(temp_db, temp_db_path_of, shape, kind): + """1,300 rows in batches of 500 (500, 500, 300: the batches are slices of one table): count, SQL digest, API digest, then the same after reopen.""" + n, dim = 1_300, 24 + matrix = _matrix(n, dim) + _make(temp_db, "Vec", kind) + assert temp_db.insert_arrow("Vec", _table(matrix, shape), batch_size=500) == n + want = (n, _expected_digest(matrix)) + assert _count(temp_db, "Vec") == n + assert _sql_digest(temp_db, "Vec") == want + assert _api_digest(temp_db, "Vec") == want + path = temp_db_path_of(temp_db) + temp_db.close() + reopened = arcadedb.open_database(path) + try: + assert _count(reopened, "Vec") == n + assert _sql_digest(reopened, "Vec") == want + assert _api_digest(reopened, "Vec") == want + finally: + reopened.close() + + +@pytest.fixture +def temp_db_path_of(): + def path_of(db): + return db.get_java_database().getDatabasePath() + + return path_of + + +def test_a_load_in_several_chunks_of_one_table_keeps_every_row(temp_db): + """Two tables concatenated (a ChunkedArray of two chunks) cut into batches of 4 (offsets that start inside a chunk).""" + a, b = _matrix(3, 6, 0), _matrix(4, 6, 3) + _make(temp_db, "Vec") + table = pa.concat_tables([_table(a, "fixed_size", 0), _table(b, "fixed_size", 3)]) + assert temp_db.insert_arrow("Vec", table, batch_size=4) == 7 + assert _sql_digest(temp_db, "Vec") == (7, _expected_digest(a, 0) ^ _expected_digest(b, 3)) + + +@pytest.mark.parametrize("shape", SHAPES) +def test_the_special_float32_values_are_stored_bit_for_bit(temp_db, shape): + special = np.array([[-0.0, np.finfo(F32).max, np.finfo(F32).tiny, np.finfo(F32).smallest_subnormal, np.nan, np.inf, -np.inf, 16777217.0]], dtype=F32) + _make(temp_db, "Vec") + assert temp_db.insert_arrow("Vec", _table(special, shape)) == 1 + stored = temp_db.query("sql", "SELECT embedding FROM Vec").first().get("embedding") + assert len(stored) == 8 + got = np.asarray(stored, dtype=np.float64).astype(F32) + assert np.isnan(got[4]) + keep = [0, 1, 2, 3, 5, 6, 7] + assert got.view(np.uint32)[keep].tolist() == special.view(np.uint32)[0, keep].tolist() + assert got[7] == F32(16777216.0) # 16777217.0 is not a float32; the row holds the float32 that was written + + +def test_a_null_row_stays_null_and_its_neighbours_are_exact(temp_db): + _make(temp_db, "Vec") + table = pa.table({"k": pa.array([0, 1, 2], pa.int32()), "embedding": pa.array([[1.0, 2.0], None, [3.0, 4.0]], pa.list_(pa.float32(), 2))}) + assert temp_db.insert_arrow("Vec", table) == 3 + rows = {r["k"]: r["embedding"] for r in temp_db.query("sql", "SELECT k, embedding FROM Vec").to_list()} + assert rows[1] is None + assert _bits(rows[0]) == _bits([1.0, 2.0]) and _bits(rows[2]) == _bits([3.0, 4.0]) + + +def test_a_null_element_is_refused_and_the_batch_is_rolled_back(temp_db): + _make(temp_db, "Vec") + table = pa.table({"k": pa.array([0, 1], pa.int32()), "embedding": pa.array([[1.0, None, 3.0], [4.0, 5.0, 6.0]], pa.list_(pa.float32()))}) + with pytest.raises(ArcadeDBError, match="null element"): + temp_db.insert_arrow("Vec", table) + assert _count(temp_db, "Vec") == 0 + + +def test_vectors_of_different_length_are_stored_as_given_without_an_index_and_refused_with_one(temp_db): + _make(temp_db, "Free") + ragged = pa.table({"k": pa.array([0, 1, 2], pa.int32()), "embedding": pa.array([[1.0, 2.0], [1.0, 2.0, 3.0], [4.0]], pa.list_(pa.float32()))}) + assert temp_db.insert_arrow("Free", ragged) == 3 + rows = {r["k"]: r["embedding"] for r in temp_db.query("sql", "SELECT k, embedding FROM Free").to_list()} + assert {k: len(v) for k, v in rows.items()} == {0: 2, 1: 3, 2: 1} + _make(temp_db, "Indexed", index=2) + with pytest.raises(ArcadeDBError, match="Vector dimension does not match index dimension 2: got float.. of length 3"): + temp_db.insert_arrow("Indexed", ragged.slice(0, 2)) + assert _count(temp_db, "Indexed") == 0 + + +def test_a_fixed_size_vector_of_the_wrong_dimension_is_refused_by_the_vector_index(temp_db): + _make(temp_db, "Indexed", index=2) + wrong = _table(_matrix(2, 3, specials=False), "fixed_size") + with pytest.raises(ArcadeDBError, match="Vector dimension does not match index dimension 2: got float.. of length 3"): + temp_db.insert_arrow("Indexed", wrong) + assert _count(temp_db, "Indexed") == 0 + right = _matrix(2, 2, specials=False) + assert temp_db.insert_arrow("Indexed", _table(right, "fixed_size")) == 2 + assert _sql_digest(temp_db, "Indexed") == (2, _expected_digest(right)) + + +def test_float64_and_integer_lists_keep_what_the_old_route_stored(temp_db): + """Not taken by the matrix route: a float64 value past the float range is refused (a cast would store an infinity), the rest rounds.""" + _make(temp_db, "Dbl") + ok = pa.table({"k": pa.array([0], pa.int32()), "embedding": pa.array([[0.1, 16777217.0, -2.5]], pa.list_(pa.float64()))}) + assert temp_db.insert_arrow("Dbl", ok) == 1 + got = temp_db.query("sql", "SELECT embedding FROM Dbl").first().get("embedding") + assert _bits(got) == _bits([0.1, 16777217.0, -2.5]) + assert got[1] == 16777216.0 + _make(temp_db, "Over") + over = pa.table({"k": pa.array([0], pa.int32()), "embedding": pa.array([[1e39, 1.0]], pa.list_(pa.float64()))}) + with pytest.raises(ArcadeDBError, match="1.0E39"): + temp_db.insert_arrow("Over", over) + assert _count(temp_db, "Over") == 0 + _make(temp_db, "Ints") + ints = pa.table({"k": pa.array([0], pa.int32()), "embedding": pa.array([[16777217, 2**40, -3]], pa.list_(pa.int64()))}) + assert temp_db.insert_arrow("Ints", ints) == 1 + assert temp_db.query("sql", "SELECT embedding FROM Ints").first().get("embedding") == [16777216.0, 1099511627776.0, -3.0] + + +@pytest.mark.parametrize("embedding", [None, "LIST"], ids=["undeclared", "declared_list"]) +def test_a_property_that_is_not_array_of_floats_keeps_a_list_and_takes_no_matrix(temp_db, crossings, embedding): + _make(temp_db, "Free", embedding=embedding) + matrix = _matrix(5, 4, specials=False) + assert temp_db.insert_arrow("Free", _table(matrix, "fixed_size")) == 5 + assert crossings["matrix"] == [] + assert _sql_digest(temp_db, "Free") == (5, _expected_digest(matrix)) + + +def test_empty_vectors_are_stored_as_empty_lists(temp_db): + _make(temp_db, "Vec") + table = pa.table({"k": pa.array([0, 1], pa.int32()), "embedding": pa.array([[], []], pa.list_(pa.float32()))}) + assert temp_db.insert_arrow("Vec", table) == 2 + assert [r["embedding"] for r in temp_db.query("sql", "SELECT embedding FROM Vec ORDER BY k").to_list()] == [[], []] + + +# ---- the three tests that fail on the code before the fix: they count what the load does, never how long it takes + + +@pytest.mark.parametrize("shape", SHAPES) +def test_a_vector_column_converts_no_python_value_per_row(temp_db, crossings, shape): + """Before: one ``convert_python_to_java`` per row (an ArrayList of boxed doubles). After: the vector column makes none.""" + n = 600 + _make(temp_db, "Vec") + assert temp_db.insert_arrow("Vec", _table(_matrix(n, 8), shape), batch_size=n) == n + assert crossings["convert"] == [] + + +def test_a_vector_column_crosses_as_one_matrix_per_batch(temp_db, crossings): + """1,300 rows in batches of 500: three batches, so three ``FloatVectors.Matrix`` of one flat float[] each (500 x 24, 500 x 24, 300 x 24).""" + _make(temp_db, "Vec") + assert temp_db.insert_arrow("Vec", _table(_matrix(1_300, 24), "fixed_size"), batch_size=500) == 1_300 + assert crossings["matrix"] == [500 * 24, 500 * 24, 300 * 24] + + +def test_the_python_heap_of_one_batch_stays_below_twice_the_flat_buffer(temp_db): + """10,000 x 96 float32 is a 3.66 MiB buffer; the per-row Python lists of floats were 31.5 MiB at their peak (tracemalloc sees Python and + numpy allocations, not the Java heap and not Arrow's pool). The bound is the buffer twice, nothing about time.""" + n, dim = 10_000, 96 + table = _table(_matrix(n, dim, specials=False), "fixed_size") + _make(temp_db, "Vec") + _make(temp_db, "Twin") + assert temp_db.insert_arrow("Twin", table.slice(0, 500)) == 500 # first-call imports and caches stay out of the measurement + flat = n * dim * 4 + tracemalloc.start() + try: + tracemalloc.reset_peak() + base = tracemalloc.get_traced_memory()[0] + assert temp_db.insert_arrow("Vec", table, batch_size=n) == n + peak = tracemalloc.get_traced_memory()[1] - base + finally: + tracemalloc.stop() + assert peak < 2 * flat, f"{peak} bytes of Python heap for a {flat} byte vector column" From aa223d06dc647ba0cb78682d5543f321226dfc6e Mon Sep 17 00:00:00 2001 From: Taewoon Kim Date: Tue, 10 Nov 2026 14:21:20 +0900 Subject: [PATCH 42/49] CI: Windows parity harness records crash exit codes, keeps a trace and retries a crashed file once (#295, #347) The Windows native parity legs of the 26.11.1.post1 release run lost one test process each (test_free_threading.py on 3.11, test_find_nearest_query_forms.py on 3.14): no traceback, no XML, no exit code, so nothing told an access violation from a runner kill. It did not reproduce in 360 isolated runs. The workflow and harness are read at the tag, so this has to ship in a new tag. Harness (native/wheel/parity_wheel.py, decision in the new parity_crash.py): - a file whose process ends with no junit result is CRASH(rc=0xC0000005,3221225477) (NTSTATUS hex and decimal; -11,SIGSEGV on POSIX), the exit code of every attempt goes to the tsv (two new columns, attempts and exit) and to .meta.json - --retry-crash (off by default; test-native.yml passes it when RUNNER_OS is Windows, so Linux and macOS stay strict) runs a crashed file once more in a fresh process: a retry that produces a result is FLAKY-CRASH(rc=), a second crash is CRASH(rc=;retry rc=) and fails the leg. A failing test, TIMEOUT and BADXML are never retried; a leg retries at most 3 files, so a wheel that crashes everywhere cannot pass on retries. The log of the first attempt is kept as .attempt1.log - the XML of an attempt is removed first, so a stale one cannot pass for a result Crash trace: tests/conftest.py disabled faulthandler on Windows because HotSpot raises handled access violations (the JVM build). The native wheel has no HotSpot, so a native run on Windows with ARCADEDB_TEST_FAULTHANDLER_FILE set (the harness sets it per attempt, on Windows only) enables faulthandler writing to that file; the JVM build and every other platform are unchanged (test_faulthandler_is_off_on_windows_only still holds). The files ride in the existing parity artifact (parity-wheel-xml/), empty ones are deleted. Loud: the gate awk (test job and pyver job) lets a FLAKY-CRASH row pass and prints it into the job summary; a new crash-report job lists every FLAKY-CRASH and CRASH of the run and, on a tag push or a dispatch only, posts ONE comment on issue 295 (run URL, attempt, Python, wheel, exit codes). It holds issues: write alone; the token failing, missing, a fork or a pull request only print a warning, it never fails the run. release-native.yml grants issues: write to its call of test-native.yml, because a called workflow cannot hold more permission than its caller (without that line the release run would refuse to start). Tests (tests/test_parity_crash_retry.py, tests/test_parity_gate_awk.py, no skips): the decision with a fake runner, real children that os._exit with a high code or die of a signal, the harness against a fixture repository with real pytest children, the faulthandler switch in a child that really faults, the report and its permission rules, the workflow wiring, and the real gate awk on small tables (the awk file is not collected on Windows). 12 mutations of the new code each fail at least one test; against the tree of release/26.11.1 the awk tests fail and the retry tests cannot import. Not exercised: Windows itself (NTSTATUS exit codes, the faulthandler trace on a real access violation in libarcadedb.dll) and the caller permission path of a tag run. Co-Authored-By: Claude Sonnet 5.5 --- .github/workflows/release-native.yml | 9 +- .github/workflows/test-native.yml | 51 +- docs/modules/ROOT/pages/development/ci.adoc | 1 + native/wheel/parity_crash.py | 159 +++++ native/wheel/parity_wheel.py | 431 ++++++++----- native/wheel/report_flaky_crash.py | 202 ++++++ tests/README.md | 2 + tests/conftest.py | 23 +- tests/test_parity_crash_retry.py | 662 ++++++++++++++++++++ tests/test_parity_gate_awk.py | 110 ++++ 10 files changed, 1495 insertions(+), 155 deletions(-) create mode 100644 native/wheel/parity_crash.py create mode 100644 native/wheel/report_flaky_crash.py create mode 100644 tests/test_parity_crash_retry.py create mode 100644 tests/test_parity_gate_awk.py diff --git a/.github/workflows/release-native.yml b/.github/workflows/release-native.yml index 4fee6176..a0a8bf1e 100644 --- a/.github/workflows/release-native.yml +++ b/.github/workflows/release-native.yml @@ -25,7 +25,8 @@ name: Release native (arcadedb) # # Nothing here stores a token: the publish job authenticates with PyPI trusted publishing (OIDC), and only that job has the id-token # permission. The guard in the publish job refuses to publish anything that is not `arcadedb`, so this workflow cannot publish the -# JVM `arcadedb-jvm` by accident. +# JVM `arcadedb-jvm` by accident. The test call carries issues: write too, for the one job of test-native.yml that logs a crash retry +# on issue 295 (see the comment at that call). on: push: @@ -89,6 +90,12 @@ jobs: name: Native build and parity needs: validate-version uses: ./.github/workflows/test-native.yml + # issues: write is for the `crash-report` job of test-native.yml alone (one comment on issue 295 when a Windows test file crashed and + # passed on its retry). A called workflow cannot hold more permission than its caller gives it, so without this line the whole release + # run would refuse to start; the called workflow's own `permissions: contents: read` keeps every other job read-only. + permissions: + contents: read + issues: write with: version: ${{ needs.validate-version.outputs.version }} release: true diff --git a/.github/workflows/test-native.yml b/.github/workflows/test-native.yml index e5a8671a..56b46feb 100644 --- a/.github/workflows/test-native.yml +++ b/.github/workflows/test-native.yml @@ -33,6 +33,12 @@ # # A leg that passes uploads a marker (native-passed--py) and the wheel (native-wheel-); release-native.yml # reads them to decide what may be published. Do not rename these artifacts without changing release-native.yml. +# +# Crashes (ArcadeData/arcadedb-python#295): a test file whose process dies without a result is a CRASH row with its exit code, and fails the leg. +# On the Windows legs only, the harness runs such a file once more in a fresh process (--retry-crash): a retry that produces a result is a +# FLAKY-CRASH row, the leg may pass, and the `crash-report` job makes it loud (the job summary, and on a release or a dispatch one comment on +# issue 295); a second crash, a failing test, a timeout and every Linux or macOS crash stay failures. Each Windows test process also writes a +# faulthandler trace into the parity artifact (parity-wheel-xml/.attemptN.fault.txt). name: Test native (arcadedb) on: @@ -568,12 +574,16 @@ jobs: cores="$(nproc 2>/dev/null || sysctl -n hw.ncpu 2>/dev/null || echo 2)" limit=() if [ "$FINAL" = true ]; then limit=(--max-notyet 0); fi + # A file whose process dies without a result is run once more on Windows only (the crashes of issue 295 are Windows only); + # Linux and macOS stay strict. The harness marks a passing retry FLAKY-CRASH and the crash-report job logs it. + retry=() + if [ "$RUNNER_OS" = Windows ]; then retry=(--retry-crash); fi # macOS runs bash 3.2: under set -u an empty "${limit[@]}" is "unbound variable", so expand it only when set uv run --no-project --python "$PY" --with "$w" --with pytest --with setuptools --with numpy --with requests --with jpype1 --with pyarrow \ --with polars --with pandas --with duckdb --with networkx --with "psycopg[binary]" --with adbc-driver-postgresql \ --with redis --with neo4j --with ladybug \ python native/wheel/parity_wheel.py --repo "$GITHUB_WORKSPACE" \ - --out "$T/parity/parity-wheel.tsv" --cores "0-$((cores - 1))" ${limit[@]+"${limit[@]}"} | tee "$T/parity/parity-wheel.run.log" + --out "$T/parity/parity-wheel.tsv" --cores "0-$((cores - 1))" ${limit[@]+"${limit[@]}"} ${retry[@]+"${retry[@]}"} | tee "$T/parity/parity-wheel.run.log" - name: "Gate: no failure, error, crash, timeout or uncounted skip" run: | @@ -583,10 +593,13 @@ jobs: awk -F'\t' ' NR == 1 { next } $1 == "TOTAL" { passed = $2; failed = $3; err = $4; notyet = $5; na = $6; skipped = $7; tot = 1; next } + $8 ~ /^FLAKY-CRASH/ { flaky = flaky " " $1 "(" $8 ")"; next } $8 != "" { bad = bad " " $1 "(" $8 ")" } END { if (!tot) { print "no TOTAL row" > "/dev/stderr"; exit 1 } printf "parity %s py%s: passed=%s failed=%s error=%s notyet=%s na_jvm=%s skipped=%s\n", ENVIRON["KEY"], ENVIRON["PY"], passed, failed, err, notyet, na, skipped + # a file that crashed and passed when run once more: the leg may pass, the summary says it did only because of the retry + if (flaky != "") printf "FLAKY-CRASH (died without a result, passed on the one retry; first exit code in brackets):%s\n", flaky if (bad != "") { print "files with a status:" bad > "/dev/stderr"; exit 1 } if (passed + 0 == 0 || failed + 0 != 0 || err + 0 != 0) exit 1 # this repository accepts no skips; NOTYET and NA-JVM are the two counted kinds of the native build @@ -614,11 +627,44 @@ jobs: name: native-parity-${{ matrix.key }}-py${{ matrix.python }} path: | ${{ runner.temp }}/parity/parity-wheel.tsv + ${{ runner.temp }}/parity/parity-wheel.meta.json ${{ runner.temp }}/parity/parity-wheel.run.log ${{ runner.temp }}/parity/parity-wheel-xml/ if-no-files-found: warn retention-days: 30 + # The one job that may write to an issue: after the parity legs it reads their meta files (parity-wheel.meta.json in each native-parity-* + # artifact) and lists every FLAKY-CRASH and CRASH in the job summary; on a release (a tag push) or a dispatch it also posts ONE comment on + # issue 295 (native/wheel/report_flaky_crash.py decides: never on a pull request, only in ArcadeData/arcadedb-python). A log is not a gate: a token + # that cannot write issues, a missing artifact or a gh failure print a warning and the job still ends green. release-native.yml grants the + # issues permission to its call of this workflow, because a called workflow cannot hold more permission than its caller gives it. + crash-report: + name: Report parity crashes + needs: [plan, test] + if: ${{ !cancelled() && needs.plan.outputs.run == 'true' }} + runs-on: ubuntu-24.04 + timeout-minutes: 10 + continue-on-error: true + permissions: + contents: read + issues: write + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + + - uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333 # v8.0.2 + continue-on-error: true + with: + pattern: native-parity-* + path: ${{ runner.temp }}/parity-artifacts + + - name: List the crashes and log them on issue 295 + continue-on-error: true + env: + GH_TOKEN: ${{ github.token }} + run: python3 -I native/wheel/report_flaky_crash.py --artifacts "$RUNNER_TEMP/parity-artifacts" + # The examples of examples/ against the INSTALLED native wheel, so the scripts a new user copies are run on what `uv add arcadedb` installs. # Nothing is built here: the wheel is the one the build job uploaded (native-wheel-), the file the parity job installs. The platforms are # those of the parity job and the Pythons are 3.12 and 3.14 (scripts/select_native_matrix.py, `examples`). The runner is scripts/run_examples.sh, the @@ -949,10 +995,13 @@ jobs: awk -F'\t' ' NR == 1 { next } $1 == "TOTAL" { passed = $2; failed = $3; err = $4; notyet = $5; na = $6; skipped = $7; tot = 1; next } + $8 ~ /^FLAKY-CRASH/ { flaky = flaky " " $1 "(" $8 ")"; next } $8 != "" { bad = bad " " $1 "(" $8 ")" } END { if (!tot) { print "no TOTAL row" > "/dev/stderr"; exit 1 } printf "parity %s py%s: passed=%s failed=%s error=%s notyet=%s na_jvm=%s skipped=%s\n", ENVIRON["KEY"], ENVIRON["PY"], passed, failed, err, notyet, na, skipped + # a file that crashed and passed when run once more: the leg may pass, the summary says it did only because of the retry + if (flaky != "") printf "FLAKY-CRASH (died without a result, passed on the one retry; first exit code in brackets):%s\n", flaky if (bad != "") { print "files with a status:" bad > "/dev/stderr"; exit 1 } if (passed + 0 == 0 || failed + 0 != 0 || err + 0 != 0) exit 1 # this repository accepts no skips; NOTYET and NA-JVM are the two counted kinds of the native build diff --git a/docs/modules/ROOT/pages/development/ci.adoc b/docs/modules/ROOT/pages/development/ci.adoc index e8d92e94..f665f9d1 100644 --- a/docs/modules/ROOT/pages/development/ci.adoc +++ b/docs/modules/ROOT/pages/development/ci.adoc @@ -98,6 +98,7 @@ The workflow `test-native.yml` builds the native image of the `arcadedb` package * A pull request runs it only when native paths change: `native/**`, `src/arcadedb/**`, `tests/**`, `scripts/select_native_matrix.py` or `test-native.yml` itself. Changes under `tests/` and `src/arcadedb/` are included because both backends share those files, so a shared test or source can break native parity without touching a native file; docs outside those paths do not run it. It then runs on `linux-x86_64` with Python 3.12 only. * A push to `main`, the nightly run, a manual run (dispatch) run all four platforms: `linux-x86_64`, `linux-aarch64`, `macos-arm64` and `windows-x64` (Python 3.12 and 3.14). A release (`release-native.yml` calls the workflow with `release: true`) runs the same four platforms on all four Python versions, 16 legs. * The `pyver` job installs the already built `linux-x86_64` wheel (no rebuild) into the Pythons that the full parity runs of the event leave out (3.11, 3.13 and 3.14 on a pull request; 3.11 and 3.13 on a push to `main`, the nightly run and a dispatch; none on a release) into 3.15 and into the free-threaded 3.14t, and runs 13 test files there (14 where polars installs; 3.15 has no jpype1 wheel, so it leaves out the two files that need jpype1), about 20 seconds of tests per Python. It is informational: `release-native.yml` reads only the `native-passed-*` markers and the `native-wheel-*` wheels, so a red `pyver` leg never blocks a release. The 3.14t leg has no polars (no free-threaded wheel exists), so it skips `test_dataframe_ingest.py`. `scripts/select_native_matrix.py` decides its legs. +* Every test file runs in its own process (`native/wheel/parity_wheel.py`). A process that dies without writing its result is a `CRASH` row in `parity-wheel.tsv` with its exit code, for example `CRASH(rc=0xC0000005,3221225477)`, and it fails the leg. On the Windows legs only (the crashes of issue 295 happen there), the harness runs such a file once more in a fresh process (`--retry-crash`). A retry that produces a result is a `FLAKY-CRASH` row: the gate lets the leg pass, the job summary lists the file with the first exit code, and on a release or a dispatch the `crash-report` job adds one comment to issue 295 (run URL, Python, wheel version, exit codes). A second crash, a failing test, a timeout and every crash on Linux or macOS stay failures. Each Windows test process also writes a faulthandler trace, `parity-wheel-xml/.attemptN.fault.txt`, into the parity artifact of the leg. Only a retry hides a crash, so treat a `FLAKY-CRASH` as an open bug: read the trace in the artifact before a release is announced. * A native image build takes about 5 to 30 minutes for each platform (macOS is the slowest). The organisation shares one runner limit with upstream ArcadeDB, so do not add runs lightly. Change the paths or the matrix of `test-native.yml` only for a reason that you can write down. == Change the selection diff --git a/native/wheel/parity_crash.py b/native/wheel/parity_crash.py new file mode 100644 index 00000000..a82ccc0e --- /dev/null +++ b/native/wheel/parity_crash.py @@ -0,0 +1,159 @@ +"""parity_crash.py: what native/wheel/parity_wheel.py does when a test file's process dies, as pure code (no files, no subprocess). + +A test file runs in its own process. Four things can happen to that process: + + result pytest wrote its junit XML. Pass or fail is read from the XML; the exit code is not consulted (a failing test is a + failure, never a crash, and it is never retried). + TIMEOUT the harness killed it after --timeout. Strict, never retried. + BADXML the XML exists and does not parse. Strict, never retried. + crash no XML at all: the process died before pytest wrote its result (access violation, abort, kill by the runner, an + os._exit in a library). The exit code is the only evidence left, so it is recorded: CRASH(rc=0xC0000005,3221225477). + +Retry (off by default, switched on with --retry-crash, which test-native.yml passes on Windows legs only): a crash is run once more in +a fresh process. If the second run produces a result the file is marked FLAKY-CRASH(rc=) and counted from the second run (a +test failure in it still fails the leg); if it crashes again the status is CRASH(rc=;retry rc=) and the leg fails. A +retry budget per leg (default 3 files) keeps a wheel that crashes everywhere from passing on retries. + +Everything here takes the facts of an attempt (Attempt) and returns the status and the attempts; the harness runs the processes. +""" + +from __future__ import annotations + +import signal +import sys +from dataclasses import dataclass +from typing import Callable, Optional, Tuple + +CRASH = "CRASH" +FLAKY_CRASH = "FLAKY-CRASH" +TIMEOUT = "TIMEOUT" +BADXML = "BADXML" + +# The exit codes pytest itself returns (0 ok, 1 failures, 2 interrupted or collection errors, 3 internal error, 4 usage, 5 nothing collected). +# Any other code after the XML was written means the process died later, in the interpreter's or a library's shutdown. +PYTEST_EXIT_CODES = range(0, 6) + + +def format_rc(rc: Optional[int], windows: Optional[bool] = None) -> str: + """The exit code the way a crash is read: 0xC0000005,3221225477 (Windows NTSTATUS) or -11,SIGSEGV (a POSIX signal). + + Python reports an unsigned 32-bit code for a Windows process and the negative signal number for a POSIX one; a negative code on + Windows is the same NTSTATUS read as signed, so it is folded back to 32 bits there. + """ + if rc is None: + return "none" + if windows is None: + windows = sys.platform == "win32" + if windows or rc >= 0: + return f"0x{rc & 0xFFFFFFFF:08X},{rc & 0xFFFFFFFF}" + try: + name = signal.Signals(-rc).name + except ValueError: + name = f"signal {-rc}" + return f"{rc},{name}" + + +@dataclass(frozen=True) +class Attempt: + """The facts about one run of one test file.""" + + rc: Optional[int] # the exit code; None when the harness killed the process (timed_out) + result: bool # the junit XML was written and parses + timed_out: bool = False + bad_xml: bool = False # the XML exists and does not parse + + +def is_crash(attempt: Attempt) -> bool: + """No result of any kind and not killed by the harness: the process died on its own.""" + return not (attempt.result or attempt.timed_out or attempt.bad_xml) + + +def dirty_exit(attempt: Attempt) -> bool: + """The XML was written but the process ended with a code pytest never returns: it died in shutdown, after the tests.""" + return attempt.result and attempt.rc is not None and attempt.rc not in PYTEST_EXIT_CODES + + +@dataclass +class RetryPolicy: + """Whether a crashed file may be run again, and how many files of one leg may be.""" + + enabled: bool = False + budget: int = 3 + used: int = 0 + + def take(self) -> bool: + if not self.enabled or self.used >= self.budget: + return False + self.used += 1 + return True + + +@dataclass(frozen=True) +class Outcome: + attempts: Tuple[Attempt, ...] + status: str # "" when the file produced a result; else TIMEOUT, BADXML, CRASH(...) or FLAKY-CRASH(...) + + @property + def final(self) -> Attempt: + return self.attempts[-1] + + @property + def retried(self) -> bool: + return len(self.attempts) > 1 + + @property + def flaky(self) -> bool: + return self.status.startswith(FLAKY_CRASH) + + @property + def hard_crash(self) -> bool: + return self.status.startswith(CRASH) + + @property + def timed_out(self) -> bool: + return self.final.timed_out + + +def run_file(run_attempt: Callable[[int], Attempt], policy: RetryPolicy) -> Outcome: + """Run one test file by `run_attempt(1)` and, only for a crash and only when the policy allows, once more by `run_attempt(2)`.""" + first = run_attempt(1) + if not is_crash(first): + return Outcome((first,), TIMEOUT if first.timed_out else BADXML if first.bad_xml else "") + crash = f"rc={format_rc(first.rc)}" + if not policy.take(): + why = ";retry budget spent" if policy.enabled else "" + return Outcome((first,), f"{CRASH}({crash}{why})") + second = run_attempt(2) + if second.result: + return Outcome((first, second), f"{FLAKY_CRASH}({crash})") + if second.timed_out: + again = "TIMEOUT" + elif second.bad_xml: + again = "BADXML" + else: + again = f"rc={format_rc(second.rc)}" + return Outcome((first, second), f"{CRASH}({crash};retry {again})") + + +def event_of(name: str, outcome: Outcome, final_failed: int) -> Optional[dict]: + """The record of a crash for the metadata file (parity-wheel.meta.json), or None for a file that never crashed. + + `final_failed` is the number of failed or errored tests of the run that produced the result (0 when there is none). + """ + if not (outcome.flaky or outcome.hard_crash): + return None + first, last = outcome.attempts[0], outcome.final + if outcome.flaky: + retry = "passed" if final_failed == 0 else f"ran, {final_failed} failed" + elif outcome.retried: + retry = "crashed again" + else: + retry = "not retried" + return { + "file": name, + "kind": FLAKY_CRASH if outcome.flaky else CRASH, + "first_rc": format_rc(first.rc), + "retry_rc": format_rc(last.rc) if outcome.retried else "", + "retry": retry, + "status": outcome.status, + } diff --git a/native/wheel/parity_wheel.py b/native/wheel/parity_wheel.py index 8cc094fc..bbefa233 100644 --- a/native/wheel/parity_wheel.py +++ b/native/wheel/parity_wheel.py @@ -6,172 +6,299 @@ A run per test file in its own process (a hang or crash costs one file), junit XML per file, then one table: passed / failed / error / NOTYET (the test needs a Java-object API, reported by tests/conftest.py as a skip whose reason starts with NOTYET, raised as NotOnNativeYet) / NA-JVM (a row of NATIVE_NA_JVM_TESTS: the test reaches into the JVM on purpose) / other skips / TIMEOUT-or-crash. Files in tests/conftest.py's NATIVE_NOT_APPLICABLE are not collected (JVM-only). + +A file whose process dies without writing its result is a CRASH, recorded with its exit code: CRASH(rc=0xC0000005,3221225477). With +--retry-crash (test-native.yml passes it on Windows legs only; the default is strict) the file is run once more in a fresh process: a retry +that produces a result is FLAKY-CRASH(rc=...) in the status column (the gate lets it pass, the job summary and issue 295 record it), a second +crash stays CRASH and fails the leg. A failing test, a TIMEOUT and a BADXML are never retried. The decision is native/wheel/parity_crash.py. +On Windows each process also writes a faulthandler trace to .attemptN.fault.txt next to its XML (tests/conftest.py, +ARCADEDB_TEST_FAULTHANDLER_FILE); an empty one is deleted. Beside the table, .meta.json lists the wheel, the Python and every crash. """ import argparse +import ast import glob +import json import os import shutil import subprocess import sys import xml.etree.ElementTree as ET -p = argparse.ArgumentParser() -p.add_argument("--repo", required=True) -p.add_argument("--python", default=sys.executable) -p.add_argument("--out", required=True) -p.add_argument("--timeout", type=int, default=900) -p.add_argument( - "--cores", - default="0-5,8-11", - help="taskset core list (Linux only; ignored where taskset does not exist)", -) -p.add_argument("--files", default="") -p.add_argument( - "--max-notyet", - type=int, - default=None, - help="fail (exit 1) when more tests than this are NOTYET skips; a final release uses 0", -) -a = p.parse_args() - -pkg = os.path.abspath(a.repo) -files = sorted(glob.glob(os.path.join(pkg, "tests", "test_*.py"))) -import ast +import parity_crash as pc # the directory of this script is sys.path[0] + +COUNT_KEYS = ["passed", "failed", "error", "notyet", "na_jvm", "skipped"] + +# On Windows the test process writes a faulthandler trace to this file (tests/conftest.py, native backend only). +FAULT_FILE_ENV = "ARCADEDB_TEST_FAULTHANDLER_FILE" -na_files, win_files = {}, {} -for node in ast.parse(open(os.path.join(pkg, "tests", "conftest.py")).read()).body: - if isinstance(node, ast.Assign): - name = getattr(node.targets[0], "id", "") - if name == "NATIVE_NOT_APPLICABLE": - na_files = ast.literal_eval(node.value) - elif name == "WINDOWS_NOT_APPLICABLE": - win_files = ast.literal_eval(node.value) -files = [f for f in files if os.path.basename(f) not in na_files] -if ( - sys.platform == "win32" -): # conftest.py's collect_ignore does not apply to a file named on the command line - files = [f for f in files if os.path.basename(f) not in win_files] -if not os.path.isfile(os.path.join(pkg, "docs", "index.md")): - # tests/conftest.py collect-ignores test_docs_examples.py without the Markdown docs site; a file named on the command line is collected anyway - files = [f for f in files if os.path.basename(f) != "test_docs_examples.py"] -if a.files: - want = set(a.files.split(",")) - files = [f for f in files if os.path.basename(f) in want] -env = dict(os.environ, PYTHONUNBUFFERED="1") -env.pop("PYTHONPATH", None) # the wheel, not a source tree, must be what is imported -env.pop("ARCADEDB_BACKEND", None) # the backend comes from the installed distribution -env.pop("ARCADEDB_NATIVE_LIB", None) # the library comes from the installed wheel -chk = subprocess.run( - [ - a.python, - "-I", - "-c", - "import arcadedb as n; print(n.__file__, n.__version__, n.__backend__, n.ENGINE_VERSION, n.SOURCE_COMMIT); " - 'assert n.__backend__ == "native"; assert "site-packages" in n.__file__', - ], - env=env, - cwd="/", - capture_output=True, - text=True, -) -print("WHEEL CHECK:", chk.stdout.strip(), chk.stderr.strip()[-300:], flush=True) -assert chk.returncode == 0, "tests would not run against the installed wheel" -a.out = os.path.abspath(a.out) -xmldir = os.path.splitext(a.out)[0] + "-xml" -os.makedirs(xmldir, exist_ok=True) -rows, tot = [], dict.fromkeys( - ["passed", "failed", "error", "notyet", "na_jvm", "skipped", "timeout"], 0 -) -for f in files: - name = os.path.basename(f) - xml = os.path.join(xmldir, name + ".xml") - cmd = [ - *( - ["taskset", "-c", a.cores] - if sys.platform == "linux" and shutil.which("taskset") - else [] - ), - *( - ["nice", "-n", "10"] - if sys.platform != "win32" and shutil.which("nice") - else [] - ), - a.python, - "-m", - "pytest", - # No --capture=sys on Windows: pytest's default fd capture (dup2 onto fd 1 and 2) is the regression check for engine log - # text landing in database files there (tests/test_log_redirect.py is the direct test; the whole-file runs of test_vector.py - # were the ones that failed before the engine stopped logging to the console on Windows). - f"tests/{name}", - "-q", - "-p", - "no:cacheprovider", - f"--junitxml={xml}", - "-o", - "junit_family=xunit1", - ] + +def parse_args(argv=None): + p = argparse.ArgumentParser() + p.add_argument("--repo", required=True) + p.add_argument("--python", default=sys.executable) + p.add_argument("--out", required=True) + p.add_argument("--timeout", type=int, default=900) + p.add_argument( + "--cores", + default="0-5,8-11", + help="taskset core list (Linux only; ignored where taskset does not exist)", + ) + p.add_argument("--files", default="") + p.add_argument( + "--max-notyet", + type=int, + default=None, + help="fail (exit 1) when more tests than this are NOTYET skips; a final release uses 0", + ) + p.add_argument( + "--retry-crash", + action="store_true", + help="run a test file whose process died without a result ONCE more in a fresh process (FLAKY-CRASH when that " + "produces a result, CRASH when it dies again). Off by default: the workflow sets it on Windows legs only.", + ) + p.add_argument( + "--max-crash-retries", + type=int, + default=3, + help="with --retry-crash, the most files of one run that may be retried; a crash after that stays a CRASH", + ) + return p.parse_args(argv) + + +def write_text(path, text): + with open(path, "w") as fh: + fh.write(text) + + +def count_xml(xml): + """(counts, names of the failing tests) of a junit XML, or None when it does not parse.""" + c = dict.fromkeys(COUNT_KEYS, 0) + fails = [] try: - r = subprocess.run( - cmd, cwd=pkg, env=env, capture_output=True, text=True, timeout=a.timeout + for tc in ET.parse(xml).getroot().iter("testcase"): + if tc.find("failure") is not None: + c["failed"] += 1 + fails.append(tc.get("name")) + elif tc.find("error") is not None: + c["error"] += 1 + fails.append(tc.get("name")) + elif tc.find("skipped") is not None: + msg = tc.find("skipped").get("message") or "" + c[ + ( + "notyet" + if msg.startswith("NOTYET") + else "na_jvm" if msg.startswith("NA-JVM") else "skipped" + ) + ] += 1 + else: + c["passed"] += 1 + except ET.ParseError: + return None + return c, fails + + +def main(argv=None): + a = parse_args(argv) + pkg = os.path.abspath(a.repo) + files = sorted(glob.glob(os.path.join(pkg, "tests", "test_*.py"))) + + na_files, win_files = {}, {} + for node in ast.parse(open(os.path.join(pkg, "tests", "conftest.py")).read()).body: + if isinstance(node, ast.Assign): + name = getattr(node.targets[0], "id", "") + if name == "NATIVE_NOT_APPLICABLE": + na_files = ast.literal_eval(node.value) + elif name == "WINDOWS_NOT_APPLICABLE": + win_files = ast.literal_eval(node.value) + files = [f for f in files if os.path.basename(f) not in na_files] + if ( + sys.platform == "win32" + ): # conftest.py's collect_ignore does not apply to a file named on the command line + files = [f for f in files if os.path.basename(f) not in win_files] + if not os.path.isfile(os.path.join(pkg, "docs", "index.md")): + # tests/conftest.py collect-ignores test_docs_examples.py without the Markdown docs site; a file named on the command line is collected anyway + files = [f for f in files if os.path.basename(f) != "test_docs_examples.py"] + if a.files: + want = set(a.files.split(",")) + files = [f for f in files if os.path.basename(f) in want] + env = dict(os.environ, PYTHONUNBUFFERED="1") + env.pop("PYTHONPATH", None) # the wheel, not a source tree, must be what is imported + env.pop("ARCADEDB_BACKEND", None) # the backend comes from the installed distribution + env.pop("ARCADEDB_NATIVE_LIB", None) # the library comes from the installed wheel + env.pop(FAULT_FILE_ENV, None) # set per attempt below, on Windows only + chk = subprocess.run( + [ + a.python, + "-I", + "-c", + "import json, sys, arcadedb as n; print(n.__file__, n.__version__, n.__backend__, n.ENGINE_VERSION, n.SOURCE_COMMIT); " + 'print("META", json.dumps({"python": sys.version.split()[0], "wheel_version": n.__version__, "engine_version": n.ENGINE_VERSION, "source_commit": n.SOURCE_COMMIT})); ' + 'assert n.__backend__ == "native"; assert "site-packages" in n.__file__', + ], + env=env, + cwd="/", + capture_output=True, + text=True, + ) + print("WHEEL CHECK:", chk.stdout.strip(), chk.stderr.strip()[-300:], flush=True) + assert chk.returncode == 0, "tests would not run against the installed wheel" + ident = {} + for line in chk.stdout.splitlines(): + if line.startswith("META "): + ident = json.loads(line[len("META ") :]) + a.out = os.path.abspath(a.out) + xmldir = os.path.splitext(a.out)[0] + "-xml" + os.makedirs(xmldir, exist_ok=True) + policy = pc.RetryPolicy(enabled=a.retry_crash, budget=a.max_crash_retries) + if policy.enabled: + print( + f"CRASH RETRY: on, a file that dies without a result is run once more (at most {policy.budget} files)", + flush=True, ) - status = "" - open(os.path.join(xmldir, name + ".log"), "w").write( - r.stdout[-20000:] + "\n" + r.stderr[-5000:] + rows, events = [], [] + tot = dict.fromkeys([*COUNT_KEYS, "timeout", "flaky_crash", "crash"], 0) + + for f in files: + name = os.path.basename(f) + xml = os.path.join(xmldir, name + ".xml") + log = os.path.join(xmldir, name + ".log") + parsed = {} # attempt number -> (counts, failing tests) of its XML + + def attempt(n): + if n > 1: + # keep the log of the first attempt: the retry writes name.log + os.replace(log, os.path.join(xmldir, name + ".attempt1.log")) + if os.path.exists(xml): + os.remove(xml) # a stale XML of an earlier run must not pass for a result + trace = os.path.join(xmldir, f"{name}.attempt{n}.fault.txt") + env_n = dict(env) + if sys.platform == "win32": + env_n[FAULT_FILE_ENV] = trace + cmd = [ + *( + ["taskset", "-c", a.cores] + if sys.platform == "linux" and shutil.which("taskset") + else [] + ), + *( + ["nice", "-n", "10"] + if sys.platform != "win32" and shutil.which("nice") + else [] + ), + a.python, + "-m", + "pytest", + # No --capture=sys on Windows: pytest's default fd capture (dup2 onto fd 1 and 2) is the regression check for engine log + # text landing in database files there (tests/test_log_redirect.py is the direct test; the whole-file runs of test_vector.py + # were the ones that failed before the engine stopped logging to the console on Windows). + f"tests/{name}", + "-q", + "-p", + "no:cacheprovider", + f"--junitxml={xml}", + "-o", + "junit_family=xunit1", + ] + timed_out, rc = False, None + try: + r = subprocess.run( + cmd, + cwd=pkg, + env=env_n, + capture_output=True, + text=True, + timeout=a.timeout, + ) + rc = r.returncode + write_text(log, r.stdout[-20000:] + "\n" + r.stderr[-5000:]) + except subprocess.TimeoutExpired as e: + timed_out = True + write_text( + log, + ( + ((e.stdout or b"")[-8000:]).decode() + if isinstance(e.stdout, bytes) + else str(e.stdout)[-8000:] + ), + ) + if os.path.exists(trace) and os.path.getsize(trace) == 0: + os.remove(trace) + result, bad_xml = False, False + if os.path.exists(xml): + got = count_xml(xml) + if got is None: + bad_xml = True + else: + result, parsed[n] = True, got + return pc.Attempt( + rc=rc, result=result, timed_out=timed_out, bad_xml=bad_xml + ) + + out = pc.run_file(attempt, policy) + c, fails = parsed.get(len(out.attempts), (dict.fromkeys(COUNT_KEYS, 0), [])) + for k in c: + tot[k] += c[k] + tot["timeout"] += out.timed_out + tot["flaky_crash"] += out.flaky + tot["crash"] += out.hard_crash + ev = pc.event_of(name, out, c["failed"] + c["error"]) + if ev: + for n in range(1, len(out.attempts) + 1): + trace = f"{name}.attempt{n}.fault.txt" + if os.path.exists(os.path.join(xmldir, trace)): + ev[f"trace{n}"] = trace + events.append(ev) + rows.append((name, c, out.status, fails, len(out.attempts), out.final.rc)) + print( + f"{name:45s} pass={c['passed']:4d} fail={c['failed']:3d} err={c['error']:3d} notyet={c['notyet']:3d} na_jvm={c['na_jvm']:3d} skip={c['skipped']:3d} {out.status}", + flush=True, ) - except subprocess.TimeoutExpired as e: - status = "TIMEOUT" - open(os.path.join(xmldir, name + ".log"), "w").write( - ((e.stdout or b"")[-8000:]).decode() - if isinstance(e.stdout, bytes) - else str(e.stdout)[-8000:] + if out.retried: + print( + f" {name}: the first run died without a result ({pc.format_rc(out.attempts[0].rc)}); " + f"the run once more: {'a result' if out.final.result else 'no result again'} ({pc.format_rc(out.final.rc)})", + flush=True, + ) + for n, at in enumerate(out.attempts, 1): + if pc.dirty_exit(at): + print( + f"::warning::{name} attempt {n} wrote its result and then exited with {pc.format_rc(at.rc)}", + flush=True, + ) + with open(a.out, "w") as fh: + fh.write( + "file\tpassed\tfailed\terror\tnotyet\tna_jvm\tskipped\tstatus\tfailing_tests\tattempts\texit\n" ) - c = dict.fromkeys(["passed", "failed", "error", "notyet", "na_jvm", "skipped"], 0) - fails = [] - if os.path.exists(xml): - try: - for tc in ET.parse(xml).getroot().iter("testcase"): - if tc.find("failure") is not None: - c["failed"] += 1 - fails.append(tc.get("name")) - elif tc.find("error") is not None: - c["error"] += 1 - fails.append(tc.get("name")) - elif tc.find("skipped") is not None: - msg = tc.find("skipped").get("message") or "" - c[ - ( - "notyet" - if msg.startswith("NOTYET") - else "na_jvm" if msg.startswith("NA-JVM") else "skipped" - ) - ] += 1 - else: - c["passed"] += 1 - except ET.ParseError: - status = status or "BADXML" - else: - status = status or "CRASH" - for k in c: - tot[k] += c[k] - if status == "TIMEOUT": - tot["timeout"] += 1 - rows.append((name, c, status, fails)) - print( - f"{name:45s} pass={c['passed']:4d} fail={c['failed']:3d} err={c['error']:3d} notyet={c['notyet']:3d} na_jvm={c['na_jvm']:3d} skip={c['skipped']:3d} {status}", - flush=True, - ) -with open(a.out, "w") as fh: - fh.write( - "file\tpassed\tfailed\terror\tnotyet\tna_jvm\tskipped\tstatus\tfailing_tests\n" - ) - for name, c, status, fails in rows: + for name, c, status, fails, attempts, rc in rows: + fh.write( + f"{name}\t{c['passed']}\t{c['failed']}\t{c['error']}\t{c['notyet']}\t{c['na_jvm']}\t{c['skipped']}\t{status}\t{';'.join(fails[:40])}\t{attempts}\t{pc.format_rc(rc)}\n" + ) fh.write( - f"{name}\t{c['passed']}\t{c['failed']}\t{c['error']}\t{c['notyet']}\t{c['na_jvm']}\t{c['skipped']}\t{status}\t{';'.join(fails[:40])}\n" + f"TOTAL\t{tot['passed']}\t{tot['failed']}\t{tot['error']}\t{tot['notyet']}\t{tot['na_jvm']}\t{tot['skipped']}\ttimeouts={tot['timeout']} flaky_crash={tot['flaky_crash']} crash={tot['crash']}\t\t\t\n" ) - fh.write( - f"TOTAL\t{tot['passed']}\t{tot['failed']}\t{tot['error']}\t{tot['notyet']}\t{tot['na_jvm']}\t{tot['skipped']}\ttimeouts={tot['timeout']}\t\n" - ) -print("TOTAL", tot) -if a.max_notyet is not None and tot["notyet"] > a.max_notyet: - sys.exit(f"{tot['notyet']} tests are NOTYET skips, above the limit {a.max_notyet}") + with open(os.path.splitext(a.out)[0] + ".meta.json", "w") as fh: + json.dump( + { + "key": env.get("KEY", ""), + "platform": sys.platform, + "python": ident.get("python", ""), + "wheel_version": ident.get("wheel_version", ""), + "engine_version": ident.get("engine_version", ""), + "source_commit": ident.get("source_commit", ""), + "retry_crash": policy.enabled, + "crash_events": events, + }, + fh, + indent=1, + ) + print("TOTAL", tot) + if a.max_notyet is not None and tot["notyet"] > a.max_notyet: + sys.exit( + f"{tot['notyet']} tests are NOTYET skips, above the limit {a.max_notyet}" + ) + + +if __name__ == "__main__": + main() diff --git a/native/wheel/report_flaky_crash.py b/native/wheel/report_flaky_crash.py new file mode 100644 index 00000000..58bbbff2 --- /dev/null +++ b/native/wheel/report_flaky_crash.py @@ -0,0 +1,202 @@ +#!/usr/bin/env python3 +"""report_flaky_crash.py: make a parity crash loud, once per workflow run. + +native/wheel/parity_wheel.py retries a test file whose process died without a result (Windows legs only) and writes +.meta.json beside its table, with one crash event per file: FLAKY-CRASH (the retry produced a result, so the leg may pass) +or CRASH (it died again, or no retry was allowed, so the leg fails). This script reads the meta files of every leg (the downloaded +`native-parity-*` artifacts), then + + 1. appends a table to the job summary ($GITHUB_STEP_SUMMARY), and + 2. posts ONE comment on issue 295 listing every event of the run (run URL, run attempt, Python, wheel, exit codes), but only on a + workflow_dispatch or a tag push (a release), never on a pull request, never from another repository than ArcadeData/arcadedb-python. + +The comment is a log, not a gate: when the token cannot write issues, or gh fails for any other reason, the script prints a `::warning::` +line and still exits 0. It never fails the run. The text carries no em dash. + + python native/wheel/report_flaky_crash.py --artifacts [--summary ] [--dry-run] +""" + +from __future__ import annotations + +import argparse +import glob +import json +import os +import subprocess # nosec B404 - runs the gh CLI with a fixed argument list +import sys + +REPO = "ArcadeData/arcadedb-python" +ISSUE = 295 +META = "parity-wheel.meta.json" + + +def load_events(root: str) -> list[dict]: + """Every crash event of every leg under `root` (one artifact directory per leg), each with its leg's identity.""" + events = [] + for path in sorted(glob.glob(os.path.join(root, "**", META), recursive=True)): + try: + with open(path, encoding="utf-8") as fh: + meta = json.load(fh) + except (OSError, ValueError) as err: + print(f"::warning::{path} could not be read: {err}", flush=True) + continue + artifact = os.path.basename(os.path.dirname(path)) + for ev in meta.get("crash_events", []): + events.append( + { + **ev, + "leg": meta.get("key") or artifact, + "python": meta.get("python", ""), + "wheel": meta.get("wheel_version", ""), + "artifact": artifact, + "retry_on": bool(meta.get("retry_crash")), + } + ) + return events + + +def _table(events: list[dict]) -> list[str]: + lines = [ + "| leg | Python | wheel | file | first exit code | retry |", + "|---|---|---|---|---|---|", + ] + for e in events: + retry = e["retry"] if not e["retry_rc"] else f"{e['retry']} ({e['retry_rc']})" + lines.append( + f"| {e['leg']} | {e['python']} | {e['wheel']} | `{e['file']}` | {e['kind']} {e['first_rc']} | {retry} |" + ) + return lines + + +def _where(events: list[dict]) -> list[str]: + lines = [] + for e in events: + files = [e[k] for k in ("trace1", "trace2") if e.get(k)] + names = ", ".join(f"`{n}`" for n in [*files, f"{e['file']}.attempt1.log"]) + lines.append( + f"- `{e['file']}` ({e['leg']}, Python {e['python']}): artifact `{e['artifact']}`, folder `parity-wheel-xml/`: {names}" + ) + return lines + + +def summary_md(events: list[dict]) -> str: + """The job summary block; empty when no file crashed.""" + if not events: + return "" + flaky = [e for e in events if e["kind"] == "FLAKY-CRASH"] + hard = [e for e in events if e["kind"] != "FLAKY-CRASH"] + out = ["### Parity crashes", ""] + if flaky: + out += [ + f"**FLAKY-CRASH: {len(flaky)} file(s) died without a result and passed when run once more in a fresh process.** " + "A native crash was hidden by the retry on these legs. Each one is logged on issue " + f"#{ISSUE} (when the run is a release or a dispatch).", + "", + ] + if hard: + out += [ + f"**CRASH: {len(hard)} file(s) died without a result and the leg failed.**", + "", + ] + out += [*_table(events), "", "Crash traces and logs of the first attempt:", ""] + out += _where(events) + return "\n".join(out) + "\n" + + +def comment_md(events: list[dict], run_url: str, attempt: str, event: str, ref: str) -> str: + """The comment for issue 295: one per run, every event in it.""" + kinds = sorted({e["kind"] for e in events}) + out = [ + f"{' and '.join(kinds)} in a native parity run.", + "", + f"Run: {run_url} (attempt {attempt or '1'}, event {event}, ref {ref})", + "", + *_table(events), + "", + "A test file died without a result and the harness ran it once more in a fresh process (`parity_wheel.py --retry-crash`, Windows legs only). " + "A retry that says `passed` made its leg green: without the retry that leg would have failed. " + "`CRASH` means the file died again (or no retry was left) and the leg failed.", + "", + "The faulthandler trace and the log of the first attempt are in the uploaded parity artifact of each leg:", + "", + *_where(events), + ] + return "\n".join(out) + "\n" + + +def may_post(event: str, ref_type: str, repo: str) -> str: + """'' when the comment may be posted, else the reason it may not.""" + if repo != REPO: + return f"repository {repo or '(unknown)'} is not {REPO}" + if event == "workflow_dispatch": + return "" + if event == "push" and ref_type == "tag": + return "" + return f"event {event or '(unknown)'} on a {ref_type or 'ref'} is neither a release nor a dispatch" + + +def post_comment(body: str, run=subprocess.run) -> tuple[bool, str]: + """POST the comment with the gh CLI and GH_TOKEN; (True, '') or (False, why). Never raises.""" + cmd = ["gh", "api", "--method", "POST", f"repos/{REPO}/issues/{ISSUE}/comments", "--input", "-"] + try: + r = run(cmd, input=json.dumps({"body": body}), capture_output=True, text=True, timeout=120) # nosec B603 + except (OSError, subprocess.SubprocessError) as err: + return False, f"{type(err).__name__}: {err}" + if r.returncode != 0: + return False, (r.stderr or r.stdout or f"gh exited {r.returncode}").strip()[:300] + return True, "" + + +def report(events, env, summary_path="", dry_run=False, run=subprocess.run) -> int: + """Write the summary, post the comment when allowed. Returns 0: a log is never a gate.""" + if not events: + print("no crash event in this run", flush=True) + return 0 + md = summary_md(events) + print(md, flush=True) + if summary_path: + with open(summary_path, "a", encoding="utf-8") as fh: + fh.write(md) + server = env.get("GITHUB_SERVER_URL", "https://github.com") + repo = env.get("GITHUB_REPOSITORY", "") + run_url = f"{server}/{repo}/actions/runs/{env.get('GITHUB_RUN_ID', '')}" + body = comment_md( + events, + run_url, + env.get("GITHUB_RUN_ATTEMPT", ""), + env.get("GITHUB_EVENT_NAME", ""), + env.get("GITHUB_REF_NAME", ""), + ) + why = may_post(env.get("GITHUB_EVENT_NAME", ""), env.get("GITHUB_REF_TYPE", ""), repo) + if why: + print(f"issue #{ISSUE}: no comment posted ({why})", flush=True) + return 0 + if dry_run: + print(f"issue #{ISSUE}: dry run, the comment would be:\n{body}", flush=True) + return 0 + if not (env.get("GH_TOKEN") or env.get("GITHUB_TOKEN")): + print(f"::warning::the FLAKY-CRASH comment on #{ISSUE} was skipped: no GH_TOKEN", flush=True) + return 0 + ok, err = post_comment(body, run=run) + if ok: + print(f"issue #{ISSUE}: comment posted", flush=True) + else: + print(f"::warning::the FLAKY-CRASH comment on #{ISSUE} was skipped, the token cannot write issues or gh failed: {err}", flush=True) + return 0 + + +def main(argv=None) -> int: + ap = argparse.ArgumentParser() + ap.add_argument("--artifacts", required=True) + ap.add_argument("--summary", default=os.environ.get("GITHUB_STEP_SUMMARY", "")) + ap.add_argument("--dry-run", action="store_true") + a = ap.parse_args(argv) + try: + return report(load_events(a.artifacts), os.environ, a.summary, a.dry_run) + except Exception as err: # noqa: BLE001 - a log is never a gate + print(f"::warning::the crash report failed: {type(err).__name__}: {err}", flush=True) + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/tests/README.md b/tests/README.md index 6d2bc8a6..b8d98f4c 100644 --- a/tests/README.md +++ b/tests/README.md @@ -39,6 +39,8 @@ The same suite runs against the native `arcadedb` wheel (`native/wheel/parity_wh - `NotOnNativeYet`: the native code raised it because the feature needs a Java object API the library does not expose. The report says `NOTYET`, and a final release allows none. A call that is JVM-only by design (`get_java_database`, `get_java_document`, a Java object a `Schema` method returns) raises it with a message that names the `arcadedb-jvm` build; a shared test that reaches one is `NA-JVM`, or reads the value through a statement or the Python API instead. - A row of `NATIVE_NA_JVM_TESTS` in `tests/conftest.py`, keyed `::[::]`: the test reaches into the JVM on purpose (`jpype.JClass`, a Java array, `db._java_db`, counted bridge crossings). The report says `NA-JVM`. Add a row only for such a test, and give the reason. When the behaviour is shared, write a native test beside it (`test_native_*.py`, collected on the native build only). +A test file whose process dies without a result is a `CRASH` row with its exit code. On Windows the parity harness runs such a file once more (`--retry-crash`, set by `test-native.yml` on Windows legs only) and marks a retry that produced a result `FLAKY-CRASH`; the job summary and issue 295 record it. A failing test is never retried, and a crash on Linux or macOS is a failure. The decision is `native/wheel/parity_crash.py`, tested by `tests/test_parity_crash_retry.py`. + A test that fails on the native build for any other reason is a failure. Fix the native code, or mark a known engine bug with a strict `xfail`. Do not add a row to hide it. ## Need Help? diff --git a/tests/conftest.py b/tests/conftest.py index 819fb71e..7bc93d30 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -35,6 +35,8 @@ # the release preflight is a maintainer tool for a POSIX shell: its check (f) runs the workflows' bash step code, # and on Windows it reports a SKIP, which its own tests cover on Linux and macOS (ArcadeData/arcadedb-python#208) "test_release_preflight.py": "runs the workflows' bash step code", + # runs the awk program of the parity gate step of test-native.yml, which is code for the bash of a runner + "test_parity_gate_awk.py": "runs the workflows' awk step code", } if sys.platform == "win32": collect_ignore.extend(WINDOWS_NOT_APPLICABLE) @@ -318,15 +320,34 @@ def pytest_configure(config): # pytest_configure, and a conftest hook otherwise runs before that. # This hook was dead from 2026-07-25 to 2026-09-29: a second # pytest_configure further down replaced it (test_jvm_args pins it now). + # + # That reason is the JVM's. The native wheel has no HotSpot in the process, so a + # fault there is a real one and the trace is what a crash leaves: with a stock run + # (faulthandler off, nothing in the log) the Windows parity legs lost a process in + # test_vector.py, test_free_threading.py and test_find_nearest_query_forms.py and + # nothing told which call it was (ArcadeData/arcadedb-python#295). native/wheel/parity_wheel.py names a + # file in ARCADEDB_TEST_FAULTHANDLER_FILE (it sets it on Windows only), and a native + # run on Windows then writes faulthandler's trace there, not to stderr. The JVM + # build ignores the variable and stays off. import sys if sys.platform == "win32": import faulthandler - if faulthandler.is_enabled(): + trace = os.environ.get(FAULT_FILE_ENV) + if trace and BACKEND == "native": + global _fault_file + _fault_file = open(trace, "w") # kept open: faulthandler writes to its descriptor at the crash + faulthandler.enable(file=_fault_file, all_threads=True) + elif faulthandler.is_enabled(): faulthandler.disable() +# The file the native run on Windows writes a faulthandler trace to (see pytest_configure above); set by native/wheel/parity_wheel.py. +FAULT_FILE_ENV = "ARCADEDB_TEST_FAULTHANDLER_FILE" +_fault_file = None + + # A test still running this long gets every Java thread's stack on stderr, # shortly before faulthandler_timeout (600 s in pyproject.toml) dumps the # Python threads. The Python dump alone shows a test waiting inside a Java diff --git a/tests/test_parity_crash_retry.py b/tests/test_parity_crash_retry.py new file mode 100644 index 00000000..9250e382 --- /dev/null +++ b/tests/test_parity_crash_retry.py @@ -0,0 +1,662 @@ +"""The crash handling of the native parity harness (native/wheel/parity_wheel.py): record, retry once, report, stay strict. + +The Windows legs of the native release run lost one test process each (ArcadeData/arcadedb-python#295), with no exit code and no trace. +The harness now records the exit code of a process that died without a result (`CRASH(rc=0xC0000005,3221225477)`), runs such a file once +more when --retry-crash is given (the workflow gives it on Windows legs only) and marks a retry that produced a result `FLAKY-CRASH`. A +failing test, a timeout, a second crash and every crash without the flag stay failures. These tests run the decision with a fake runner, +with real child processes that exit with a high code or die from a signal, and the harness itself against a small fixture repository +(real pytest children, the wheel check stubbed), so the code that ships is the code that is tested. Nothing here needs Windows: the exit +code of a crash differs by platform and the tests read it from the platform they run on. +""" + +from __future__ import annotations + +import importlib.util +import json +import re +import signal +import subprocess # nosec B404 - test-controlled child processes +import sys +import textwrap +from pathlib import Path + +import pytest + +ROOT = Path(__file__).resolve().parents[1] +WHEEL_DIR = ROOT / "native" / "wheel" +WORKFLOWS = ROOT / ".github" / "workflows" + +# the harness imports parity_crash as a sibling of the script +if str(WHEEL_DIR) not in sys.path: + sys.path.insert(0, str(WHEEL_DIR)) + + +def _load(name: str): + spec = importlib.util.spec_from_file_location(name, WHEEL_DIR / f"{name}.py") + module = importlib.util.module_from_spec(spec) + sys.modules[name] = module # a dataclass looks its module up here + spec.loader.exec_module(module) + return module + + +pc = _load("parity_crash") +parity_wheel = _load("parity_wheel") +report = _load("report_flaky_crash") + +WIN = sys.platform == "win32" +# An exit code a test process can die with: the NTSTATUS of an access violation on Windows, 139 (128 + SIGSEGV) elsewhere (POSIX keeps 8 bits). +HIGH = 0xC0000005 if WIN else 139 +HIGH_TEXT = "0xC0000005,3221225477" if WIN else "0x0000008B,139" +# os._exit takes a C int: the same NTSTATUS as a signed 32-bit number on Windows (the exit code stays 0xC0000005 to the parent) +HIGH_ARG = HIGH - (1 << 32) if WIN else HIGH +OTHER = 0xC0000374 if WIN else 134 +OTHER_TEXT = "0xC0000374,3221226356" if WIN else "0x00000086,134" +EM_DASH = "\u2014" # written as an escape so that this file holds none itself + + +def attempts(*items): + """A fake runner: attempt n returns items[n - 1]; `calls` records which attempts ran.""" + calls = [] + + def run(n): + calls.append(n) + return items[n - 1] + + return run, calls + + +RESULT = pc.Attempt(rc=0, result=True) +FAILED = pc.Attempt(rc=1, result=True) # pytest wrote its XML, with failures in it +DIED = pc.Attempt(rc=HIGH, result=False) +DIED_AGAIN = pc.Attempt(rc=OTHER, result=False) +ON = lambda budget=3: pc.RetryPolicy(enabled=True, budget=budget) # noqa: E731 + + +# --------------------------------------------------------------------------------------------------------------------------------- +# the decision, with a fake runner +# --------------------------------------------------------------------------------------------------------------------------------- + + +def test_a_file_with_a_result_runs_once_even_when_the_retry_is_on(): + run, calls = attempts(RESULT, RESULT) + out = pc.run_file(run, ON()) + assert calls == [1] + assert out.status == "" + assert (out.retried, out.flaky, out.hard_crash) == (False, False, False) + + +def test_a_file_with_failing_tests_is_never_retried(): + # exit code 1 with an XML is a test failure: the count of failures comes from the XML and stays strict + run, calls = attempts(FAILED, RESULT) + out = pc.run_file(run, ON()) + assert calls == [1] + assert out.status == "" + assert out.final is FAILED + + +def test_a_crash_without_the_retry_flag_is_a_crash_with_its_exit_code(): + # the default policy is what Linux and macOS use: strict, one run, the exit code recorded + run, calls = attempts(DIED, RESULT) + out = pc.run_file(run, pc.RetryPolicy()) + assert calls == [1] + assert out.status == f"CRASH(rc={HIGH_TEXT})" + assert out.hard_crash and not out.flaky and not out.retried + + +def test_a_crash_that_produces_a_result_when_run_again_is_flaky_not_clean(): + run, calls = attempts(DIED, RESULT) + out = pc.run_file(run, ON()) + assert calls == [1, 2] + assert out.status == f"FLAKY-CRASH(rc={HIGH_TEXT})" + assert out.flaky and out.retried and not out.hard_crash + assert out.final is RESULT + + +def test_a_second_crash_is_a_crash_with_both_exit_codes(): + run, calls = attempts(DIED, DIED_AGAIN) + out = pc.run_file(run, ON()) + assert calls == [1, 2] + assert out.status == f"CRASH(rc={HIGH_TEXT};retry rc={OTHER_TEXT})" + assert out.hard_crash and not out.flaky + + +def test_a_timeout_and_an_unreadable_xml_are_not_crashes_and_are_not_retried(): + run, calls = attempts(pc.Attempt(rc=None, result=False, timed_out=True), RESULT) + out = pc.run_file(run, ON()) + assert (calls, out.status, out.timed_out) == ([1], "TIMEOUT", True) + run, calls = attempts(pc.Attempt(rc=1, result=False, bad_xml=True), RESULT) + out = pc.run_file(run, ON()) + assert (calls, out.status) == ([1], "BADXML") + + +def test_a_retry_that_times_out_or_writes_a_bad_xml_fails_the_leg(): + run, _ = attempts(DIED, pc.Attempt(rc=None, result=False, timed_out=True)) + out = pc.run_file(run, ON()) + assert out.status == f"CRASH(rc={HIGH_TEXT};retry TIMEOUT)" + assert out.timed_out + run, _ = attempts(DIED, pc.Attempt(rc=1, result=False, bad_xml=True)) + assert pc.run_file(run, ON()).status == f"CRASH(rc={HIGH_TEXT};retry BADXML)" + + +def test_a_retry_that_runs_and_has_failures_is_flaky_and_the_counts_decide(): + # the status says the file crashed once; the failures of the second run are counted by the harness and fail the leg + run, _ = attempts(DIED, FAILED) + out = pc.run_file(run, ON()) + assert out.flaky + assert pc.event_of("test_x.py", out, final_failed=2)["retry"] == "ran, 2 failed" + + +def test_the_retry_budget_of_a_leg_is_spent_by_the_files_not_the_crashes(): + policy = ON(budget=2) + statuses = [] + for _ in range(3): + run, calls = attempts(DIED, RESULT) + out = pc.run_file(run, policy) + statuses.append((out.status.split("(")[0], calls)) + # two files retried and flaky, the third crashes where the budget is spent: a wheel that crashes everywhere cannot pass on retries + assert statuses == [("FLAKY-CRASH", [1, 2]), ("FLAKY-CRASH", [1, 2]), ("CRASH", [1])] + assert policy.used == 2 + run, _ = attempts(DIED) + assert pc.run_file(run, policy).status == f"CRASH(rc={HIGH_TEXT};retry budget spent)" + + +def test_exit_codes_are_written_as_ntstatus_and_decimal_or_as_a_signal(): + assert pc.format_rc(3221225477, windows=True) == "0xC0000005,3221225477" + # the same NTSTATUS read as a signed number folds back to 32 bits on Windows + assert pc.format_rc(-1073741819, windows=True) == "0xC0000005,3221225477" + assert pc.format_rc(139, windows=False) == "0x0000008B,139" + assert pc.format_rc(0, windows=False) == "0x00000000,0" + assert pc.format_rc(-signal.SIGABRT, windows=False) == "-6,SIGABRT" + assert pc.format_rc(-250, windows=False) == "-250,signal 250" + assert pc.format_rc(None) == "none" + + +def test_a_crash_event_names_the_kind_and_what_the_retry_did(): + run, _ = attempts(DIED, RESULT) + ev = pc.event_of("test_free_threading.py", pc.run_file(run, ON()), final_failed=0) + assert ev == { + "file": "test_free_threading.py", + "kind": "FLAKY-CRASH", + "first_rc": HIGH_TEXT, + "retry_rc": "0x00000000,0", + "retry": "passed", + "status": f"FLAKY-CRASH(rc={HIGH_TEXT})", + } + run, _ = attempts(DIED, DIED_AGAIN) + ev = pc.event_of("a.py", pc.run_file(run, ON()), final_failed=0) + assert (ev["kind"], ev["retry"], ev["retry_rc"]) == ("CRASH", "crashed again", OTHER_TEXT) + run, _ = attempts(DIED) + ev = pc.event_of("a.py", pc.run_file(run, pc.RetryPolicy()), final_failed=0) + assert (ev["kind"], ev["retry"], ev["retry_rc"]) == ("CRASH", "not retried", "") + run, _ = attempts(RESULT) + assert pc.event_of("a.py", pc.run_file(run, ON()), final_failed=0) is None + + +def test_a_result_followed_by_an_odd_exit_code_is_noticed_but_not_a_crash(): + # the junit XML is written at the end of the session; a process that dies in shutdown after it leaves a result + assert pc.dirty_exit(pc.Attempt(rc=HIGH, result=True)) is True + assert pc.dirty_exit(pc.Attempt(rc=1, result=True)) is False + assert pc.dirty_exit(pc.Attempt(rc=0, result=True)) is False + assert pc.dirty_exit(DIED) is False + assert pc.is_crash(pc.Attempt(rc=HIGH, result=True)) is False + + +# --------------------------------------------------------------------------------------------------------------------------------- +# real child processes: a high exit code, a signal, and a fresh process for the retry +# --------------------------------------------------------------------------------------------------------------------------------- + +EXIT_ONCE = """ +import os, pathlib, sys +d, n, high = pathlib.Path(sys.argv[1]), sys.argv[2], int(sys.argv[3]) +with open(d / "pids", "a") as fh: + fh.write(f"{os.getpid()}\\n") +if not (d / "died").exists(): + (d / "died").write_text("1") + os._exit(high) +(d / f"result{n}").write_text("ok") +""" + +EXIT_ALWAYS = """ +import os, sys +os._exit(int(sys.argv[3])) +""" + +SEGV = """ +import os, sys +if sys.platform == "win32": + # a real access violation (0xC0000005); ctypes would turn one into an OSError, and raise(SIGSEGV) exits with 3 + import ctypes, faulthandler + ctypes.windll.kernel32.SetErrorMode(0x8003) # no crash dialog and no Windows Error Reporting for this child + faulthandler._read_null() +else: + import resource, signal + resource.setrlimit(resource.RLIMIT_CORE, (0, 0)) # no core file for a test child + os.kill(os.getpid(), signal.SIGSEGV) +""" + + +def real_runner(code, d): + """attempt(n) runs `code` in a new python process; the process is a crash unless it wrote result in directory d.""" + + def run(n): + r = subprocess.run( # nosec B603 - fixed argument list, no shell + [sys.executable, "-c", code, str(d), str(n), str(HIGH_ARG)], + capture_output=True, + text=True, + timeout=120, + ) + return pc.Attempt(rc=r.returncode, result=(d / f"result{n}").exists()) + + return run + + +def test_a_real_process_that_exits_with_a_high_code_is_recorded_with_that_code(tmp_path): + out = pc.run_file(real_runner(EXIT_ALWAYS, tmp_path), pc.RetryPolicy()) + assert out.final.rc == HIGH + assert out.status == f"CRASH(rc={HIGH_TEXT})" + + +def test_a_real_process_killed_by_a_fault_is_recorded_as_one(tmp_path): + out = pc.run_file(real_runner(SEGV, tmp_path), pc.RetryPolicy()) + if WIN: + assert out.final.rc == 0xC0000005 + assert out.status == "CRASH(rc=0xC0000005,3221225477)" + else: + assert out.final.rc == -signal.SIGSEGV + assert out.status == "CRASH(rc=-11,SIGSEGV)" + + +def test_the_retry_of_a_real_crash_is_a_new_process_and_a_pass_is_flaky(tmp_path): + out = pc.run_file(real_runner(EXIT_ONCE, tmp_path), ON()) + pids = (tmp_path / "pids").read_text().split() + assert len(pids) == 2 and pids[0] != pids[1] + assert (tmp_path / "result1").exists() is False and (tmp_path / "result2").exists() + assert out.status == f"FLAKY-CRASH(rc={HIGH_TEXT})" + assert out.final.rc == 0 + + +def test_a_real_file_that_crashes_every_time_fails_after_one_retry(tmp_path): + out = pc.run_file(real_runner(EXIT_ALWAYS, tmp_path), ON()) + assert out.status == f"CRASH(rc={HIGH_TEXT};retry rc={HIGH_TEXT})" + assert len(out.attempts) == 2 + + +# --------------------------------------------------------------------------------------------------------------------------------- +# the harness against a small repository: real pytest children, the wheel check stubbed +# --------------------------------------------------------------------------------------------------------------------------------- + +FIXTURE_FILES = { + "test_ok.py": "def test_a():\n pass\n\n\ndef test_b():\n pass\n", + "test_fails.py": "def test_a():\n assert False, 'a real failure'\n", + "test_dies_once.py": textwrap.dedent( + f""" + import os, pathlib + + + def test_a(): + marker = pathlib.Path(os.environ["DIES_ONCE_MARKER"]) + if not marker.exists(): + marker.write_text("1") + os._exit({HIGH_ARG}) + """ + ), + "test_dies_always.py": f"import os\n\n\ndef test_a():\n os._exit({HIGH_ARG})\n", +} + +IDENT = ( + "/x/site-packages/arcadedb/__init__.py 26.11.1.post2 native 26.11.1 abc1234\n" + 'META {"python": "3.14.0", "wheel_version": "26.11.1.post2", "engine_version": "26.11.1", "source_commit": "abc1234"}\n' +) + + +@pytest.fixture +def harness(tmp_path, monkeypatch): + """run(*flags) -> (rows by file name, the TOTAL row, meta.json, the xml directory); the fixture repo is rebuilt for each call.""" + real_run = subprocess.run + + def run(cmd, *args, **kwargs): + if "-I" in cmd and any("import json, sys, arcadedb" in str(c) for c in cmd): + return subprocess.CompletedProcess(cmd, 0, stdout=IDENT, stderr="") + return real_run(cmd, *args, **kwargs) + + monkeypatch.setattr(parity_wheel.subprocess, "run", run) + # no taskset and no nice: the cores of this machine are not the harness's to choose + monkeypatch.setattr(parity_wheel.shutil, "which", lambda name: None) + monkeypatch.delenv("PYTHONPATH", raising=False) + counter = iter(range(100)) + + def go(*flags): + repo = tmp_path / f"repo{next(counter)}" + (repo / "tests").mkdir(parents=True) + (repo / "tests" / "conftest.py").write_text("# fixture\n") + for name, text in FIXTURE_FILES.items(): + (repo / "tests" / name).write_text(text) + monkeypatch.setenv("DIES_ONCE_MARKER", str(repo / "marker")) + monkeypatch.setenv("KEY", "windows-x64") + out = repo / "out" / "parity-wheel.tsv" + parity_wheel.main(["--repo", str(repo), "--python", sys.executable, "--out", str(out), "--timeout", "120", *flags]) + lines = [ln.split("\t") for ln in out.read_text().splitlines()] + header = lines[0] + rows = {ln[0]: dict(zip(header, ln)) for ln in lines[1:]} + meta = json.loads((repo / "out" / "parity-wheel.meta.json").read_text()) + return rows, meta, repo / "out" / "parity-wheel-xml" + + return go + + +def test_the_harness_is_strict_without_the_flag(harness): + rows, meta, xmldir = harness() + assert rows["test_ok.py"]["status"] == "" and rows["test_ok.py"]["passed"] == "2" + assert rows["test_fails.py"]["failed"] == "1" and rows["test_fails.py"]["status"] == "" + # both crashing files are CRASH rows with the exit code, one attempt each, no retry + for name in ("test_dies_once.py", "test_dies_always.py"): + assert rows[name]["status"] == f"CRASH(rc={HIGH_TEXT})", name + assert rows[name]["attempts"] == "1", name + assert rows[name]["exit"] == HIGH_TEXT, name + assert rows["TOTAL"]["status"] == "timeouts=0 flaky_crash=0 crash=2" + assert meta["retry_crash"] is False + assert [e["kind"] for e in meta["crash_events"]] == ["CRASH", "CRASH"] + assert not (xmldir / "test_dies_once.py.attempt1.log").exists() + + +def test_the_harness_retries_only_the_crashes_with_the_flag(harness): + rows, meta, xmldir = harness("--retry-crash") + # a clean file and a file with a failing test run once, whatever the flag says + assert (rows["test_ok.py"]["attempts"], rows["test_ok.py"]["status"]) == ("1", "") + assert (rows["test_fails.py"]["attempts"], rows["test_fails.py"]["status"]) == ("1", "") + assert rows["test_fails.py"]["failed"] == "1" + # the file that died once passes on the retry: FLAKY-CRASH with the first exit code, counted from the second run + once = rows["test_dies_once.py"] + assert once["status"] == f"FLAKY-CRASH(rc={HIGH_TEXT})" + assert (once["attempts"], once["passed"], once["failed"]) == ("2", "1", "0") + assert once["exit"] == "0x00000000,0" + # the file that always dies is retried once and then fails the leg + always = rows["test_dies_always.py"] + assert always["status"] == f"CRASH(rc={HIGH_TEXT};retry rc={HIGH_TEXT})" + assert always["attempts"] == "2" + assert rows["TOTAL"]["status"] == "timeouts=0 flaky_crash=1 crash=1" + # the log of the first attempt is kept beside the log of the retry + assert (xmldir / "test_dies_once.py.attempt1.log").exists() + assert (xmldir / "test_dies_once.py.log").exists() + assert (xmldir / "test_dies_once.py.xml").exists() + # the metadata names the wheel and the Python once and every crash event + assert meta["wheel_version"] == "26.11.1.post2" and meta["python"] == "3.14.0" + assert meta["retry_crash"] is True + events = {e["file"]: e for e in meta["crash_events"]} + assert set(events) == {"test_dies_once.py", "test_dies_always.py"} + assert events["test_dies_once.py"]["kind"] == "FLAKY-CRASH" + assert events["test_dies_once.py"]["retry"] == "passed" + assert events["test_dies_always.py"]["retry"] == "crashed again" + + +def test_the_harness_retry_is_off_unless_asked_for(): + assert parity_wheel.parse_args(["--repo", "r", "--out", "o"]).retry_crash is False + assert parity_wheel.parse_args(["--repo", "r", "--out", "o", "--retry-crash"]).retry_crash is True + assert parity_wheel.parse_args(["--repo", "r", "--out", "o"]).max_crash_retries == 3 + + +# --------------------------------------------------------------------------------------------------------------------------------- +# the faulthandler trace of a native run on Windows (tests/conftest.py) +# --------------------------------------------------------------------------------------------------------------------------------- + +FAULT_CHILD = """ +import faulthandler, os, sys + +if os.name == "posix": + import resource + + resource.setrlimit(resource.RLIMIT_CORE, (0, 0)) # no core file for a test child +else: + import ctypes + + ctypes.windll.kernel32.SetErrorMode(0x8003) # no crash dialog and no Windows Error Reporting for this child + +sys.path.insert(0, sys.argv[1]) +from tests import conftest + +trace = sys.argv[2] +os.environ[conftest.FAULT_FILE_ENV] = trace + +# the JVM build ignores the variable: faulthandler stays off on Windows, as before +conftest.BACKEND = "jvm" +sys.platform = "win32" +faulthandler.enable() +conftest.pytest_configure(None) +assert not faulthandler.is_enabled(), "jvm on win32" +assert not os.path.exists(trace), "jvm on win32 opened the trace file" + +# another platform keeps what it has, whatever the variable says +conftest.BACKEND = "native" +sys.platform = "linux" +faulthandler.enable() +conftest.pytest_configure(None) +assert faulthandler.is_enabled(), "native on linux" +assert not os.path.exists(trace), "native on linux opened the trace file" + +# the native build on Windows without the variable: off, as before +sys.platform = "win32" +del os.environ[conftest.FAULT_FILE_ENV] +conftest.pytest_configure(None) +assert not faulthandler.is_enabled(), "native on win32 without the variable" + +# the native build on Windows with the variable: on, and a fault leaves its trace in the file +os.environ[conftest.FAULT_FILE_ENV] = trace +conftest.pytest_configure(None) +assert faulthandler.is_enabled(), "native on win32 with the variable" +print("configured", flush=True) +faulthandler._read_null() # a read of address 0: an access violation on Windows, SIGSEGV elsewhere +""" + + +def test_a_native_run_on_windows_keeps_a_faulthandler_trace_and_nothing_else_does(tmp_path): + trace = tmp_path / "fault.txt" + r = subprocess.run( # nosec B603 - fixed argument list, no shell + [sys.executable, "-c", FAULT_CHILD, str(ROOT), str(trace)], + capture_output=True, + text=True, + timeout=120, + ) + # every assert of the child passed (a failed one would end the child before it printed) and it then died of the fault + assert "AssertionError" not in r.stderr, r.stderr + assert "configured" in r.stdout, r.stderr + assert r.returncode == (0xC0000005 if WIN else -signal.SIGSEGV), (r.returncode, r.stderr) + text = trace.read_text() + assert "Segmentation fault" in text or "access violation" in text + assert "most recent call first" in text + + +# --------------------------------------------------------------------------------------------------------------------------------- +# the report: the job summary and one comment on issue 295 +# --------------------------------------------------------------------------------------------------------------------------------- + + +def write_meta(root, artifact, key, python, wheel, events, retry=True): + d = root / artifact + d.mkdir(parents=True) + (d / "parity-wheel.meta.json").write_text( + json.dumps( + { + "key": key, + "platform": "win32", + "python": python, + "wheel_version": wheel, + "retry_crash": retry, + "crash_events": events, + } + ) + ) + + +FLAKY_EVENT = { + "file": "test_free_threading.py", + "kind": "FLAKY-CRASH", + "first_rc": "0xC0000005,3221225477", + "retry_rc": "", + "retry": "passed", + "status": "FLAKY-CRASH(rc=0xC0000005,3221225477)", + "trace1": "test_free_threading.py.attempt1.fault.txt", +} +CRASH_EVENT = { + "file": "test_vector.py", + "kind": "CRASH", + "first_rc": "0xC0000374,3221226356", + "retry_rc": "0xC0000374,3221226356", + "retry": "crashed again", + "status": "CRASH(rc=0xC0000374,3221226356;retry rc=0xC0000374,3221226356)", +} +RELEASE_ENV = { + "GITHUB_REPOSITORY": "ArcadeData/arcadedb-python", + "GITHUB_RUN_ID": "42744219731", + "GITHUB_RUN_ATTEMPT": "2", + "GITHUB_EVENT_NAME": "push", + "GITHUB_REF_TYPE": "tag", + "GITHUB_REF_NAME": "26.11.1.post2", + "GITHUB_SERVER_URL": "https://github.com", + "GH_TOKEN": "token", +} + + +@pytest.fixture +def artifacts(tmp_path): + root = tmp_path / "artifacts" + write_meta(root, "native-parity-windows-x64-py3.11", "windows-x64", "3.11.9", "26.11.1.post2", [FLAKY_EVENT]) + write_meta(root, "native-parity-windows-x64-py3.12", "windows-x64", "3.12.7", "26.11.1.post2", []) + write_meta(root, "native-parity-windows-x64-py3.14", "windows-x64", "3.14.0", "26.11.1.post2", [CRASH_EVENT]) + return root + + +class FakeGh: + def __init__(self, returncode=0, stderr=""): + self.calls, self.returncode, self.stderr = [], returncode, stderr + + def __call__(self, cmd, **kwargs): + self.calls.append((cmd, json.loads(kwargs["input"])["body"])) + return subprocess.CompletedProcess(cmd, self.returncode, stdout="", stderr=self.stderr) + + +def test_the_events_of_every_leg_are_collected_with_the_identity_of_their_leg(artifacts): + events = report.load_events(str(artifacts)) + assert [(e["file"], e["python"], e["wheel"]) for e in events] == [ + ("test_free_threading.py", "3.11.9", "26.11.1.post2"), + ("test_vector.py", "3.14.0", "26.11.1.post2"), + ] + assert report.load_events(str(artifacts / "none")) == [] + + +def test_a_release_posts_one_comment_listing_every_event_and_the_summary_has_the_table(artifacts, tmp_path, capsys): + gh, summary = FakeGh(), tmp_path / "summary.md" + code = report.report(report.load_events(str(artifacts)), RELEASE_ENV, str(summary), run=gh) + assert code == 0 + assert len(gh.calls) == 1 + cmd, body = gh.calls[0] + assert cmd[:5] == ["gh", "api", "--method", "POST", "repos/ArcadeData/arcadedb-python/issues/295/comments"] + for text in ( + "https://github.com/ArcadeData/arcadedb-python/actions/runs/42744219731", + "attempt 2", + "26.11.1.post2", + "test_free_threading.py", + "test_vector.py", + "FLAKY-CRASH 0xC0000005,3221225477", + "3.11.9", + "3.14.0", + "native-parity-windows-x64-py3.11", + ): + assert text in body, text + assert body.startswith("CRASH and FLAKY-CRASH in a native parity run.") + assert EM_DASH not in body + md = summary.read_text() + assert md.startswith("### Parity crashes") + assert "**FLAKY-CRASH: 1 file(s)" in md and "**CRASH: 1 file(s)" in md + assert "| windows-x64 | 3.11.9 | 26.11.1.post2 | `test_free_threading.py` | FLAKY-CRASH 0xC0000005,3221225477 | passed |" in md + assert EM_DASH not in md + + +def test_a_dispatch_posts_and_a_pull_request_a_branch_push_and_a_fork_do_not(artifacts): + events = report.load_events(str(artifacts)) + posted = {} + for name, env in { + "dispatch": {**RELEASE_ENV, "GITHUB_EVENT_NAME": "workflow_dispatch", "GITHUB_REF_TYPE": "branch"}, + "pull_request": {**RELEASE_ENV, "GITHUB_EVENT_NAME": "pull_request", "GITHUB_REF_TYPE": "branch"}, + "branch push": {**RELEASE_ENV, "GITHUB_EVENT_NAME": "push", "GITHUB_REF_TYPE": "branch"}, + "schedule": {**RELEASE_ENV, "GITHUB_EVENT_NAME": "schedule", "GITHUB_REF_TYPE": "branch"}, + "fork": {**RELEASE_ENV, "GITHUB_REPOSITORY": "humemai/arcadedb-python"}, + }.items(): + gh = FakeGh() + assert report.report(events, env, run=gh) == 0 + posted[name] = len(gh.calls) + assert posted == {"dispatch": 1, "pull_request": 0, "branch push": 0, "schedule": 0, "fork": 0} + + +def test_a_token_that_cannot_write_issues_is_a_visible_warning_and_never_a_failure(artifacts, capsys): + gh = FakeGh(returncode=1, stderr="gh: Resource not accessible by integration (HTTP 403)") + assert report.report(report.load_events(str(artifacts)), RELEASE_ENV, run=gh) == 0 + out = capsys.readouterr().out + assert len(gh.calls) == 1 + assert "::warning::the FLAKY-CRASH comment on #295 was skipped" in out + assert "HTTP 403" in out + # no token at all: gh is not even called + gh = FakeGh() + assert report.report(report.load_events(str(artifacts)), {**RELEASE_ENV, "GH_TOKEN": ""}, run=gh) == 0 + assert gh.calls == [] + assert "::warning::the FLAKY-CRASH comment on #295 was skipped: no GH_TOKEN" in capsys.readouterr().out + + +def test_a_run_without_a_crash_writes_nothing_and_posts_nothing(tmp_path, capsys): + root = tmp_path / "a" + write_meta(root, "native-parity-windows-x64-py3.12", "windows-x64", "3.12.7", "26.11.1.post2", []) + gh, summary = FakeGh(), tmp_path / "summary.md" + assert report.report(report.load_events(str(root)), RELEASE_ENV, str(summary), run=gh) == 0 + assert gh.calls == [] and not summary.exists() + + +def test_a_broken_artifact_directory_does_not_fail_the_report(tmp_path, capsys): + (tmp_path / "a" / "x").mkdir(parents=True) + (tmp_path / "a" / "x" / "parity-wheel.meta.json").write_text("{not json") + assert report.main(["--artifacts", str(tmp_path / "a"), "--summary", ""]) == 0 + assert "could not be read" in capsys.readouterr().out + + +# --------------------------------------------------------------------------------------------------------------------------------- +# the workflows: the flag on Windows legs only, and issues: write on one job only +# --------------------------------------------------------------------------------------------------------------------------------- + + +def jobs_of(text): + """{job name: its text} for the jobs: block of a workflow file (a job starts at two spaces of indentation).""" + body = text.split("\njobs:\n", 1)[1] + parts = re.split(r"^ ([A-Za-z0-9_-]+):\n", body, flags=re.M) + return dict(zip(parts[1::2], parts[2::2])) + + +def test_the_workflow_passes_the_retry_flag_on_windows_legs_only(): + text = (WORKFLOWS / "test-native.yml").read_text(encoding="utf-8") + assert text.count("--retry-crash") == 2 # once in the comment of the header, once in the Windows branch + assert re.search(r'if \[ "\$RUNNER_OS" = Windows \]; then retry=\(--retry-crash\); fi', text) + jobs = jobs_of(text) + assert 'retry=(--retry-crash)' in jobs["test"] + assert '${retry[@]+"${retry[@]}"}' in jobs["test"] + # the other jobs that run the harness never pass it + for name, job in jobs.items(): + if name != "test": + assert "--retry-crash" not in job, name + # the examples, pyver and build jobs call no harness with it; the Windows-only condition is the only place it is set + assert jobs["pyver"].count("parity_wheel.py") == 1 + + +def test_issues_write_is_held_by_the_crash_report_job_and_the_release_call_only(): + jobs = jobs_of((WORKFLOWS / "test-native.yml").read_text(encoding="utf-8")) + for name, job in jobs.items(): + assert ("issues: write" in job) == (name == "crash-report"), name + report_job = jobs["crash-report"] + assert "continue-on-error: true" in report_job + assert "contents: read" in report_job + assert "native/wheel/report_flaky_crash.py" in report_job + release = jobs_of((WORKFLOWS / "release-native.yml").read_text(encoding="utf-8")) + for name, job in release.items(): + assert ("issues: write" in job) == (name == "test"), name + assert "uses: ./.github/workflows/test-native.yml" in release["test"] + + +def test_the_flaky_comment_text_has_no_em_dash_in_any_source_that_writes_it(): + for path in (WHEEL_DIR / "report_flaky_crash.py", WHEEL_DIR / "parity_crash.py", WHEEL_DIR / "parity_wheel.py"): + assert EM_DASH not in path.read_text(encoding="utf-8"), path.name diff --git a/tests/test_parity_gate_awk.py b/tests/test_parity_gate_awk.py new file mode 100644 index 00000000..135b494e --- /dev/null +++ b/tests/test_parity_gate_awk.py @@ -0,0 +1,110 @@ +"""The gate of the native parity legs lets a FLAKY-CRASH row pass, says so, and fails every other status. + +test-native.yml reads parity-wheel.tsv with an awk program in the step "Gate: no failure, error, crash, timeout or uncounted skip" (the +`test` job and the `pyver` job carry the same program). These tests cut that program out of the workflow file and run it with the awk +of the machine on small tables, so the code that ships is the code that is tested. A retried crash (FLAKY-CRASH) must not block a +release, and it must be impossible to miss; a crash that stayed a crash, a timeout, a failing test and a skip must still fail the leg. +The file is not collected on Windows (tests/conftest.py WINDOWS_NOT_APPLICABLE): the gate runs in the bash of a runner, as in +test_release_preflight.py. +""" + +from __future__ import annotations + +import os +import re +import subprocess # nosec B404 - test-controlled child processes +from pathlib import Path + +import pytest + +WORKFLOW = Path(__file__).resolve().parents[1] / ".github" / "workflows" / "test-native.yml" +HEADER = "file\tpassed\tfailed\terror\tnotyet\tna_jvm\tskipped\tstatus\tfailing_tests\tattempts\texit\n" + + +def gate_programs(): + """The awk program of every gate step in the workflow, by job.""" + text = WORKFLOW.read_text(encoding="utf-8") + found = re.findall(r"awk -F'\\t' '\n(.*?)' \"\$tsv\"", text, flags=re.S) + assert len(found) == 2, "the test job and the pyver job each carry one gate" + return found + + +def row(name, passed=3, failed=0, error=0, status="", attempts=1, exit_="0x00000000,0"): + return f"{name}\t{passed}\t{failed}\t{error}\t0\t0\t0\t{status}\t\t{attempts}\t{exit_}\n" + + +def total(passed=10, failed=0, error=0, skipped=0, status="timeouts=0 flaky_crash=0 crash=0"): + return f"TOTAL\t{passed}\t{failed}\t{error}\t0\t0\t{skipped}\t{status}\t\t\t\n" + + +def run_gate(program, tmp_path, *rows): + tsv = tmp_path / "parity-wheel.tsv" + tsv.write_text(HEADER + "".join(rows)) + return subprocess.run( # nosec B603 B607 - fixed argument list, no shell + ["awk", "-F\\t", program, str(tsv)], # -F'\t' of the workflow: awk reads the backslash-t itself + capture_output=True, + text=True, + env={**os.environ, "KEY": "windows-x64", "PY": "3.14"}, + timeout=60, + ) + + +@pytest.fixture(params=[0, 1], ids=["test-job", "pyver-job"]) +def program(request): + return gate_programs()[request.param] + + +def test_a_clean_table_passes_and_prints_no_flaky_line(program, tmp_path): + r = run_gate(program, tmp_path, row("test_a.py"), total()) + assert r.returncode == 0, r.stderr + assert r.stdout.startswith("parity windows-x64 py3.14: passed=10 failed=0 error=0") + assert "FLAKY-CRASH" not in r.stdout + + +def test_a_file_that_crashed_and_passed_on_its_retry_passes_the_gate_loudly(program, tmp_path): + status = "FLAKY-CRASH(rc=0xC0000005,3221225477)" + r = run_gate(program, tmp_path, row("test_a.py"), row("test_free_threading.py", status=status, attempts=2), total()) + assert r.returncode == 0, r.stderr + assert f"FLAKY-CRASH (died without a result, passed on the one retry; first exit code in brackets): test_free_threading.py({status})" in r.stdout + + +def test_a_crash_that_stayed_a_crash_fails_the_gate(program, tmp_path): + for status in ( + "CRASH(rc=0xC0000005,3221225477)", + "CRASH(rc=0xC0000005,3221225477;retry rc=0xC0000005,3221225477)", + "CRASH(rc=0xC0000005,3221225477;retry budget spent)", + ): + r = run_gate(program, tmp_path, row("test_a.py", passed=0, status=status), total()) + assert r.returncode == 1, status + assert f"test_a.py({status})" in r.stderr, status + + +def test_a_flaky_file_does_not_hide_another_files_crash(program, tmp_path): + r = run_gate( + program, + tmp_path, + row("test_flaky.py", status="FLAKY-CRASH(rc=0xC0000005,3221225477)", attempts=2), + row("test_dead.py", passed=0, status="CRASH(rc=0xC0000374,3221226356;retry rc=0xC0000374,3221226356)", attempts=2), + total(), + ) + assert r.returncode == 1 + assert "test_dead.py(CRASH" in r.stderr and "test_flaky.py" not in r.stderr + + +def test_a_flaky_file_whose_retry_had_a_failing_test_fails_the_gate(program, tmp_path): + # the status passes, the counts of the second run decide + r = run_gate( + program, + tmp_path, + row("test_flaky.py", failed=1, status="FLAKY-CRASH(rc=0xC0000005,3221225477)", attempts=2), + total(failed=1), + ) + assert r.returncode == 1 + + +def test_a_timeout_an_unreadable_xml_a_failure_and_a_skip_still_fail_the_gate(program, tmp_path): + assert run_gate(program, tmp_path, row("test_a.py", status="TIMEOUT"), total()).returncode == 1 + assert run_gate(program, tmp_path, row("test_a.py", status="BADXML"), total()).returncode == 1 + assert run_gate(program, tmp_path, row("test_a.py", failed=1), total(failed=1)).returncode == 1 + assert run_gate(program, tmp_path, row("test_a.py"), total(skipped=1)).returncode == 1 + assert run_gate(program, tmp_path, row("test_a.py")).returncode == 1 # no TOTAL row From fc7b5e213b6bde9e025534412401ee1553a11288 Mon Sep 17 00:00:00 2001 From: Taewoon Kim Date: Tue, 10 Nov 2026 14:36:37 +0900 Subject: [PATCH 43/49] CI: parity harness records DIRTY-EXIT, names the log that exists, tolerates locked housekeeping files (review of aa223d0) Review findings on the crash handling, all fixed here: - A process that wrote its XML and then died in shutdown was only a ::warning:: line. It is now a DIRTY-EXIT record (exit code hex and decimal, file, attempt) in meta.json, the job summary and the issue 295 comment. It is still not retried and does not fail the leg: the status stays what the XML says, so the gate is unchanged. Probed with a fixture conftest whose pytest_unconfigure calls os._exit. - The report named .attempt1.log for every crash, but that file exists only after a retry. A crash that was not retried (Linux, macOS, a spent budget) now names .log. - The claim that a stale XML cannot pass for a result had no test. A stale XML pre-seeded for a file that then crashes stays CRASH. If the stale XML cannot be removed (locked), an XML older than the attempt's start is not taken as its result. - Removing an empty trace, keeping the first attempt's log, writing a log and removing a stale XML go through housekeeping(): one retry after a short pause on OSError, then a ::warning:: and on. A virus-scanner lock on a just-closed file can no longer fail the harness or hide a result. Tests added to tests/test_parity_crash_retry.py and tests/test_parity_gate_awk.py; 9 further mutations of the new code each fail at least one test (20 in all). Co-Authored-By: Claude Sonnet 5.5 --- docs/modules/ROOT/pages/development/ci.adoc | 2 +- native/wheel/parity_crash.py | 25 +++ native/wheel/parity_wheel.py | 69 ++++++-- native/wheel/report_flaky_crash.py | 23 ++- tests/README.md | 2 +- tests/test_parity_crash_retry.py | 185 +++++++++++++++++++- tests/test_parity_gate_awk.py | 7 + 7 files changed, 290 insertions(+), 23 deletions(-) diff --git a/docs/modules/ROOT/pages/development/ci.adoc b/docs/modules/ROOT/pages/development/ci.adoc index f665f9d1..8c171227 100644 --- a/docs/modules/ROOT/pages/development/ci.adoc +++ b/docs/modules/ROOT/pages/development/ci.adoc @@ -98,7 +98,7 @@ The workflow `test-native.yml` builds the native image of the `arcadedb` package * A pull request runs it only when native paths change: `native/**`, `src/arcadedb/**`, `tests/**`, `scripts/select_native_matrix.py` or `test-native.yml` itself. Changes under `tests/` and `src/arcadedb/` are included because both backends share those files, so a shared test or source can break native parity without touching a native file; docs outside those paths do not run it. It then runs on `linux-x86_64` with Python 3.12 only. * A push to `main`, the nightly run, a manual run (dispatch) run all four platforms: `linux-x86_64`, `linux-aarch64`, `macos-arm64` and `windows-x64` (Python 3.12 and 3.14). A release (`release-native.yml` calls the workflow with `release: true`) runs the same four platforms on all four Python versions, 16 legs. * The `pyver` job installs the already built `linux-x86_64` wheel (no rebuild) into the Pythons that the full parity runs of the event leave out (3.11, 3.13 and 3.14 on a pull request; 3.11 and 3.13 on a push to `main`, the nightly run and a dispatch; none on a release) into 3.15 and into the free-threaded 3.14t, and runs 13 test files there (14 where polars installs; 3.15 has no jpype1 wheel, so it leaves out the two files that need jpype1), about 20 seconds of tests per Python. It is informational: `release-native.yml` reads only the `native-passed-*` markers and the `native-wheel-*` wheels, so a red `pyver` leg never blocks a release. The 3.14t leg has no polars (no free-threaded wheel exists), so it skips `test_dataframe_ingest.py`. `scripts/select_native_matrix.py` decides its legs. -* Every test file runs in its own process (`native/wheel/parity_wheel.py`). A process that dies without writing its result is a `CRASH` row in `parity-wheel.tsv` with its exit code, for example `CRASH(rc=0xC0000005,3221225477)`, and it fails the leg. On the Windows legs only (the crashes of issue 295 happen there), the harness runs such a file once more in a fresh process (`--retry-crash`). A retry that produces a result is a `FLAKY-CRASH` row: the gate lets the leg pass, the job summary lists the file with the first exit code, and on a release or a dispatch the `crash-report` job adds one comment to issue 295 (run URL, Python, wheel version, exit codes). A second crash, a failing test, a timeout and every crash on Linux or macOS stay failures. Each Windows test process also writes a faulthandler trace, `parity-wheel-xml/.attemptN.fault.txt`, into the parity artifact of the leg. Only a retry hides a crash, so treat a `FLAKY-CRASH` as an open bug: read the trace in the artifact before a release is announced. +* Every test file runs in its own process (`native/wheel/parity_wheel.py`). A process that dies without writing its result is a `CRASH` row in `parity-wheel.tsv` with its exit code, for example `CRASH(rc=0xC0000005,3221225477)`, and it fails the leg. On the Windows legs only (the crashes of issue 295 happen there), the harness runs such a file once more in a fresh process (`--retry-crash`). A retry that produces a result is a `FLAKY-CRASH` row: the gate lets the leg pass, the job summary lists the file with the first exit code, and on a release or a dispatch the `crash-report` job adds one comment to issue 295 (run URL, Python, wheel version, exit codes). A second crash, a failing test, a timeout and every crash on Linux or macOS stay failures. A process that wrote its result and then died in shutdown is a `DIRTY-EXIT` record (exit code, file, attempt) in the metadata, the job summary and the issue comment: it is not retried and does not fail the leg. Each Windows test process also writes a faulthandler trace, `parity-wheel-xml/.attemptN.fault.txt`, into the parity artifact of the leg. Only a retry hides a crash, so treat a `FLAKY-CRASH` as an open bug: read the trace in the artifact before a release is announced. * A native image build takes about 5 to 30 minutes for each platform (macOS is the slowest). The organisation shares one runner limit with upstream ArcadeDB, so do not add runs lightly. Change the paths or the matrix of `test-native.yml` only for a reason that you can write down. == Change the selection diff --git a/native/wheel/parity_crash.py b/native/wheel/parity_crash.py index a82ccc0e..e66f000d 100644 --- a/native/wheel/parity_crash.py +++ b/native/wheel/parity_crash.py @@ -14,6 +14,8 @@ test failure in it still fails the leg); if it crashes again the status is CRASH(rc=;retry rc=) and the leg fails. A retry budget per leg (default 3 files) keeps a wheel that crashes everywhere from passing on retries. +A process that wrote its result and then died in shutdown (DIRTY-EXIT) is recorded, never retried and never failed. + Everything here takes the facts of an attempt (Attempt) and returns the status and the attempts; the harness runs the processes. """ @@ -26,6 +28,7 @@ CRASH = "CRASH" FLAKY_CRASH = "FLAKY-CRASH" +DIRTY_EXIT = "DIRTY-EXIT" TIMEOUT = "TIMEOUT" BADXML = "BADXML" @@ -73,6 +76,28 @@ def dirty_exit(attempt: Attempt) -> bool: return attempt.result and attempt.rc is not None and attempt.rc not in PYTEST_EXIT_CODES +def dirty_events(name: str, outcome: "Outcome") -> list: + """One DIRTY-EXIT record per attempt that wrote its result and then died in shutdown (the metadata, the summary and the issue comment list it). + + It is a record and nothing else: the status of the file is what its XML says (the outcome is not retried and the leg is not failed). + The log of such an attempt is the file's final log, name.log. + """ + return [ + { + "file": name, + "kind": DIRTY_EXIT, + "first_rc": format_rc(a.rc), + "retry_rc": "", + "retry": "not retried, result kept", + "status": outcome.status, + "attempt": n, + "log": f"{name}.log", + } + for n, a in enumerate(outcome.attempts, 1) + if dirty_exit(a) + ] + + @dataclass class RetryPolicy: """Whether a crashed file may be run again, and how many files of one leg may be.""" diff --git a/native/wheel/parity_wheel.py b/native/wheel/parity_wheel.py index bbefa233..8cd88388 100644 --- a/native/wheel/parity_wheel.py +++ b/native/wheel/parity_wheel.py @@ -23,6 +23,7 @@ import shutil import subprocess import sys +import time import xml.etree.ElementTree as ET import parity_crash as pc # the directory of this script is sys.path[0] @@ -66,11 +67,42 @@ def parse_args(argv=None): return p.parse_args(argv) +# A file that was closed a moment ago can still be locked on Windows (Defender, the indexer). Housekeeping on logs and traces waits this long once. +HOUSEKEEPING_PAUSE = 0.5 + + +def housekeeping(what, op, *args): + """op(*args); on an OSError once more after a short pause; a second failure prints a ::warning:: and returns False. Never raises. + + Only for the files the harness keeps for people (logs, traces, a stale XML): losing one of them must never fail the leg or hide a result. + """ + for attempt in (1, 2): + try: + op(*args) + return True + except OSError as err: + if attempt == 1: + time.sleep(HOUSEKEEPING_PAUSE) + else: + print(f"::warning::{what} was skipped: {type(err).__name__}: {err}", flush=True) + return False + + def write_text(path, text): with open(path, "w") as fh: fh.write(text) +def traces_of(name, xmldir, attempts): + """{'trace1': file name, ...} for the faulthandler trace files that exist (an empty one was deleted).""" + found = {} + for n in attempts: + trace = f"{name}.attempt{n}.fault.txt" + if os.path.exists(os.path.join(xmldir, trace)): + found[f"trace{n}"] = trace + return found + + def count_xml(xml): """(counts, names of the failing tests) of a junit XML, or None when it does not parse.""" c = dict.fromkeys(COUNT_KEYS, 0) @@ -169,9 +201,17 @@ def main(argv=None): def attempt(n): if n > 1: # keep the log of the first attempt: the retry writes name.log - os.replace(log, os.path.join(xmldir, name + ".attempt1.log")) + housekeeping( + f"keeping the log of the first attempt of {name}", + os.replace, + log, + os.path.join(xmldir, name + ".attempt1.log"), + ) + stale = False if os.path.exists(xml): - os.remove(xml) # a stale XML of an earlier run must not pass for a result + # a stale XML of an earlier run must not pass for a result + stale = not housekeeping(f"removing a stale {name}.xml", os.remove, xml) + started = time.time() trace = os.path.join(xmldir, f"{name}.attempt{n}.fault.txt") env_n = dict(env) if sys.platform == "win32": @@ -212,10 +252,17 @@ def attempt(n): timeout=a.timeout, ) rc = r.returncode - write_text(log, r.stdout[-20000:] + "\n" + r.stderr[-5000:]) + housekeeping( + f"writing the log of {name}", + write_text, + log, + r.stdout[-20000:] + "\n" + r.stderr[-5000:], + ) except subprocess.TimeoutExpired as e: timed_out = True - write_text( + housekeeping( + f"writing the log of {name}", + write_text, log, ( ((e.stdout or b"")[-8000:]).decode() @@ -224,9 +271,10 @@ def attempt(n): ), ) if os.path.exists(trace) and os.path.getsize(trace) == 0: - os.remove(trace) + housekeeping(f"removing the empty trace of {name}", os.remove, trace) result, bad_xml = False, False - if os.path.exists(xml): + # an XML that could not be removed and was not written again by this attempt is not its result + if os.path.exists(xml) and not (stale and os.path.getmtime(xml) < started - 2): got = count_xml(xml) if got is None: bad_xml = True @@ -245,11 +293,12 @@ def attempt(n): tot["crash"] += out.hard_crash ev = pc.event_of(name, out, c["failed"] + c["error"]) if ev: - for n in range(1, len(out.attempts) + 1): - trace = f"{name}.attempt{n}.fault.txt" - if os.path.exists(os.path.join(xmldir, trace)): - ev[f"trace{n}"] = trace + ev.update(traces_of(name, xmldir, range(1, len(out.attempts) + 1))) events.append(ev) + # a process that wrote its result and then died in shutdown: recorded, never retried, never failed + for dirty in pc.dirty_events(name, out): + dirty.update(traces_of(name, xmldir, [dirty["attempt"]])) + events.append(dirty) rows.append((name, c, out.status, fails, len(out.attempts), out.final.rc)) print( f"{name:45s} pass={c['passed']:4d} fail={c['failed']:3d} err={c['error']:3d} notyet={c['notyet']:3d} na_jvm={c['na_jvm']:3d} skip={c['skipped']:3d} {out.status}", diff --git a/native/wheel/report_flaky_crash.py b/native/wheel/report_flaky_crash.py index 58bbbff2..9b3e97dc 100644 --- a/native/wheel/report_flaky_crash.py +++ b/native/wheel/report_flaky_crash.py @@ -3,7 +3,8 @@ native/wheel/parity_wheel.py retries a test file whose process died without a result (Windows legs only) and writes .meta.json beside its table, with one crash event per file: FLAKY-CRASH (the retry produced a result, so the leg may pass) -or CRASH (it died again, or no retry was allowed, so the leg fails). This script reads the meta files of every leg (the downloaded +or CRASH (it died again, or no retry was allowed, so the leg fails), or DIRTY-EXIT (the file wrote its result and the process then +died in shutdown: recorded, never retried, the leg is not failed). This script reads the meta files of every leg (the downloaded `native-parity-*` artifacts), then 1. appends a table to the job summary ($GITHUB_STEP_SUMMARY), and @@ -72,7 +73,9 @@ def _where(events: list[dict]) -> list[str]: lines = [] for e in events: files = [e[k] for k in ("trace1", "trace2") if e.get(k)] - names = ", ".join(f"`{n}`" for n in [*files, f"{e['file']}.attempt1.log"]) + # name.attempt1.log exists only when a retry ran (the retry's own log is name.log); a crash that was not retried has name.log alone + log = e.get("log") or (f"{e['file']}.attempt1.log" if e.get("retry_rc") else f"{e['file']}.log") + names = ", ".join(f"`{n}`" for n in [*files, log]) lines.append( f"- `{e['file']}` ({e['leg']}, Python {e['python']}): artifact `{e['artifact']}`, folder `parity-wheel-xml/`: {names}" ) @@ -84,7 +87,8 @@ def summary_md(events: list[dict]) -> str: if not events: return "" flaky = [e for e in events if e["kind"] == "FLAKY-CRASH"] - hard = [e for e in events if e["kind"] != "FLAKY-CRASH"] + hard = [e for e in events if e["kind"] == "CRASH"] + dirty = [e for e in events if e["kind"] == "DIRTY-EXIT"] out = ["### Parity crashes", ""] if flaky: out += [ @@ -98,7 +102,13 @@ def summary_md(events: list[dict]) -> str: f"**CRASH: {len(hard)} file(s) died without a result and the leg failed.**", "", ] - out += [*_table(events), "", "Crash traces and logs of the first attempt:", ""] + if dirty: + out += [ + f"**DIRTY-EXIT: {len(dirty)} file(s) wrote their result and then the process died in shutdown.** " + "The result stands: the file was not retried and the leg was not failed by this.", + "", + ] + out += [*_table(events), "", "Crash traces and logs:", ""] out += _where(events) return "\n".join(out) + "\n" @@ -115,9 +125,10 @@ def comment_md(events: list[dict], run_url: str, attempt: str, event: str, ref: "", "A test file died without a result and the harness ran it once more in a fresh process (`parity_wheel.py --retry-crash`, Windows legs only). " "A retry that says `passed` made its leg green: without the retry that leg would have failed. " - "`CRASH` means the file died again (or no retry was left) and the leg failed.", + "`CRASH` means the file died again (or no retry was left) and the leg failed. " + "`DIRTY-EXIT` means the file wrote its result and the process then died in shutdown: it was not retried and did not fail the leg.", "", - "The faulthandler trace and the log of the first attempt are in the uploaded parity artifact of each leg:", + "The faulthandler trace and the log are in the uploaded parity artifact of each leg:", "", *_where(events), ] diff --git a/tests/README.md b/tests/README.md index b8d98f4c..f1681e43 100644 --- a/tests/README.md +++ b/tests/README.md @@ -39,7 +39,7 @@ The same suite runs against the native `arcadedb` wheel (`native/wheel/parity_wh - `NotOnNativeYet`: the native code raised it because the feature needs a Java object API the library does not expose. The report says `NOTYET`, and a final release allows none. A call that is JVM-only by design (`get_java_database`, `get_java_document`, a Java object a `Schema` method returns) raises it with a message that names the `arcadedb-jvm` build; a shared test that reaches one is `NA-JVM`, or reads the value through a statement or the Python API instead. - A row of `NATIVE_NA_JVM_TESTS` in `tests/conftest.py`, keyed `::[::]`: the test reaches into the JVM on purpose (`jpype.JClass`, a Java array, `db._java_db`, counted bridge crossings). The report says `NA-JVM`. Add a row only for such a test, and give the reason. When the behaviour is shared, write a native test beside it (`test_native_*.py`, collected on the native build only). -A test file whose process dies without a result is a `CRASH` row with its exit code. On Windows the parity harness runs such a file once more (`--retry-crash`, set by `test-native.yml` on Windows legs only) and marks a retry that produced a result `FLAKY-CRASH`; the job summary and issue 295 record it. A failing test is never retried, and a crash on Linux or macOS is a failure. The decision is `native/wheel/parity_crash.py`, tested by `tests/test_parity_crash_retry.py`. +A test file whose process dies without a result is a `CRASH` row with its exit code. On Windows the parity harness runs such a file once more (`--retry-crash`, set by `test-native.yml` on Windows legs only) and marks a retry that produced a result `FLAKY-CRASH`; the job summary and issue 295 record it. A failing test is never retried, and a crash on Linux or macOS is a failure. A process that wrote its result and then died in shutdown is only recorded (`DIRTY-EXIT`). The decision is `native/wheel/parity_crash.py`, tested by `tests/test_parity_crash_retry.py`. A test that fails on the native build for any other reason is a failure. Fix the native code, or mark a known engine bug with a strict `xfail`. Do not add a row to hide it. diff --git a/tests/test_parity_crash_retry.py b/tests/test_parity_crash_retry.py index 9250e382..d737a300 100644 --- a/tests/test_parity_crash_retry.py +++ b/tests/test_parity_crash_retry.py @@ -13,6 +13,7 @@ import importlib.util import json +import os import re import signal import subprocess # nosec B404 - test-controlled child processes @@ -303,6 +304,16 @@ def test_a(): "test_dies_always.py": f"import os\n\n\ndef test_a():\n os._exit({HIGH_ARG})\n", } +# a process that dies in shutdown, after pytest wrote its XML (pytest_sessionfinish runs before pytest_unconfigure) +DIRTY_CONFTEST = f""" +import os + + +def pytest_unconfigure(config): + if any("test_dirty" in str(a) for a in config.args): + os._exit({HIGH_ARG}) +""" + IDENT = ( "/x/site-packages/arcadedb/__init__.py 26.11.1.post2 native 26.11.1 abc1234\n" 'META {"python": "3.14.0", "wheel_version": "26.11.1.post2", "engine_version": "26.11.1", "source_commit": "abc1234"}\n' @@ -325,12 +336,24 @@ def run(cmd, *args, **kwargs): monkeypatch.delenv("PYTHONPATH", raising=False) counter = iter(range(100)) - def go(*flags): + def go(*flags, dirty=False, seed=None): + """`dirty` adds test_dirty.py and a conftest whose process dies after the XML; `seed` puts files into the XML directory first + ({name: text} or {name: (text, age in seconds)}).""" repo = tmp_path / f"repo{next(counter)}" (repo / "tests").mkdir(parents=True) - (repo / "tests" / "conftest.py").write_text("# fixture\n") - for name, text in FIXTURE_FILES.items(): + files = dict(FIXTURE_FILES) + if dirty: + files["test_dirty.py"] = "def test_a():\n pass\n" + (repo / "tests" / "conftest.py").write_text(DIRTY_CONFTEST if dirty else "# fixture\n") + for name, text in files.items(): (repo / "tests" / name).write_text(text) + for name, item in (seed or {}).items(): + text, age = item if isinstance(item, tuple) else (item, 0) + path = repo / "out" / "parity-wheel-xml" / name + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(text) + if age: + os.utime(path, (path.stat().st_mtime - age, path.stat().st_mtime - age)) monkeypatch.setenv("DIES_ONCE_MARKER", str(repo / "marker")) monkeypatch.setenv("KEY", "windows-x64") out = repo / "out" / "parity-wheel.tsv" @@ -491,7 +514,7 @@ def write_meta(root, artifact, key, python, wheel, events, retry=True): "file": "test_free_threading.py", "kind": "FLAKY-CRASH", "first_rc": "0xC0000005,3221225477", - "retry_rc": "", + "retry_rc": "0x00000000,0", "retry": "passed", "status": "FLAKY-CRASH(rc=0xC0000005,3221225477)", "trace1": "test_free_threading.py.attempt1.fault.txt", @@ -567,7 +590,7 @@ def test_a_release_posts_one_comment_listing_every_event_and_the_summary_has_the md = summary.read_text() assert md.startswith("### Parity crashes") assert "**FLAKY-CRASH: 1 file(s)" in md and "**CRASH: 1 file(s)" in md - assert "| windows-x64 | 3.11.9 | 26.11.1.post2 | `test_free_threading.py` | FLAKY-CRASH 0xC0000005,3221225477 | passed |" in md + assert "| windows-x64 | 3.11.9 | 26.11.1.post2 | `test_free_threading.py` | FLAKY-CRASH 0xC0000005,3221225477 | passed (0x00000000,0) |" in md assert EM_DASH not in md @@ -660,3 +683,155 @@ def test_issues_write_is_held_by_the_crash_report_job_and_the_release_call_only( def test_the_flaky_comment_text_has_no_em_dash_in_any_source_that_writes_it(): for path in (WHEEL_DIR / "report_flaky_crash.py", WHEEL_DIR / "parity_crash.py", WHEEL_DIR / "parity_wheel.py"): assert EM_DASH not in path.read_text(encoding="utf-8"), path.name + + +# --------------------------------------------------------------------------------------------------------------------------------- +# a process that dies after its result, a stale XML, a locked log: the harness records, stays strict where it must and never fails on housekeeping +# --------------------------------------------------------------------------------------------------------------------------------- + +STALE_XML = '' + + +def lock(monkeypatch, name, suffix, times): + """os. raises PermissionError for a path ending in `suffix`: `times` times (None: always). Returns the list of the calls it saw.""" + real, left, seen = getattr(os, name), [times], [] + + def wrapper(*args, **kwargs): + target = str(args[1] if name == "replace" else args[0]) + if target.endswith(suffix): + seen.append(target) + if left[0] is None or left[0] > 0: + if left[0] is not None: + left[0] -= 1 + raise PermissionError(13, "locked by a virus scanner", target) + return real(*args, **kwargs) + + monkeypatch.setattr(os, name, wrapper) + return seen + + +def test_a_process_that_dies_after_its_result_is_a_dirty_exit_and_fails_nothing(harness, tmp_path): + rows, meta, xmldir = harness("--files", "test_dirty.py", "--retry-crash", dirty=True) + row = rows["test_dirty.py"] + # the XML says one passed test: the status stays clean, the file is not retried (even with the retry on), the exit code is kept + assert (row["status"], row["passed"], row["failed"], row["attempts"], row["exit"]) == ("", "1", "0", "1", HIGH_TEXT) + assert rows["TOTAL"]["status"] == "timeouts=0 flaky_crash=0 crash=0" + assert [(e["kind"], e["file"], e["first_rc"], e["attempt"], e["log"]) for e in meta["crash_events"]] == [ + ("DIRTY-EXIT", "test_dirty.py", HIGH_TEXT, 1, "test_dirty.py.log") + ] + assert not (xmldir / "test_dirty.py.attempt1.log").exists() + # the report lists it in the job summary and in the comment on issue 295 + root = tmp_path / "artifacts" + (root / "native-parity-windows-x64-py3.14").mkdir(parents=True) + (root / "native-parity-windows-x64-py3.14" / "parity-wheel.meta.json").write_text(json.dumps(meta)) + gh, summary = FakeGh(), tmp_path / "summary.md" + assert report.report(report.load_events(str(root)), RELEASE_ENV, str(summary), run=gh) == 0 + md = summary.read_text() + assert md.startswith("### Parity crashes") + assert "**DIRTY-EXIT: 1 file(s) wrote their result and then the process died in shutdown.**" in md + assert f"| windows-x64 | 3.14.0 | 26.11.1.post2 | `test_dirty.py` | DIRTY-EXIT {HIGH_TEXT} | not retried, result kept |" in md + assert "`test_dirty.py.log`" in md and "attempt1" not in md + assert len(gh.calls) == 1 + body = gh.calls[0][1] + assert body.startswith("DIRTY-EXIT in a native parity run.") + assert f"DIRTY-EXIT {HIGH_TEXT}" in body and "not retried, result kept" in body and "did not fail the leg" in body + assert EM_DASH not in body and EM_DASH not in md + + +def test_a_stale_xml_cannot_pass_for_the_result_of_a_file_that_crashes(harness): + rows, meta, xmldir = harness("--files", "test_dies_always.py", seed={"test_dies_always.py.xml": STALE_XML}) + row = rows["test_dies_always.py"] + assert row["status"] == f"CRASH(rc={HIGH_TEXT})" + assert (row["passed"], row["attempts"]) == ("0", "1") + assert not (xmldir / "test_dies_always.py.xml").exists() + + +def test_a_stale_xml_that_cannot_be_removed_is_still_not_the_result_of_this_run(harness, monkeypatch, capsys): + seen = lock(monkeypatch, "remove", ".xml", None) + monkeypatch.setattr(parity_wheel, "HOUSEKEEPING_PAUSE", 0) + rows, meta, xmldir = harness("--files", "test_dies_always.py", seed={"test_dies_always.py.xml": (STALE_XML, 600)}) + row = rows["test_dies_always.py"] + assert row["status"] == f"CRASH(rc={HIGH_TEXT})" and row["passed"] == "0" + assert len(seen) == 2 # one try and one more after the pause + assert "::warning::removing a stale test_dies_always.py.xml was skipped: PermissionError" in capsys.readouterr().out + + +def test_housekeeping_tries_twice_waits_once_and_never_raises(monkeypatch, capsys): + sleeps = [] + monkeypatch.setattr(parity_wheel, "HOUSEKEEPING_PAUSE", 0.25) + monkeypatch.setattr(parity_wheel.time, "sleep", sleeps.append) + calls = [] + + def once(): + calls.append(1) + if len(calls) < 2: + raise PermissionError("locked") + + assert parity_wheel.housekeeping("a log", once) is True + assert (len(calls), sleeps, capsys.readouterr().out) == (2, [0.25], "") + calls.clear() + sleeps.clear() + + def always(): + calls.append(1) + raise PermissionError("locked") + + assert parity_wheel.housekeeping("keeping a log", always) is False + assert (len(calls), sleeps) == (2, [0.25]) + assert capsys.readouterr().out.startswith("::warning::keeping a log was skipped: PermissionError") + + +def test_a_log_that_is_locked_once_is_kept_after_a_short_wait(harness, monkeypatch, capsys): + monkeypatch.setattr(parity_wheel, "HOUSEKEEPING_PAUSE", 0) + seen = lock(monkeypatch, "replace", ".attempt1.log", 1) + rows, meta, xmldir = harness("--files", "test_dies_once.py", "--retry-crash") + assert rows["test_dies_once.py"]["status"] == f"FLAKY-CRASH(rc={HIGH_TEXT})" + assert len(seen) == 2 + assert (xmldir / "test_dies_once.py.attempt1.log").exists() + assert "::warning::" not in capsys.readouterr().out + + +def test_a_log_that_stays_locked_is_a_warning_and_the_result_still_counts(harness, monkeypatch, capsys): + monkeypatch.setattr(parity_wheel, "HOUSEKEEPING_PAUSE", 0) + seen = lock(monkeypatch, "replace", ".attempt1.log", None) + rows, meta, xmldir = harness("--files", "test_dies_once.py", "--retry-crash") + once = rows["test_dies_once.py"] + assert (once["status"], once["passed"], once["failed"], once["attempts"]) == (f"FLAKY-CRASH(rc={HIGH_TEXT})", "1", "0", "2") + assert len(seen) == 2 + assert not (xmldir / "test_dies_once.py.attempt1.log").exists() + assert "::warning::keeping the log of the first attempt of test_dies_once.py was skipped: PermissionError" in capsys.readouterr().out + + +def test_an_empty_trace_that_cannot_be_deleted_is_a_warning_and_not_a_failure(harness, monkeypatch, capsys): + monkeypatch.setattr(parity_wheel, "HOUSEKEEPING_PAUSE", 0) + seen = lock(monkeypatch, "remove", ".fault.txt", None) + rows, meta, xmldir = harness("--files", "test_ok.py", seed={"test_ok.py.attempt1.fault.txt": ""}) + assert (rows["test_ok.py"]["status"], rows["test_ok.py"]["passed"]) == ("", "2") + assert len(seen) == 2 and (xmldir / "test_ok.py.attempt1.fault.txt").exists() + assert "::warning::removing the empty trace of test_ok.py was skipped: PermissionError" in capsys.readouterr().out + + +NOT_RETRIED_EVENT = { + "file": "test_find_nearest_query_forms.py", + "kind": "CRASH", + "first_rc": "-11,SIGSEGV", + "retry_rc": "", + "retry": "not retried", + "status": "CRASH(rc=-11,SIGSEGV)", +} + + +def test_the_report_names_the_log_that_exists(tmp_path): + root = tmp_path / "a" + write_meta(root, "native-parity-linux-x86_64-py3.12", "linux-x86_64", "3.12.7", "26.11.1.post2", [NOT_RETRIED_EVENT], retry=False) + write_meta(root, "native-parity-windows-x64-py3.11", "windows-x64", "3.11.9", "26.11.1.post2", [FLAKY_EVENT]) + write_meta(root, "native-parity-windows-x64-py3.14", "windows-x64", "3.14.0", "26.11.1.post2", [CRASH_EVENT]) + events = report.load_events(str(root)) + where = {line.split("`")[1]: line for line in report._where(events)} + # a crash that was not retried (Linux, macOS, a spent budget) has name.log alone; attempt1.log exists only after a retry + assert "`test_find_nearest_query_forms.py.log`" in where["test_find_nearest_query_forms.py"] + assert "attempt1.log" not in where["test_find_nearest_query_forms.py"] + assert "`test_free_threading.py.attempt1.log`" in where["test_free_threading.py"] + assert "`test_vector.py.attempt1.log`" in where["test_vector.py"] + body = report.comment_md(events, "https://example.invalid/run", "1", "push", "26.11.1.post2") + assert "`test_find_nearest_query_forms.py.log`" in body and "test_find_nearest_query_forms.py.attempt1.log" not in body diff --git a/tests/test_parity_gate_awk.py b/tests/test_parity_gate_awk.py index 135b494e..732e329e 100644 --- a/tests/test_parity_gate_awk.py +++ b/tests/test_parity_gate_awk.py @@ -108,3 +108,10 @@ def test_a_timeout_an_unreadable_xml_a_failure_and_a_skip_still_fail_the_gate(pr assert run_gate(program, tmp_path, row("test_a.py", failed=1), total(failed=1)).returncode == 1 assert run_gate(program, tmp_path, row("test_a.py"), total(skipped=1)).returncode == 1 assert run_gate(program, tmp_path, row("test_a.py")).returncode == 1 # no TOTAL row + + +def test_a_file_that_died_after_its_result_does_not_fail_the_gate(program, tmp_path): + # DIRTY-EXIT is a record in the metadata, not a status: the row is clean and only its exit column differs + r = run_gate(program, tmp_path, row("test_a.py", exit_="0x0000008B,139"), total()) + assert r.returncode == 0, r.stderr + assert "FLAKY-CRASH" not in r.stdout From d82067d78b47e6c68fa282abccf74451663681cb Mon Sep 17 00:00:00 2001 From: Taewoon Kim Date: Tue, 10 Nov 2026 15:48:27 +0900 Subject: [PATCH 44/49] Tests: bandit B105 on the fake tokens, SIGABRT number from the platform, black 26.10.1 (CI dispatch on the post2 candidate) The dispatch found two failures in the new tests, and a gate I had not run: - bandit (test-jvm.yml: uvx bandit==1.9.4 -c pyproject.toml -r src tests --severity-level low --confidence-level low) flagged B105 on the fake GH_TOKEN values. The repo's convention is a `# nosec B105` marker on the string's line (tests/conftest.py TEST_PASSWORD): the made-up token carries one, the no-token case now builds the environment without the key (the real case) and needs none. Bandit is clean locally: "No issues identified", 271 nosec-disabled. - test_exit_codes_are_written_as_ntstatus_and_decimal_or_as_a_signal expected "-6,SIGABRT": signal.SIGABRT is 22 on Windows. The expectation, and the SIGSEGV one in the real-fault test, are now computed from the signal module. Other Windows-reachable assumptions re-read: the new test files read the workflow, summary and trace files as UTF-8 explicitly; nothing else needed a change. - black 26.10.1 (the pre-commit hook; the base versions of conftest.py and parity_wheel.py were black-clean) would reformat all six touched files: applied. isort --profile black, scripts/check_no_embedded_name.py and scripts/check_test_skips.py are clean. 130 tests pass, 0 skips; the 20 mutations of the new code are each still caught (anchors updated to the black layout). Co-Authored-By: Claude Sonnet 5.5 --- native/wheel/parity_crash.py | 14 +- native/wheel/parity_wheel.py | 17 +- native/wheel/report_flaky_crash.py | 50 +++- tests/conftest.py | 3 +- tests/test_parity_crash_retry.py | 362 +++++++++++++++++++++++------ tests/test_parity_gate_awk.py | 83 +++++-- 6 files changed, 423 insertions(+), 106 deletions(-) diff --git a/native/wheel/parity_crash.py b/native/wheel/parity_crash.py index e66f000d..51f0629b 100644 --- a/native/wheel/parity_crash.py +++ b/native/wheel/parity_crash.py @@ -60,7 +60,9 @@ def format_rc(rc: Optional[int], windows: Optional[bool] = None) -> str: class Attempt: """The facts about one run of one test file.""" - rc: Optional[int] # the exit code; None when the harness killed the process (timed_out) + rc: Optional[ + int + ] # the exit code; None when the harness killed the process (timed_out) result: bool # the junit XML was written and parses timed_out: bool = False bad_xml: bool = False # the XML exists and does not parse @@ -73,7 +75,11 @@ def is_crash(attempt: Attempt) -> bool: def dirty_exit(attempt: Attempt) -> bool: """The XML was written but the process ended with a code pytest never returns: it died in shutdown, after the tests.""" - return attempt.result and attempt.rc is not None and attempt.rc not in PYTEST_EXIT_CODES + return ( + attempt.result + and attempt.rc is not None + and attempt.rc not in PYTEST_EXIT_CODES + ) def dirty_events(name: str, outcome: "Outcome") -> list: @@ -143,7 +149,9 @@ def run_file(run_attempt: Callable[[int], Attempt], policy: RetryPolicy) -> Outc """Run one test file by `run_attempt(1)` and, only for a crash and only when the policy allows, once more by `run_attempt(2)`.""" first = run_attempt(1) if not is_crash(first): - return Outcome((first,), TIMEOUT if first.timed_out else BADXML if first.bad_xml else "") + return Outcome( + (first,), TIMEOUT if first.timed_out else BADXML if first.bad_xml else "" + ) crash = f"rc={format_rc(first.rc)}" if not policy.take(): why = ";retry budget spent" if policy.enabled else "" diff --git a/native/wheel/parity_wheel.py b/native/wheel/parity_wheel.py index 8cd88388..3fe363c5 100644 --- a/native/wheel/parity_wheel.py +++ b/native/wheel/parity_wheel.py @@ -84,7 +84,10 @@ def housekeeping(what, op, *args): if attempt == 1: time.sleep(HOUSEKEEPING_PAUSE) else: - print(f"::warning::{what} was skipped: {type(err).__name__}: {err}", flush=True) + print( + f"::warning::{what} was skipped: {type(err).__name__}: {err}", + flush=True, + ) return False @@ -156,8 +159,12 @@ def main(argv=None): want = set(a.files.split(",")) files = [f for f in files if os.path.basename(f) in want] env = dict(os.environ, PYTHONUNBUFFERED="1") - env.pop("PYTHONPATH", None) # the wheel, not a source tree, must be what is imported - env.pop("ARCADEDB_BACKEND", None) # the backend comes from the installed distribution + env.pop( + "PYTHONPATH", None + ) # the wheel, not a source tree, must be what is imported + env.pop( + "ARCADEDB_BACKEND", None + ) # the backend comes from the installed distribution env.pop("ARCADEDB_NATIVE_LIB", None) # the library comes from the installed wheel env.pop(FAULT_FILE_ENV, None) # set per attempt below, on Windows only chk = subprocess.run( @@ -274,7 +281,9 @@ def attempt(n): housekeeping(f"removing the empty trace of {name}", os.remove, trace) result, bad_xml = False, False # an XML that could not be removed and was not written again by this attempt is not its result - if os.path.exists(xml) and not (stale and os.path.getmtime(xml) < started - 2): + if os.path.exists(xml) and not ( + stale and os.path.getmtime(xml) < started - 2 + ): got = count_xml(xml) if got is None: bad_xml = True diff --git a/native/wheel/report_flaky_crash.py b/native/wheel/report_flaky_crash.py index 9b3e97dc..2affe9ed 100644 --- a/native/wheel/report_flaky_crash.py +++ b/native/wheel/report_flaky_crash.py @@ -74,7 +74,9 @@ def _where(events: list[dict]) -> list[str]: for e in events: files = [e[k] for k in ("trace1", "trace2") if e.get(k)] # name.attempt1.log exists only when a retry ran (the retry's own log is name.log); a crash that was not retried has name.log alone - log = e.get("log") or (f"{e['file']}.attempt1.log" if e.get("retry_rc") else f"{e['file']}.log") + log = e.get("log") or ( + f"{e['file']}.attempt1.log" if e.get("retry_rc") else f"{e['file']}.log" + ) names = ", ".join(f"`{n}`" for n in [*files, log]) lines.append( f"- `{e['file']}` ({e['leg']}, Python {e['python']}): artifact `{e['artifact']}`, folder `parity-wheel-xml/`: {names}" @@ -113,7 +115,9 @@ def summary_md(events: list[dict]) -> str: return "\n".join(out) + "\n" -def comment_md(events: list[dict], run_url: str, attempt: str, event: str, ref: str) -> str: +def comment_md( + events: list[dict], run_url: str, attempt: str, event: str, ref: str +) -> str: """The comment for issue 295: one per run, every event in it.""" kinds = sorted({e["kind"] for e in events}) out = [ @@ -148,13 +152,30 @@ def may_post(event: str, ref_type: str, repo: str) -> str: def post_comment(body: str, run=subprocess.run) -> tuple[bool, str]: """POST the comment with the gh CLI and GH_TOKEN; (True, '') or (False, why). Never raises.""" - cmd = ["gh", "api", "--method", "POST", f"repos/{REPO}/issues/{ISSUE}/comments", "--input", "-"] + cmd = [ + "gh", + "api", + "--method", + "POST", + f"repos/{REPO}/issues/{ISSUE}/comments", + "--input", + "-", + ] try: - r = run(cmd, input=json.dumps({"body": body}), capture_output=True, text=True, timeout=120) # nosec B603 + r = run( + cmd, + input=json.dumps({"body": body}), + capture_output=True, + text=True, + timeout=120, + ) # nosec B603 except (OSError, subprocess.SubprocessError) as err: return False, f"{type(err).__name__}: {err}" if r.returncode != 0: - return False, (r.stderr or r.stdout or f"gh exited {r.returncode}").strip()[:300] + return ( + False, + (r.stderr or r.stdout or f"gh exited {r.returncode}").strip()[:300], + ) return True, "" @@ -178,7 +199,9 @@ def report(events, env, summary_path="", dry_run=False, run=subprocess.run) -> i env.get("GITHUB_EVENT_NAME", ""), env.get("GITHUB_REF_NAME", ""), ) - why = may_post(env.get("GITHUB_EVENT_NAME", ""), env.get("GITHUB_REF_TYPE", ""), repo) + why = may_post( + env.get("GITHUB_EVENT_NAME", ""), env.get("GITHUB_REF_TYPE", ""), repo + ) if why: print(f"issue #{ISSUE}: no comment posted ({why})", flush=True) return 0 @@ -186,13 +209,19 @@ def report(events, env, summary_path="", dry_run=False, run=subprocess.run) -> i print(f"issue #{ISSUE}: dry run, the comment would be:\n{body}", flush=True) return 0 if not (env.get("GH_TOKEN") or env.get("GITHUB_TOKEN")): - print(f"::warning::the FLAKY-CRASH comment on #{ISSUE} was skipped: no GH_TOKEN", flush=True) + print( + f"::warning::the FLAKY-CRASH comment on #{ISSUE} was skipped: no GH_TOKEN", + flush=True, + ) return 0 ok, err = post_comment(body, run=run) if ok: print(f"issue #{ISSUE}: comment posted", flush=True) else: - print(f"::warning::the FLAKY-CRASH comment on #{ISSUE} was skipped, the token cannot write issues or gh failed: {err}", flush=True) + print( + f"::warning::the FLAKY-CRASH comment on #{ISSUE} was skipped, the token cannot write issues or gh failed: {err}", + flush=True, + ) return 0 @@ -205,7 +234,10 @@ def main(argv=None) -> int: try: return report(load_events(a.artifacts), os.environ, a.summary, a.dry_run) except Exception as err: # noqa: BLE001 - a log is never a gate - print(f"::warning::the crash report failed: {type(err).__name__}: {err}", flush=True) + print( + f"::warning::the crash report failed: {type(err).__name__}: {err}", + flush=True, + ) return 0 diff --git a/tests/conftest.py b/tests/conftest.py index 7bc93d30..3671a7a7 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -337,7 +337,8 @@ def pytest_configure(config): trace = os.environ.get(FAULT_FILE_ENV) if trace and BACKEND == "native": global _fault_file - _fault_file = open(trace, "w") # kept open: faulthandler writes to its descriptor at the crash + # kept open: faulthandler writes to its descriptor at the crash + _fault_file = open(trace, "w") faulthandler.enable(file=_fault_file, all_threads=True) elif faulthandler.is_enabled(): faulthandler.disable() diff --git a/tests/test_parity_crash_retry.py b/tests/test_parity_crash_retry.py index d737a300..1aab86b7 100644 --- a/tests/test_parity_crash_retry.py +++ b/tests/test_parity_crash_retry.py @@ -155,10 +155,16 @@ def test_the_retry_budget_of_a_leg_is_spent_by_the_files_not_the_crashes(): out = pc.run_file(run, policy) statuses.append((out.status.split("(")[0], calls)) # two files retried and flaky, the third crashes where the budget is spent: a wheel that crashes everywhere cannot pass on retries - assert statuses == [("FLAKY-CRASH", [1, 2]), ("FLAKY-CRASH", [1, 2]), ("CRASH", [1])] + assert statuses == [ + ("FLAKY-CRASH", [1, 2]), + ("FLAKY-CRASH", [1, 2]), + ("CRASH", [1]), + ] assert policy.used == 2 run, _ = attempts(DIED) - assert pc.run_file(run, policy).status == f"CRASH(rc={HIGH_TEXT};retry budget spent)" + assert ( + pc.run_file(run, policy).status == f"CRASH(rc={HIGH_TEXT};retry budget spent)" + ) def test_exit_codes_are_written_as_ntstatus_and_decimal_or_as_a_signal(): @@ -167,7 +173,8 @@ def test_exit_codes_are_written_as_ntstatus_and_decimal_or_as_a_signal(): assert pc.format_rc(-1073741819, windows=True) == "0xC0000005,3221225477" assert pc.format_rc(139, windows=False) == "0x0000008B,139" assert pc.format_rc(0, windows=False) == "0x00000000,0" - assert pc.format_rc(-signal.SIGABRT, windows=False) == "-6,SIGABRT" + # the number of SIGABRT is the platform's (6 on POSIX, 22 on Windows) + assert pc.format_rc(-signal.SIGABRT, windows=False) == f"{-signal.SIGABRT},SIGABRT" assert pc.format_rc(-250, windows=False) == "-250,signal 250" assert pc.format_rc(None) == "none" @@ -185,7 +192,11 @@ def test_a_crash_event_names_the_kind_and_what_the_retry_did(): } run, _ = attempts(DIED, DIED_AGAIN) ev = pc.event_of("a.py", pc.run_file(run, ON()), final_failed=0) - assert (ev["kind"], ev["retry"], ev["retry_rc"]) == ("CRASH", "crashed again", OTHER_TEXT) + assert (ev["kind"], ev["retry"], ev["retry_rc"]) == ( + "CRASH", + "crashed again", + OTHER_TEXT, + ) run, _ = attempts(DIED) ev = pc.event_of("a.py", pc.run_file(run, pc.RetryPolicy()), final_failed=0) assert (ev["kind"], ev["retry"], ev["retry_rc"]) == ("CRASH", "not retried", "") @@ -251,7 +262,9 @@ def run(n): return run -def test_a_real_process_that_exits_with_a_high_code_is_recorded_with_that_code(tmp_path): +def test_a_real_process_that_exits_with_a_high_code_is_recorded_with_that_code( + tmp_path, +): out = pc.run_file(real_runner(EXIT_ALWAYS, tmp_path), pc.RetryPolicy()) assert out.final.rc == HIGH assert out.status == f"CRASH(rc={HIGH_TEXT})" @@ -264,7 +277,7 @@ def test_a_real_process_killed_by_a_fault_is_recorded_as_one(tmp_path): assert out.status == "CRASH(rc=0xC0000005,3221225477)" else: assert out.final.rc == -signal.SIGSEGV - assert out.status == "CRASH(rc=-11,SIGSEGV)" + assert out.status == f"CRASH(rc={-signal.SIGSEGV},SIGSEGV)" def test_the_retry_of_a_real_crash_is_a_new_process_and_a_pass_is_flaky(tmp_path): @@ -289,8 +302,7 @@ def test_a_real_file_that_crashes_every_time_fails_after_one_retry(tmp_path): FIXTURE_FILES = { "test_ok.py": "def test_a():\n pass\n\n\ndef test_b():\n pass\n", "test_fails.py": "def test_a():\n assert False, 'a real failure'\n", - "test_dies_once.py": textwrap.dedent( - f""" + "test_dies_once.py": textwrap.dedent(f""" import os, pathlib @@ -299,8 +311,7 @@ def test_a(): if not marker.exists(): marker.write_text("1") os._exit({HIGH_ARG}) - """ - ), + """), "test_dies_always.py": f"import os\n\n\ndef test_a():\n os._exit({HIGH_ARG})\n", } @@ -344,7 +355,9 @@ def go(*flags, dirty=False, seed=None): files = dict(FIXTURE_FILES) if dirty: files["test_dirty.py"] = "def test_a():\n pass\n" - (repo / "tests" / "conftest.py").write_text(DIRTY_CONFTEST if dirty else "# fixture\n") + (repo / "tests" / "conftest.py").write_text( + DIRTY_CONFTEST if dirty else "# fixture\n" + ) for name, text in files.items(): (repo / "tests" / name).write_text(text) for name, item in (seed or {}).items(): @@ -357,7 +370,19 @@ def go(*flags, dirty=False, seed=None): monkeypatch.setenv("DIES_ONCE_MARKER", str(repo / "marker")) monkeypatch.setenv("KEY", "windows-x64") out = repo / "out" / "parity-wheel.tsv" - parity_wheel.main(["--repo", str(repo), "--python", sys.executable, "--out", str(out), "--timeout", "120", *flags]) + parity_wheel.main( + [ + "--repo", + str(repo), + "--python", + sys.executable, + "--out", + str(out), + "--timeout", + "120", + *flags, + ] + ) lines = [ln.split("\t") for ln in out.read_text().splitlines()] header = lines[0] rows = {ln[0]: dict(zip(header, ln)) for ln in lines[1:]} @@ -370,7 +395,9 @@ def go(*flags, dirty=False, seed=None): def test_the_harness_is_strict_without_the_flag(harness): rows, meta, xmldir = harness() assert rows["test_ok.py"]["status"] == "" and rows["test_ok.py"]["passed"] == "2" - assert rows["test_fails.py"]["failed"] == "1" and rows["test_fails.py"]["status"] == "" + assert ( + rows["test_fails.py"]["failed"] == "1" and rows["test_fails.py"]["status"] == "" + ) # both crashing files are CRASH rows with the exit code, one attempt each, no retry for name in ("test_dies_once.py", "test_dies_always.py"): assert rows[name]["status"] == f"CRASH(rc={HIGH_TEXT})", name @@ -386,7 +413,10 @@ def test_the_harness_retries_only_the_crashes_with_the_flag(harness): rows, meta, xmldir = harness("--retry-crash") # a clean file and a file with a failing test run once, whatever the flag says assert (rows["test_ok.py"]["attempts"], rows["test_ok.py"]["status"]) == ("1", "") - assert (rows["test_fails.py"]["attempts"], rows["test_fails.py"]["status"]) == ("1", "") + assert (rows["test_fails.py"]["attempts"], rows["test_fails.py"]["status"]) == ( + "1", + "", + ) assert rows["test_fails.py"]["failed"] == "1" # the file that died once passes on the retry: FLAKY-CRASH with the first exit code, counted from the second run once = rows["test_dies_once.py"] @@ -414,7 +444,12 @@ def test_the_harness_retries_only_the_crashes_with_the_flag(harness): def test_the_harness_retry_is_off_unless_asked_for(): assert parity_wheel.parse_args(["--repo", "r", "--out", "o"]).retry_crash is False - assert parity_wheel.parse_args(["--repo", "r", "--out", "o", "--retry-crash"]).retry_crash is True + assert ( + parity_wheel.parse_args( + ["--repo", "r", "--out", "o", "--retry-crash"] + ).retry_crash + is True + ) assert parity_wheel.parse_args(["--repo", "r", "--out", "o"]).max_crash_retries == 3 @@ -471,7 +506,9 @@ def test_the_harness_retry_is_off_unless_asked_for(): """ -def test_a_native_run_on_windows_keeps_a_faulthandler_trace_and_nothing_else_does(tmp_path): +def test_a_native_run_on_windows_keeps_a_faulthandler_trace_and_nothing_else_does( + tmp_path, +): trace = tmp_path / "fault.txt" r = subprocess.run( # nosec B603 - fixed argument list, no shell [sys.executable, "-c", FAULT_CHILD, str(ROOT), str(trace)], @@ -482,8 +519,11 @@ def test_a_native_run_on_windows_keeps_a_faulthandler_trace_and_nothing_else_doe # every assert of the child passed (a failed one would end the child before it printed) and it then died of the fault assert "AssertionError" not in r.stderr, r.stderr assert "configured" in r.stdout, r.stderr - assert r.returncode == (0xC0000005 if WIN else -signal.SIGSEGV), (r.returncode, r.stderr) - text = trace.read_text() + assert r.returncode == (0xC0000005 if WIN else -signal.SIGSEGV), ( + r.returncode, + r.stderr, + ) + text = trace.read_text(encoding="utf-8") assert "Segmentation fault" in text or "access violation" in text assert "most recent call first" in text @@ -535,16 +575,37 @@ def write_meta(root, artifact, key, python, wheel, events, retry=True): "GITHUB_REF_TYPE": "tag", "GITHUB_REF_NAME": "26.11.1.post2", "GITHUB_SERVER_URL": "https://github.com", - "GH_TOKEN": "token", + "GH_TOKEN": "token", # nosec B105 - a made-up token in a test environment, it reaches no service } @pytest.fixture def artifacts(tmp_path): root = tmp_path / "artifacts" - write_meta(root, "native-parity-windows-x64-py3.11", "windows-x64", "3.11.9", "26.11.1.post2", [FLAKY_EVENT]) - write_meta(root, "native-parity-windows-x64-py3.12", "windows-x64", "3.12.7", "26.11.1.post2", []) - write_meta(root, "native-parity-windows-x64-py3.14", "windows-x64", "3.14.0", "26.11.1.post2", [CRASH_EVENT]) + write_meta( + root, + "native-parity-windows-x64-py3.11", + "windows-x64", + "3.11.9", + "26.11.1.post2", + [FLAKY_EVENT], + ) + write_meta( + root, + "native-parity-windows-x64-py3.12", + "windows-x64", + "3.12.7", + "26.11.1.post2", + [], + ) + write_meta( + root, + "native-parity-windows-x64-py3.14", + "windows-x64", + "3.14.0", + "26.11.1.post2", + [CRASH_EVENT], + ) return root @@ -554,10 +615,14 @@ def __init__(self, returncode=0, stderr=""): def __call__(self, cmd, **kwargs): self.calls.append((cmd, json.loads(kwargs["input"])["body"])) - return subprocess.CompletedProcess(cmd, self.returncode, stdout="", stderr=self.stderr) + return subprocess.CompletedProcess( + cmd, self.returncode, stdout="", stderr=self.stderr + ) -def test_the_events_of_every_leg_are_collected_with_the_identity_of_their_leg(artifacts): +def test_the_events_of_every_leg_are_collected_with_the_identity_of_their_leg( + artifacts, +): events = report.load_events(str(artifacts)) assert [(e["file"], e["python"], e["wheel"]) for e in events] == [ ("test_free_threading.py", "3.11.9", "26.11.1.post2"), @@ -566,13 +631,23 @@ def test_the_events_of_every_leg_are_collected_with_the_identity_of_their_leg(ar assert report.load_events(str(artifacts / "none")) == [] -def test_a_release_posts_one_comment_listing_every_event_and_the_summary_has_the_table(artifacts, tmp_path, capsys): +def test_a_release_posts_one_comment_listing_every_event_and_the_summary_has_the_table( + artifacts, tmp_path, capsys +): gh, summary = FakeGh(), tmp_path / "summary.md" - code = report.report(report.load_events(str(artifacts)), RELEASE_ENV, str(summary), run=gh) + code = report.report( + report.load_events(str(artifacts)), RELEASE_ENV, str(summary), run=gh + ) assert code == 0 assert len(gh.calls) == 1 cmd, body = gh.calls[0] - assert cmd[:5] == ["gh", "api", "--method", "POST", "repos/ArcadeData/arcadedb-python/issues/295/comments"] + assert cmd[:5] == [ + "gh", + "api", + "--method", + "POST", + "repos/ArcadeData/arcadedb-python/issues/295/comments", + ] for text in ( "https://github.com/ArcadeData/arcadedb-python/actions/runs/42744219731", "attempt 2", @@ -587,10 +662,13 @@ def test_a_release_posts_one_comment_listing_every_event_and_the_summary_has_the assert text in body, text assert body.startswith("CRASH and FLAKY-CRASH in a native parity run.") assert EM_DASH not in body - md = summary.read_text() + md = summary.read_text(encoding="utf-8") assert md.startswith("### Parity crashes") assert "**FLAKY-CRASH: 1 file(s)" in md and "**CRASH: 1 file(s)" in md - assert "| windows-x64 | 3.11.9 | 26.11.1.post2 | `test_free_threading.py` | FLAKY-CRASH 0xC0000005,3221225477 | passed (0x00000000,0) |" in md + assert ( + "| windows-x64 | 3.11.9 | 26.11.1.post2 | `test_free_threading.py` | FLAKY-CRASH 0xC0000005,3221225477 | passed (0x00000000,0) |" + in md + ) assert EM_DASH not in md @@ -598,20 +676,46 @@ def test_a_dispatch_posts_and_a_pull_request_a_branch_push_and_a_fork_do_not(art events = report.load_events(str(artifacts)) posted = {} for name, env in { - "dispatch": {**RELEASE_ENV, "GITHUB_EVENT_NAME": "workflow_dispatch", "GITHUB_REF_TYPE": "branch"}, - "pull_request": {**RELEASE_ENV, "GITHUB_EVENT_NAME": "pull_request", "GITHUB_REF_TYPE": "branch"}, - "branch push": {**RELEASE_ENV, "GITHUB_EVENT_NAME": "push", "GITHUB_REF_TYPE": "branch"}, - "schedule": {**RELEASE_ENV, "GITHUB_EVENT_NAME": "schedule", "GITHUB_REF_TYPE": "branch"}, + "dispatch": { + **RELEASE_ENV, + "GITHUB_EVENT_NAME": "workflow_dispatch", + "GITHUB_REF_TYPE": "branch", + }, + "pull_request": { + **RELEASE_ENV, + "GITHUB_EVENT_NAME": "pull_request", + "GITHUB_REF_TYPE": "branch", + }, + "branch push": { + **RELEASE_ENV, + "GITHUB_EVENT_NAME": "push", + "GITHUB_REF_TYPE": "branch", + }, + "schedule": { + **RELEASE_ENV, + "GITHUB_EVENT_NAME": "schedule", + "GITHUB_REF_TYPE": "branch", + }, "fork": {**RELEASE_ENV, "GITHUB_REPOSITORY": "humemai/arcadedb-python"}, }.items(): gh = FakeGh() assert report.report(events, env, run=gh) == 0 posted[name] = len(gh.calls) - assert posted == {"dispatch": 1, "pull_request": 0, "branch push": 0, "schedule": 0, "fork": 0} + assert posted == { + "dispatch": 1, + "pull_request": 0, + "branch push": 0, + "schedule": 0, + "fork": 0, + } -def test_a_token_that_cannot_write_issues_is_a_visible_warning_and_never_a_failure(artifacts, capsys): - gh = FakeGh(returncode=1, stderr="gh: Resource not accessible by integration (HTTP 403)") +def test_a_token_that_cannot_write_issues_is_a_visible_warning_and_never_a_failure( + artifacts, capsys +): + gh = FakeGh( + returncode=1, stderr="gh: Resource not accessible by integration (HTTP 403)" + ) assert report.report(report.load_events(str(artifacts)), RELEASE_ENV, run=gh) == 0 out = capsys.readouterr().out assert len(gh.calls) == 1 @@ -619,16 +723,30 @@ def test_a_token_that_cannot_write_issues_is_a_visible_warning_and_never_a_failu assert "HTTP 403" in out # no token at all: gh is not even called gh = FakeGh() - assert report.report(report.load_events(str(artifacts)), {**RELEASE_ENV, "GH_TOKEN": ""}, run=gh) == 0 + no_token = {k: v for k, v in RELEASE_ENV.items() if k != "GH_TOKEN"} + assert report.report(report.load_events(str(artifacts)), no_token, run=gh) == 0 assert gh.calls == [] - assert "::warning::the FLAKY-CRASH comment on #295 was skipped: no GH_TOKEN" in capsys.readouterr().out + assert ( + "::warning::the FLAKY-CRASH comment on #295 was skipped: no GH_TOKEN" + in capsys.readouterr().out + ) def test_a_run_without_a_crash_writes_nothing_and_posts_nothing(tmp_path, capsys): root = tmp_path / "a" - write_meta(root, "native-parity-windows-x64-py3.12", "windows-x64", "3.12.7", "26.11.1.post2", []) + write_meta( + root, + "native-parity-windows-x64-py3.12", + "windows-x64", + "3.12.7", + "26.11.1.post2", + [], + ) gh, summary = FakeGh(), tmp_path / "summary.md" - assert report.report(report.load_events(str(root)), RELEASE_ENV, str(summary), run=gh) == 0 + assert ( + report.report(report.load_events(str(root)), RELEASE_ENV, str(summary), run=gh) + == 0 + ) assert gh.calls == [] and not summary.exists() @@ -653,10 +771,14 @@ def jobs_of(text): def test_the_workflow_passes_the_retry_flag_on_windows_legs_only(): text = (WORKFLOWS / "test-native.yml").read_text(encoding="utf-8") - assert text.count("--retry-crash") == 2 # once in the comment of the header, once in the Windows branch - assert re.search(r'if \[ "\$RUNNER_OS" = Windows \]; then retry=\(--retry-crash\); fi', text) + assert ( + text.count("--retry-crash") == 2 + ) # once in the comment of the header, once in the Windows branch + assert re.search( + r'if \[ "\$RUNNER_OS" = Windows \]; then retry=\(--retry-crash\); fi', text + ) jobs = jobs_of(text) - assert 'retry=(--retry-crash)' in jobs["test"] + assert "retry=(--retry-crash)" in jobs["test"] assert '${retry[@]+"${retry[@]}"}' in jobs["test"] # the other jobs that run the harness never pass it for name, job in jobs.items(): @@ -681,7 +803,11 @@ def test_issues_write_is_held_by_the_crash_report_job_and_the_release_call_only( def test_the_flaky_comment_text_has_no_em_dash_in_any_source_that_writes_it(): - for path in (WHEEL_DIR / "report_flaky_crash.py", WHEEL_DIR / "parity_crash.py", WHEEL_DIR / "parity_wheel.py"): + for path in ( + WHEEL_DIR / "report_flaky_crash.py", + WHEEL_DIR / "parity_crash.py", + WHEEL_DIR / "parity_wheel.py", + ): assert EM_DASH not in path.read_text(encoding="utf-8"), path.name @@ -710,50 +836,87 @@ def wrapper(*args, **kwargs): return seen -def test_a_process_that_dies_after_its_result_is_a_dirty_exit_and_fails_nothing(harness, tmp_path): - rows, meta, xmldir = harness("--files", "test_dirty.py", "--retry-crash", dirty=True) +def test_a_process_that_dies_after_its_result_is_a_dirty_exit_and_fails_nothing( + harness, tmp_path +): + rows, meta, xmldir = harness( + "--files", "test_dirty.py", "--retry-crash", dirty=True + ) row = rows["test_dirty.py"] # the XML says one passed test: the status stays clean, the file is not retried (even with the retry on), the exit code is kept - assert (row["status"], row["passed"], row["failed"], row["attempts"], row["exit"]) == ("", "1", "0", "1", HIGH_TEXT) + assert ( + row["status"], + row["passed"], + row["failed"], + row["attempts"], + row["exit"], + ) == ("", "1", "0", "1", HIGH_TEXT) assert rows["TOTAL"]["status"] == "timeouts=0 flaky_crash=0 crash=0" - assert [(e["kind"], e["file"], e["first_rc"], e["attempt"], e["log"]) for e in meta["crash_events"]] == [ - ("DIRTY-EXIT", "test_dirty.py", HIGH_TEXT, 1, "test_dirty.py.log") - ] + assert [ + (e["kind"], e["file"], e["first_rc"], e["attempt"], e["log"]) + for e in meta["crash_events"] + ] == [("DIRTY-EXIT", "test_dirty.py", HIGH_TEXT, 1, "test_dirty.py.log")] assert not (xmldir / "test_dirty.py.attempt1.log").exists() # the report lists it in the job summary and in the comment on issue 295 root = tmp_path / "artifacts" (root / "native-parity-windows-x64-py3.14").mkdir(parents=True) - (root / "native-parity-windows-x64-py3.14" / "parity-wheel.meta.json").write_text(json.dumps(meta)) + (root / "native-parity-windows-x64-py3.14" / "parity-wheel.meta.json").write_text( + json.dumps(meta) + ) gh, summary = FakeGh(), tmp_path / "summary.md" - assert report.report(report.load_events(str(root)), RELEASE_ENV, str(summary), run=gh) == 0 - md = summary.read_text() + assert ( + report.report(report.load_events(str(root)), RELEASE_ENV, str(summary), run=gh) + == 0 + ) + md = summary.read_text(encoding="utf-8") assert md.startswith("### Parity crashes") - assert "**DIRTY-EXIT: 1 file(s) wrote their result and then the process died in shutdown.**" in md - assert f"| windows-x64 | 3.14.0 | 26.11.1.post2 | `test_dirty.py` | DIRTY-EXIT {HIGH_TEXT} | not retried, result kept |" in md + assert ( + "**DIRTY-EXIT: 1 file(s) wrote their result and then the process died in shutdown.**" + in md + ) + assert ( + f"| windows-x64 | 3.14.0 | 26.11.1.post2 | `test_dirty.py` | DIRTY-EXIT {HIGH_TEXT} | not retried, result kept |" + in md + ) assert "`test_dirty.py.log`" in md and "attempt1" not in md assert len(gh.calls) == 1 body = gh.calls[0][1] assert body.startswith("DIRTY-EXIT in a native parity run.") - assert f"DIRTY-EXIT {HIGH_TEXT}" in body and "not retried, result kept" in body and "did not fail the leg" in body + assert ( + f"DIRTY-EXIT {HIGH_TEXT}" in body + and "not retried, result kept" in body + and "did not fail the leg" in body + ) assert EM_DASH not in body and EM_DASH not in md def test_a_stale_xml_cannot_pass_for_the_result_of_a_file_that_crashes(harness): - rows, meta, xmldir = harness("--files", "test_dies_always.py", seed={"test_dies_always.py.xml": STALE_XML}) + rows, meta, xmldir = harness( + "--files", "test_dies_always.py", seed={"test_dies_always.py.xml": STALE_XML} + ) row = rows["test_dies_always.py"] assert row["status"] == f"CRASH(rc={HIGH_TEXT})" assert (row["passed"], row["attempts"]) == ("0", "1") assert not (xmldir / "test_dies_always.py.xml").exists() -def test_a_stale_xml_that_cannot_be_removed_is_still_not_the_result_of_this_run(harness, monkeypatch, capsys): +def test_a_stale_xml_that_cannot_be_removed_is_still_not_the_result_of_this_run( + harness, monkeypatch, capsys +): seen = lock(monkeypatch, "remove", ".xml", None) monkeypatch.setattr(parity_wheel, "HOUSEKEEPING_PAUSE", 0) - rows, meta, xmldir = harness("--files", "test_dies_always.py", seed={"test_dies_always.py.xml": (STALE_XML, 600)}) + rows, meta, xmldir = harness( + "--files", + "test_dies_always.py", + seed={"test_dies_always.py.xml": (STALE_XML, 600)}, + ) row = rows["test_dies_always.py"] assert row["status"] == f"CRASH(rc={HIGH_TEXT})" and row["passed"] == "0" assert len(seen) == 2 # one try and one more after the pause - assert "::warning::removing a stale test_dies_always.py.xml was skipped: PermissionError" in capsys.readouterr().out + assert ( + "::warning::removing a stale test_dies_always.py.xml was skipped: PermissionError" + in capsys.readouterr().out + ) def test_housekeeping_tries_twice_waits_once_and_never_raises(monkeypatch, capsys): @@ -778,10 +941,14 @@ def always(): assert parity_wheel.housekeeping("keeping a log", always) is False assert (len(calls), sleeps) == (2, [0.25]) - assert capsys.readouterr().out.startswith("::warning::keeping a log was skipped: PermissionError") + assert capsys.readouterr().out.startswith( + "::warning::keeping a log was skipped: PermissionError" + ) -def test_a_log_that_is_locked_once_is_kept_after_a_short_wait(harness, monkeypatch, capsys): +def test_a_log_that_is_locked_once_is_kept_after_a_short_wait( + harness, monkeypatch, capsys +): monkeypatch.setattr(parity_wheel, "HOUSEKEEPING_PAUSE", 0) seen = lock(monkeypatch, "replace", ".attempt1.log", 1) rows, meta, xmldir = harness("--files", "test_dies_once.py", "--retry-crash") @@ -791,24 +958,41 @@ def test_a_log_that_is_locked_once_is_kept_after_a_short_wait(harness, monkeypat assert "::warning::" not in capsys.readouterr().out -def test_a_log_that_stays_locked_is_a_warning_and_the_result_still_counts(harness, monkeypatch, capsys): +def test_a_log_that_stays_locked_is_a_warning_and_the_result_still_counts( + harness, monkeypatch, capsys +): monkeypatch.setattr(parity_wheel, "HOUSEKEEPING_PAUSE", 0) seen = lock(monkeypatch, "replace", ".attempt1.log", None) rows, meta, xmldir = harness("--files", "test_dies_once.py", "--retry-crash") once = rows["test_dies_once.py"] - assert (once["status"], once["passed"], once["failed"], once["attempts"]) == (f"FLAKY-CRASH(rc={HIGH_TEXT})", "1", "0", "2") + assert (once["status"], once["passed"], once["failed"], once["attempts"]) == ( + f"FLAKY-CRASH(rc={HIGH_TEXT})", + "1", + "0", + "2", + ) assert len(seen) == 2 assert not (xmldir / "test_dies_once.py.attempt1.log").exists() - assert "::warning::keeping the log of the first attempt of test_dies_once.py was skipped: PermissionError" in capsys.readouterr().out + assert ( + "::warning::keeping the log of the first attempt of test_dies_once.py was skipped: PermissionError" + in capsys.readouterr().out + ) -def test_an_empty_trace_that_cannot_be_deleted_is_a_warning_and_not_a_failure(harness, monkeypatch, capsys): +def test_an_empty_trace_that_cannot_be_deleted_is_a_warning_and_not_a_failure( + harness, monkeypatch, capsys +): monkeypatch.setattr(parity_wheel, "HOUSEKEEPING_PAUSE", 0) seen = lock(monkeypatch, "remove", ".fault.txt", None) - rows, meta, xmldir = harness("--files", "test_ok.py", seed={"test_ok.py.attempt1.fault.txt": ""}) + rows, meta, xmldir = harness( + "--files", "test_ok.py", seed={"test_ok.py.attempt1.fault.txt": ""} + ) assert (rows["test_ok.py"]["status"], rows["test_ok.py"]["passed"]) == ("", "2") assert len(seen) == 2 and (xmldir / "test_ok.py.attempt1.fault.txt").exists() - assert "::warning::removing the empty trace of test_ok.py was skipped: PermissionError" in capsys.readouterr().out + assert ( + "::warning::removing the empty trace of test_ok.py was skipped: PermissionError" + in capsys.readouterr().out + ) NOT_RETRIED_EVENT = { @@ -823,15 +1007,45 @@ def test_an_empty_trace_that_cannot_be_deleted_is_a_warning_and_not_a_failure(ha def test_the_report_names_the_log_that_exists(tmp_path): root = tmp_path / "a" - write_meta(root, "native-parity-linux-x86_64-py3.12", "linux-x86_64", "3.12.7", "26.11.1.post2", [NOT_RETRIED_EVENT], retry=False) - write_meta(root, "native-parity-windows-x64-py3.11", "windows-x64", "3.11.9", "26.11.1.post2", [FLAKY_EVENT]) - write_meta(root, "native-parity-windows-x64-py3.14", "windows-x64", "3.14.0", "26.11.1.post2", [CRASH_EVENT]) + write_meta( + root, + "native-parity-linux-x86_64-py3.12", + "linux-x86_64", + "3.12.7", + "26.11.1.post2", + [NOT_RETRIED_EVENT], + retry=False, + ) + write_meta( + root, + "native-parity-windows-x64-py3.11", + "windows-x64", + "3.11.9", + "26.11.1.post2", + [FLAKY_EVENT], + ) + write_meta( + root, + "native-parity-windows-x64-py3.14", + "windows-x64", + "3.14.0", + "26.11.1.post2", + [CRASH_EVENT], + ) events = report.load_events(str(root)) where = {line.split("`")[1]: line for line in report._where(events)} # a crash that was not retried (Linux, macOS, a spent budget) has name.log alone; attempt1.log exists only after a retry - assert "`test_find_nearest_query_forms.py.log`" in where["test_find_nearest_query_forms.py"] + assert ( + "`test_find_nearest_query_forms.py.log`" + in where["test_find_nearest_query_forms.py"] + ) assert "attempt1.log" not in where["test_find_nearest_query_forms.py"] assert "`test_free_threading.py.attempt1.log`" in where["test_free_threading.py"] assert "`test_vector.py.attempt1.log`" in where["test_vector.py"] - body = report.comment_md(events, "https://example.invalid/run", "1", "push", "26.11.1.post2") - assert "`test_find_nearest_query_forms.py.log`" in body and "test_find_nearest_query_forms.py.attempt1.log" not in body + body = report.comment_md( + events, "https://example.invalid/run", "1", "push", "26.11.1.post2" + ) + assert ( + "`test_find_nearest_query_forms.py.log`" in body + and "test_find_nearest_query_forms.py.attempt1.log" not in body + ) diff --git a/tests/test_parity_gate_awk.py b/tests/test_parity_gate_awk.py index 732e329e..a2e8069a 100644 --- a/tests/test_parity_gate_awk.py +++ b/tests/test_parity_gate_awk.py @@ -17,7 +17,9 @@ import pytest -WORKFLOW = Path(__file__).resolve().parents[1] / ".github" / "workflows" / "test-native.yml" +WORKFLOW = ( + Path(__file__).resolve().parents[1] / ".github" / "workflows" / "test-native.yml" +) HEADER = "file\tpassed\tfailed\terror\tnotyet\tna_jvm\tskipped\tstatus\tfailing_tests\tattempts\texit\n" @@ -33,7 +35,9 @@ def row(name, passed=3, failed=0, error=0, status="", attempts=1, exit_="0x00000 return f"{name}\t{passed}\t{failed}\t{error}\t0\t0\t0\t{status}\t\t{attempts}\t{exit_}\n" -def total(passed=10, failed=0, error=0, skipped=0, status="timeouts=0 flaky_crash=0 crash=0"): +def total( + passed=10, failed=0, error=0, skipped=0, status="timeouts=0 flaky_crash=0 crash=0" +): return f"TOTAL\t{passed}\t{failed}\t{error}\t0\t0\t{skipped}\t{status}\t\t\t\n" @@ -41,7 +45,12 @@ def run_gate(program, tmp_path, *rows): tsv = tmp_path / "parity-wheel.tsv" tsv.write_text(HEADER + "".join(rows)) return subprocess.run( # nosec B603 B607 - fixed argument list, no shell - ["awk", "-F\\t", program, str(tsv)], # -F'\t' of the workflow: awk reads the backslash-t itself + [ + "awk", + "-F\\t", + program, + str(tsv), + ], # -F'\t' of the workflow: awk reads the backslash-t itself capture_output=True, text=True, env={**os.environ, "KEY": "windows-x64", "PY": "3.14"}, @@ -61,11 +70,22 @@ def test_a_clean_table_passes_and_prints_no_flaky_line(program, tmp_path): assert "FLAKY-CRASH" not in r.stdout -def test_a_file_that_crashed_and_passed_on_its_retry_passes_the_gate_loudly(program, tmp_path): +def test_a_file_that_crashed_and_passed_on_its_retry_passes_the_gate_loudly( + program, tmp_path +): status = "FLAKY-CRASH(rc=0xC0000005,3221225477)" - r = run_gate(program, tmp_path, row("test_a.py"), row("test_free_threading.py", status=status, attempts=2), total()) + r = run_gate( + program, + tmp_path, + row("test_a.py"), + row("test_free_threading.py", status=status, attempts=2), + total(), + ) assert r.returncode == 0, r.stderr - assert f"FLAKY-CRASH (died without a result, passed on the one retry; first exit code in brackets): test_free_threading.py({status})" in r.stdout + assert ( + f"FLAKY-CRASH (died without a result, passed on the one retry; first exit code in brackets): test_free_threading.py({status})" + in r.stdout + ) def test_a_crash_that_stayed_a_crash_fails_the_gate(program, tmp_path): @@ -74,7 +94,9 @@ def test_a_crash_that_stayed_a_crash_fails_the_gate(program, tmp_path): "CRASH(rc=0xC0000005,3221225477;retry rc=0xC0000005,3221225477)", "CRASH(rc=0xC0000005,3221225477;retry budget spent)", ): - r = run_gate(program, tmp_path, row("test_a.py", passed=0, status=status), total()) + r = run_gate( + program, tmp_path, row("test_a.py", passed=0, status=status), total() + ) assert r.returncode == 1, status assert f"test_a.py({status})" in r.stderr, status @@ -83,8 +105,15 @@ def test_a_flaky_file_does_not_hide_another_files_crash(program, tmp_path): r = run_gate( program, tmp_path, - row("test_flaky.py", status="FLAKY-CRASH(rc=0xC0000005,3221225477)", attempts=2), - row("test_dead.py", passed=0, status="CRASH(rc=0xC0000374,3221226356;retry rc=0xC0000374,3221226356)", attempts=2), + row( + "test_flaky.py", status="FLAKY-CRASH(rc=0xC0000005,3221225477)", attempts=2 + ), + row( + "test_dead.py", + passed=0, + status="CRASH(rc=0xC0000374,3221226356;retry rc=0xC0000374,3221226356)", + attempts=2, + ), total(), ) assert r.returncode == 1 @@ -96,17 +125,41 @@ def test_a_flaky_file_whose_retry_had_a_failing_test_fails_the_gate(program, tmp r = run_gate( program, tmp_path, - row("test_flaky.py", failed=1, status="FLAKY-CRASH(rc=0xC0000005,3221225477)", attempts=2), + row( + "test_flaky.py", + failed=1, + status="FLAKY-CRASH(rc=0xC0000005,3221225477)", + attempts=2, + ), total(failed=1), ) assert r.returncode == 1 -def test_a_timeout_an_unreadable_xml_a_failure_and_a_skip_still_fail_the_gate(program, tmp_path): - assert run_gate(program, tmp_path, row("test_a.py", status="TIMEOUT"), total()).returncode == 1 - assert run_gate(program, tmp_path, row("test_a.py", status="BADXML"), total()).returncode == 1 - assert run_gate(program, tmp_path, row("test_a.py", failed=1), total(failed=1)).returncode == 1 - assert run_gate(program, tmp_path, row("test_a.py"), total(skipped=1)).returncode == 1 +def test_a_timeout_an_unreadable_xml_a_failure_and_a_skip_still_fail_the_gate( + program, tmp_path +): + assert ( + run_gate( + program, tmp_path, row("test_a.py", status="TIMEOUT"), total() + ).returncode + == 1 + ) + assert ( + run_gate( + program, tmp_path, row("test_a.py", status="BADXML"), total() + ).returncode + == 1 + ) + assert ( + run_gate( + program, tmp_path, row("test_a.py", failed=1), total(failed=1) + ).returncode + == 1 + ) + assert ( + run_gate(program, tmp_path, row("test_a.py"), total(skipped=1)).returncode == 1 + ) assert run_gate(program, tmp_path, row("test_a.py")).returncode == 1 # no TOTAL row From f6b40441edad17928cee788827e49279ea653e70 Mon Sep 17 00:00:00 2001 From: Taewoon Kim Date: Tue, 10 Nov 2026 16:42:18 +0900 Subject: [PATCH 45/49] Tests: black 26.10.1 layout for test_arrow_vector_column_jvm.py (the pre-commit hook the CI runs; found by running the static gates on the integration branch before dispatch) Co-Authored-By: Claude Sonnet 5.5 --- tests/test_arrow_vector_column_jvm.py | 190 +++++++++++++++++++++----- 1 file changed, 156 insertions(+), 34 deletions(-) diff --git a/tests/test_arrow_vector_column_jvm.py b/tests/test_arrow_vector_column_jvm.py index be7ba0cc..af1ecb14 100644 --- a/tests/test_arrow_vector_column_jvm.py +++ b/tests/test_arrow_vector_column_jvm.py @@ -60,7 +60,12 @@ def _column(matrix, shape): def _table(matrix, shape, start=0): n = matrix.shape[0] - return pa.table({"k": pa.array(np.arange(start, start + n), type=pa.int32()), "embedding": _column(matrix, shape)}) + return pa.table( + { + "k": pa.array(np.arange(start, start + n), type=pa.int32()), + "embedding": _column(matrix, shape), + } + ) def _make(db, name, kind="VERTEX", embedding="ARRAY_OF_FLOATS", index=None): @@ -69,15 +74,23 @@ def _make(db, name, kind="VERTEX", embedding="ARRAY_OF_FLOATS", index=None): if embedding: db.command("sql", f"CREATE PROPERTY {name}.embedding {embedding}") if index: - db.command("sql", f"CREATE INDEX ON {name} (embedding) LSM_VECTOR METADATA {{dimensions: {index}, similarity: 'COSINE'}}") + db.command( + "sql", + f"CREATE INDEX ON {name} (embedding) LSM_VECTOR METADATA {{dimensions: {index}, similarity: 'COSINE'}}", + ) def _count(db, name): - return db.query("sql", f"SELECT count(*) AS c FROM {name}").first().get("c") # nosec B608 - test-owned name + return ( + db.query("sql", f"SELECT count(*) AS c FROM {name}").first().get("c") + ) # nosec B608 - test-owned name def _key(k, vector_bytes): - return int.from_bytes(hashlib.sha256(int(k).to_bytes(8, "little") + vector_bytes).digest()[:16], "little") + return int.from_bytes( + hashlib.sha256(int(k).to_bytes(8, "little") + vector_bytes).digest()[:16], + "little", + ) def _expected_digest(matrix, start=0): @@ -90,8 +103,13 @@ def _expected_digest(matrix, start=0): def _sql_digest(db, name): h, n = 0, 0 - for row in db.query("sql", f"SELECT k, embedding FROM {name}").to_list(): # nosec B608 - test-owned name - h ^= _key(row["k"], np.asarray(row["embedding"], dtype=np.float64).astype(F32).tobytes()) + for row in db.query( + "sql", f"SELECT k, embedding FROM {name}" + ).to_list(): # nosec B608 - test-owned name + h ^= _key( + row["k"], + np.asarray(row["embedding"], dtype=np.float64).astype(F32).tobytes(), + ) n += 1 return n, h @@ -101,7 +119,10 @@ def _api_digest(db, name): h, n = 0, 0 result_set = db.query("sql", f"SELECT FROM {name}") # nosec B608 - test-owned name for result in result_set: - h ^= _key(result.get("k"), np.asarray(result.get("embedding"), dtype=np.float64).astype(F32).tobytes()) + h ^= _key( + result.get("k"), + np.asarray(result.get("embedding"), dtype=np.float64).astype(F32).tobytes(), + ) n += 1 return n, h @@ -134,7 +155,9 @@ def __getattr__(self, name): def java_class(name): cls = real_class(name) - return _Counting(cls) if name == "com.arcadedb.python.FloatVectors$Matrix" else cls + return ( + _Counting(cls) if name == "com.arcadedb.python.FloatVectors$Matrix" else cls + ) monkeypatch.setattr(core, "convert_python_to_java", convert) monkeypatch.setattr(core, "_java_class", java_class) @@ -143,7 +166,9 @@ def java_class(name): @pytest.mark.parametrize("kind", TYPES) @pytest.mark.parametrize("shape", SHAPES) -def test_count_digest_and_reopen_of_a_batched_vector_load(temp_db, temp_db_path_of, shape, kind): +def test_count_digest_and_reopen_of_a_batched_vector_load( + temp_db, temp_db_path_of, shape, kind +): """1,300 rows in batches of 500 (500, 500, 300: the batches are slices of one table): count, SQL digest, API digest, then the same after reopen.""" n, dim = 1_300, 24 matrix = _matrix(n, dim) @@ -178,12 +203,29 @@ def test_a_load_in_several_chunks_of_one_table_keeps_every_row(temp_db): _make(temp_db, "Vec") table = pa.concat_tables([_table(a, "fixed_size", 0), _table(b, "fixed_size", 3)]) assert temp_db.insert_arrow("Vec", table, batch_size=4) == 7 - assert _sql_digest(temp_db, "Vec") == (7, _expected_digest(a, 0) ^ _expected_digest(b, 3)) + assert _sql_digest(temp_db, "Vec") == ( + 7, + _expected_digest(a, 0) ^ _expected_digest(b, 3), + ) @pytest.mark.parametrize("shape", SHAPES) def test_the_special_float32_values_are_stored_bit_for_bit(temp_db, shape): - special = np.array([[-0.0, np.finfo(F32).max, np.finfo(F32).tiny, np.finfo(F32).smallest_subnormal, np.nan, np.inf, -np.inf, 16777217.0]], dtype=F32) + special = np.array( + [ + [ + -0.0, + np.finfo(F32).max, + np.finfo(F32).tiny, + np.finfo(F32).smallest_subnormal, + np.nan, + np.inf, + -np.inf, + 16777217.0, + ] + ], + dtype=F32, + ) _make(temp_db, "Vec") assert temp_db.insert_arrow("Vec", _table(special, shape)) == 1 stored = temp_db.query("sql", "SELECT embedding FROM Vec").first().get("embedding") @@ -191,43 +233,84 @@ def test_the_special_float32_values_are_stored_bit_for_bit(temp_db, shape): got = np.asarray(stored, dtype=np.float64).astype(F32) assert np.isnan(got[4]) keep = [0, 1, 2, 3, 5, 6, 7] - assert got.view(np.uint32)[keep].tolist() == special.view(np.uint32)[0, keep].tolist() - assert got[7] == F32(16777216.0) # 16777217.0 is not a float32; the row holds the float32 that was written + assert ( + got.view(np.uint32)[keep].tolist() == special.view(np.uint32)[0, keep].tolist() + ) + assert got[7] == F32( + 16777216.0 + ) # 16777217.0 is not a float32; the row holds the float32 that was written def test_a_null_row_stays_null_and_its_neighbours_are_exact(temp_db): _make(temp_db, "Vec") - table = pa.table({"k": pa.array([0, 1, 2], pa.int32()), "embedding": pa.array([[1.0, 2.0], None, [3.0, 4.0]], pa.list_(pa.float32(), 2))}) + table = pa.table( + { + "k": pa.array([0, 1, 2], pa.int32()), + "embedding": pa.array( + [[1.0, 2.0], None, [3.0, 4.0]], pa.list_(pa.float32(), 2) + ), + } + ) assert temp_db.insert_arrow("Vec", table) == 3 - rows = {r["k"]: r["embedding"] for r in temp_db.query("sql", "SELECT k, embedding FROM Vec").to_list()} + rows = { + r["k"]: r["embedding"] + for r in temp_db.query("sql", "SELECT k, embedding FROM Vec").to_list() + } assert rows[1] is None assert _bits(rows[0]) == _bits([1.0, 2.0]) and _bits(rows[2]) == _bits([3.0, 4.0]) def test_a_null_element_is_refused_and_the_batch_is_rolled_back(temp_db): _make(temp_db, "Vec") - table = pa.table({"k": pa.array([0, 1], pa.int32()), "embedding": pa.array([[1.0, None, 3.0], [4.0, 5.0, 6.0]], pa.list_(pa.float32()))}) + table = pa.table( + { + "k": pa.array([0, 1], pa.int32()), + "embedding": pa.array( + [[1.0, None, 3.0], [4.0, 5.0, 6.0]], pa.list_(pa.float32()) + ), + } + ) with pytest.raises(ArcadeDBError, match="null element"): temp_db.insert_arrow("Vec", table) assert _count(temp_db, "Vec") == 0 -def test_vectors_of_different_length_are_stored_as_given_without_an_index_and_refused_with_one(temp_db): +def test_vectors_of_different_length_are_stored_as_given_without_an_index_and_refused_with_one( + temp_db, +): _make(temp_db, "Free") - ragged = pa.table({"k": pa.array([0, 1, 2], pa.int32()), "embedding": pa.array([[1.0, 2.0], [1.0, 2.0, 3.0], [4.0]], pa.list_(pa.float32()))}) + ragged = pa.table( + { + "k": pa.array([0, 1, 2], pa.int32()), + "embedding": pa.array( + [[1.0, 2.0], [1.0, 2.0, 3.0], [4.0]], pa.list_(pa.float32()) + ), + } + ) assert temp_db.insert_arrow("Free", ragged) == 3 - rows = {r["k"]: r["embedding"] for r in temp_db.query("sql", "SELECT k, embedding FROM Free").to_list()} + rows = { + r["k"]: r["embedding"] + for r in temp_db.query("sql", "SELECT k, embedding FROM Free").to_list() + } assert {k: len(v) for k, v in rows.items()} == {0: 2, 1: 3, 2: 1} _make(temp_db, "Indexed", index=2) - with pytest.raises(ArcadeDBError, match="Vector dimension does not match index dimension 2: got float.. of length 3"): + with pytest.raises( + ArcadeDBError, + match="Vector dimension does not match index dimension 2: got float.. of length 3", + ): temp_db.insert_arrow("Indexed", ragged.slice(0, 2)) assert _count(temp_db, "Indexed") == 0 -def test_a_fixed_size_vector_of_the_wrong_dimension_is_refused_by_the_vector_index(temp_db): +def test_a_fixed_size_vector_of_the_wrong_dimension_is_refused_by_the_vector_index( + temp_db, +): _make(temp_db, "Indexed", index=2) wrong = _table(_matrix(2, 3, specials=False), "fixed_size") - with pytest.raises(ArcadeDBError, match="Vector dimension does not match index dimension 2: got float.. of length 3"): + with pytest.raises( + ArcadeDBError, + match="Vector dimension does not match index dimension 2: got float.. of length 3", + ): temp_db.insert_arrow("Indexed", wrong) assert _count(temp_db, "Indexed") == 0 right = _matrix(2, 2, specials=False) @@ -238,24 +321,45 @@ def test_a_fixed_size_vector_of_the_wrong_dimension_is_refused_by_the_vector_ind def test_float64_and_integer_lists_keep_what_the_old_route_stored(temp_db): """Not taken by the matrix route: a float64 value past the float range is refused (a cast would store an infinity), the rest rounds.""" _make(temp_db, "Dbl") - ok = pa.table({"k": pa.array([0], pa.int32()), "embedding": pa.array([[0.1, 16777217.0, -2.5]], pa.list_(pa.float64()))}) + ok = pa.table( + { + "k": pa.array([0], pa.int32()), + "embedding": pa.array([[0.1, 16777217.0, -2.5]], pa.list_(pa.float64())), + } + ) assert temp_db.insert_arrow("Dbl", ok) == 1 got = temp_db.query("sql", "SELECT embedding FROM Dbl").first().get("embedding") assert _bits(got) == _bits([0.1, 16777217.0, -2.5]) assert got[1] == 16777216.0 _make(temp_db, "Over") - over = pa.table({"k": pa.array([0], pa.int32()), "embedding": pa.array([[1e39, 1.0]], pa.list_(pa.float64()))}) + over = pa.table( + { + "k": pa.array([0], pa.int32()), + "embedding": pa.array([[1e39, 1.0]], pa.list_(pa.float64())), + } + ) with pytest.raises(ArcadeDBError, match="1.0E39"): temp_db.insert_arrow("Over", over) assert _count(temp_db, "Over") == 0 _make(temp_db, "Ints") - ints = pa.table({"k": pa.array([0], pa.int32()), "embedding": pa.array([[16777217, 2**40, -3]], pa.list_(pa.int64()))}) + ints = pa.table( + { + "k": pa.array([0], pa.int32()), + "embedding": pa.array([[16777217, 2**40, -3]], pa.list_(pa.int64())), + } + ) assert temp_db.insert_arrow("Ints", ints) == 1 - assert temp_db.query("sql", "SELECT embedding FROM Ints").first().get("embedding") == [16777216.0, 1099511627776.0, -3.0] + assert temp_db.query("sql", "SELECT embedding FROM Ints").first().get( + "embedding" + ) == [16777216.0, 1099511627776.0, -3.0] -@pytest.mark.parametrize("embedding", [None, "LIST"], ids=["undeclared", "declared_list"]) -def test_a_property_that_is_not_array_of_floats_keeps_a_list_and_takes_no_matrix(temp_db, crossings, embedding): +@pytest.mark.parametrize( + "embedding", [None, "LIST"], ids=["undeclared", "declared_list"] +) +def test_a_property_that_is_not_array_of_floats_keeps_a_list_and_takes_no_matrix( + temp_db, crossings, embedding +): _make(temp_db, "Free", embedding=embedding) matrix = _matrix(5, 4, specials=False) assert temp_db.insert_arrow("Free", _table(matrix, "fixed_size")) == 5 @@ -265,9 +369,17 @@ def test_a_property_that_is_not_array_of_floats_keeps_a_list_and_takes_no_matrix def test_empty_vectors_are_stored_as_empty_lists(temp_db): _make(temp_db, "Vec") - table = pa.table({"k": pa.array([0, 1], pa.int32()), "embedding": pa.array([[], []], pa.list_(pa.float32()))}) + table = pa.table( + { + "k": pa.array([0, 1], pa.int32()), + "embedding": pa.array([[], []], pa.list_(pa.float32())), + } + ) assert temp_db.insert_arrow("Vec", table) == 2 - assert [r["embedding"] for r in temp_db.query("sql", "SELECT embedding FROM Vec ORDER BY k").to_list()] == [[], []] + assert [ + r["embedding"] + for r in temp_db.query("sql", "SELECT embedding FROM Vec ORDER BY k").to_list() + ] == [[], []] # ---- the three tests that fail on the code before the fix: they count what the load does, never how long it takes @@ -285,18 +397,26 @@ def test_a_vector_column_converts_no_python_value_per_row(temp_db, crossings, sh def test_a_vector_column_crosses_as_one_matrix_per_batch(temp_db, crossings): """1,300 rows in batches of 500: three batches, so three ``FloatVectors.Matrix`` of one flat float[] each (500 x 24, 500 x 24, 300 x 24).""" _make(temp_db, "Vec") - assert temp_db.insert_arrow("Vec", _table(_matrix(1_300, 24), "fixed_size"), batch_size=500) == 1_300 + assert ( + temp_db.insert_arrow( + "Vec", _table(_matrix(1_300, 24), "fixed_size"), batch_size=500 + ) + == 1_300 + ) assert crossings["matrix"] == [500 * 24, 500 * 24, 300 * 24] def test_the_python_heap_of_one_batch_stays_below_twice_the_flat_buffer(temp_db): """10,000 x 96 float32 is a 3.66 MiB buffer; the per-row Python lists of floats were 31.5 MiB at their peak (tracemalloc sees Python and - numpy allocations, not the Java heap and not Arrow's pool). The bound is the buffer twice, nothing about time.""" + numpy allocations, not the Java heap and not Arrow's pool). The bound is the buffer twice, nothing about time. + """ n, dim = 10_000, 96 table = _table(_matrix(n, dim, specials=False), "fixed_size") _make(temp_db, "Vec") _make(temp_db, "Twin") - assert temp_db.insert_arrow("Twin", table.slice(0, 500)) == 500 # first-call imports and caches stay out of the measurement + assert ( + temp_db.insert_arrow("Twin", table.slice(0, 500)) == 500 + ) # first-call imports and caches stay out of the measurement flat = n * dim * 4 tracemalloc.start() try: @@ -306,4 +426,6 @@ def test_the_python_heap_of_one_batch_stays_below_twice_the_flat_buffer(temp_db) peak = tracemalloc.get_traced_memory()[1] - base finally: tracemalloc.stop() - assert peak < 2 * flat, f"{peak} bytes of Python heap for a {flat} byte vector column" + assert ( + peak < 2 * flat + ), f"{peak} bytes of Python heap for a {flat} byte vector column" From 1fa8c5c73126bb28c5c8eab19f43cde16e44742d Mon Sep 17 00:00:00 2001 From: Taewoon Kim Date: Tue, 10 Nov 2026 18:03:52 +0900 Subject: [PATCH 46/49] Tests: the nosec B608 markers sit on the line bandit reports (black had moved them onto the closing line of a wrapped call; the CI bandit job failed on the first dispatch of the final commit) Co-Authored-By: Claude Sonnet 5.5 --- tests/test_arrow_vector_column_jvm.py | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/tests/test_arrow_vector_column_jvm.py b/tests/test_arrow_vector_column_jvm.py index af1ecb14..bb1d5f73 100644 --- a/tests/test_arrow_vector_column_jvm.py +++ b/tests/test_arrow_vector_column_jvm.py @@ -81,9 +81,8 @@ def _make(db, name, kind="VERTEX", embedding="ARRAY_OF_FLOATS", index=None): def _count(db, name): - return ( - db.query("sql", f"SELECT count(*) AS c FROM {name}").first().get("c") - ) # nosec B608 - test-owned name + sql = f"SELECT count(*) AS c FROM {name}" # nosec B608 - test-owned name + return db.query("sql", sql).first().get("c") def _key(k, vector_bytes): @@ -103,9 +102,8 @@ def _expected_digest(matrix, start=0): def _sql_digest(db, name): h, n = 0, 0 - for row in db.query( - "sql", f"SELECT k, embedding FROM {name}" - ).to_list(): # nosec B608 - test-owned name + sql = f"SELECT k, embedding FROM {name}" # nosec B608 - test-owned name + for row in db.query("sql", sql).to_list(): h ^= _key( row["k"], np.asarray(row["embedding"], dtype=np.float64).astype(F32).tobytes(), From 14548090045494d8ce1d1595aea813e84ee7fa67 Mon Sep 17 00:00:00 2001 From: Taewoon Kim Date: Tue, 10 Nov 2026 18:16:16 +0900 Subject: [PATCH 47/49] Test: the vector column heap test no longer uses tracemalloc (it crashed Python 3.11 in CI: SIGSEGV in a JVM thread after tracemalloc.stop()) The full-matrix CI of the post2 integration branch died with "Fatal Python error: Segmentation fault" in test_the_python_heap_of_one_batch_stays_below_twice_the_flat_buffer on Python 3.11 (Linux and Windows); 3.12 to 3.14 passed. Reproduced locally on 3.11.15 and traced to the test method, not to the production path: - fresh processes, faulthandler on, exit codes: insert_arrow of the new route under tracemalloc.start()/stop() around the call: 3 of 12, 1 of 8 and 1 of 2 died with SIGSEGV (139), the single test 1 of 1; the same load with tracemalloc started and never stopped: 0 of 12; the same load with no tracemalloc: 0 of 8 and 0 of 1; the old to_pylist route under tracemalloc start/stop: 0 of 8; the released 26.11.1 GraphBatch route under tracemalloc: 0 of 8 (plain 0 of 8). - no ArcadeDB code in the loop: jpype.JArray(JFloat) of a 10,000 x 96 float32 numpy array plus FloatVectors.Matrix, tracemalloc start/stop around it: 6 of 12 died; never stopped: 0 of 12; no tracemalloc: 0 of 12. The crash is in a thread with no Python frame (a JVM thread), after tracemalloc.stop(), which is a known 3.11 weakness (stop swaps the allocators while a thread outside the GIL still allocates); the other Pythons passed. So the production path (_float32_matrix, _column_to_java) is not at fault and stays. The memory-shape assertion now looks at what the conversion hands over: a spy on dataframe._column_values must see ONE 2-D float32 numpy array of shape (10000, 96) that is a view (not a copy, not per-row lists). On the old dataframe.py it fails (isinstance of the list of lists); the two call-count tests are unchanged. Result: the file passes (23) on 3.11, 3.12 and 3.14, and the new test alone passes in 12 of 12 fresh 3.11 processes. Co-Authored-By: Claude Sonnet 5.5 --- tests/test_arrow_vector_column_jvm.py | 45 +++++++++++++-------------- 1 file changed, 22 insertions(+), 23 deletions(-) diff --git a/tests/test_arrow_vector_column_jvm.py b/tests/test_arrow_vector_column_jvm.py index bb1d5f73..fee897a0 100644 --- a/tests/test_arrow_vector_column_jvm.py +++ b/tests/test_arrow_vector_column_jvm.py @@ -10,7 +10,7 @@ ``_column_to_java`` sends it as one ``float[]`` inside the bridge's ``FloatVectors.Matrix`` (the route a 2-D numpy column of ``insert_columns`` already took); the documents get a row each Java-side. -Three tests fail on the code before (they count the per-row conversions, the matrix crossings and the Python heap of one batch); the others +Three tests fail on the code before (they count the per-row conversions, the matrix crossings and look at what the conversion hands over: one float32 view, not per-row lists); the others pin what the old route stored, so the new one must store the same: count, an order-independent digest of every row's vector bytes read back through SQL and through the result API, both again after close and reopen, the special float32 values, a null row, a null element, vectors of different length, a wrong dimension against a vector index, float64 and integer lists (which are NOT taken: the old route refuses a @@ -20,14 +20,13 @@ """ import hashlib -import tracemalloc import numpy as np import pyarrow as pa import pytest import arcadedb -from arcadedb import core +from arcadedb import core, dataframe from arcadedb.exceptions import ArcadeDBError F32 = np.float32 @@ -404,26 +403,26 @@ def test_a_vector_column_crosses_as_one_matrix_per_batch(temp_db, crossings): assert crossings["matrix"] == [500 * 24, 500 * 24, 300 * 24] -def test_the_python_heap_of_one_batch_stays_below_twice_the_flat_buffer(temp_db): - """10,000 x 96 float32 is a 3.66 MiB buffer; the per-row Python lists of floats were 31.5 MiB at their peak (tracemalloc sees Python and - numpy allocations, not the Java heap and not Arrow's pool). The bound is the buffer twice, nothing about time. +def test_the_vector_column_reaches_the_bridge_as_one_float32_view(temp_db, monkeypatch): + """10,000 x 96 float32 is a 3.66 MiB buffer. ``_column_values`` must hand it over as ONE 2-D float32 numpy array that is a view of Arrow's + value buffer (nothing copied, no Python float per element); the per-row lists of floats it returned before were 31.5 MiB of Python heap at + their peak. The memory shape is asserted on what the conversion returns, not with tracemalloc: starting and stopping tracemalloc around a + JPype float[] conversion crashed the interpreter now and then (SIGSEGV in a JVM thread) on Python 3.11 in CI, on Linux and Windows. """ + seen = {} + real = dataframe._column_values + + def spy(name, chunk, *args, **kwargs): + values = real(name, chunk, *args, **kwargs) + seen[name] = values + return values + + monkeypatch.setattr(dataframe, "_column_values", spy) n, dim = 10_000, 96 - table = _table(_matrix(n, dim, specials=False), "fixed_size") _make(temp_db, "Vec") - _make(temp_db, "Twin") - assert ( - temp_db.insert_arrow("Twin", table.slice(0, 500)) == 500 - ) # first-call imports and caches stay out of the measurement - flat = n * dim * 4 - tracemalloc.start() - try: - tracemalloc.reset_peak() - base = tracemalloc.get_traced_memory()[0] - assert temp_db.insert_arrow("Vec", table, batch_size=n) == n - peak = tracemalloc.get_traced_memory()[1] - base - finally: - tracemalloc.stop() - assert ( - peak < 2 * flat - ), f"{peak} bytes of Python heap for a {flat} byte vector column" + table = _table(_matrix(n, dim, specials=False), "fixed_size") + assert temp_db.insert_arrow("Vec", table, batch_size=n) == n + column = seen["embedding"] + assert isinstance(column, np.ndarray) + assert (column.shape, column.dtype) == ((n, dim), F32) + assert column.base is not None and not column.flags.owndata From 0c67d8530aeeeedb55314a3b1ba327f7efe43ce6 Mon Sep 17 00:00:00 2001 From: Taewoon Kim Date: Tue, 10 Nov 2026 20:32:11 +0900 Subject: [PATCH 48/49] Docs: the source comment of "DataFrames, Arrow and columns" names the modules that read the columns now The page came with main and its first comment line named _native/columns.py (#321) as the source of to_columns, to_arrow, to_dataframe and to_polars. The release line deleted that module with the native column frames (#331), so the merge of the release line leaves the comment pointing at a file that is gone. It now names _columnar.py and _native/colframes.py. The comment does not render; no statement of the page changed. Co-Authored-By: Claude Sonnet 5.5 --- docs/modules/ROOT/pages/how-to/dataframes-and-columns.adoc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/modules/ROOT/pages/how-to/dataframes-and-columns.adoc b/docs/modules/ROOT/pages/how-to/dataframes-and-columns.adoc index 4e70c870..efa17431 100644 --- a/docs/modules/ROOT/pages/how-to/dataframes-and-columns.adoc +++ b/docs/modules/ROOT/pages/how-to/dataframes-and-columns.adoc @@ -1,7 +1,7 @@ = DataFrames, Arrow and columns :description: Load a pandas or Polars frame, an Arrow table or whole columns with insert_dataframe, insert_arrow and insert_columns, what each Arrow type becomes, how nulls and NaN are stored, what a declared property accepts, how many rows are converted at a time, what a failed load leaves, and what to_columns, to_arrow, to_dataframe and to_polars return, including the difference between the two builds. -// Sources: src/arcadedb/core.py (insert_columns, insert_dataframe, insert_arrow), dataframe.py, results.py and _native/columns.py (to_columns, to_arrow, to_dataframe, to_polars). The statements below are pinned by tests/test_dataframe_columns_behavior.py, and for the Arrow types, the declared types, the column checks, the readers and the dates also by tests/test_dataframe_ingest.py, tests/test_insert_columns.py, tests/test_resultset_arrow.py, tests/test_columnar_readers.py and tests/test_resultset.py. +// Sources: src/arcadedb/core.py (insert_columns, insert_dataframe, insert_arrow), dataframe.py, results.py, _columnar.py and _native/colframes.py (to_columns, to_arrow, to_dataframe, to_polars). The statements below are pinned by tests/test_dataframe_columns_behavior.py, and for the Arrow types, the declared types, the column checks, the readers and the dates also by tests/test_dataframe_ingest.py, tests/test_insert_columns.py, tests/test_resultset_arrow.py, tests/test_columnar_readers.py and tests/test_resultset.py. This page covers column-shaped data in and out of a database. Load it with `insert_dataframe` (a pandas or Polars frame), `insert_arrow` (an Arrow table) or `insert_columns` (numpy arrays and lists). Read a result as columns with `to_columns`, `to_arrow`, `to_dataframe` or `to_polars`. For a list of Python dicts use `insert_many`, and for a file use the importer: xref:how-to/bulk-loads.adoc[Bulk loads and commit size] and xref:how-to/import-export.adoc[Importing and exporting data]. The calls here are bindings calls and have no statement in SQL, so the main manual has nothing to say about them. The types that a stored value can have are the engine's: {arcadedb-docs-url}/reference/datatypes[Data types]. From b0703899448c0d91fc1b89170d36a7a9f9066780 Mon Sep 17 00:00:00 2001 From: Taewoon Kim Date: Tue, 10 Nov 2026 21:02:37 +0900 Subject: [PATCH 49/49] Native readers: the docs-pinning test and the page say what the column frames do, not the dict drain they replaced The CI of the merge PR (#351, parity against the installed wheel, linux-x86_64, Python 3.12) failed one test of 3401: tests/test_dataframe_columns_behavior.py::test_how_the_rows_reach_the_columns_differs_between_the_builds, `assert [] == [2]`. The test came with main (#282) and pinned the native path before the column frames (#331): every row read into Python dicts (`_drain_dicts`) and `batch_size` unused. After #331 the native readers (`to_columns`, `to_arrow`, `to_polars`, `to_dataframe`) read the rows still in the cursor as column frames through `lib.call_frame`, at most `batch_size` rows per request (25,000 for `to_dataframe`); only the rows the result already holds in Python (the first reply of 256 rows, rows taken by iteration) are dicts. `_drain_dicts` is still used by `to_list` and `to_json_list`, not by the column readers. On a 600-row result both builds ask the engine for exactly `batch_size` rows per request (native 4 requests of 100 for the 344 rows past the first reply, JVM 7). The test is replaced by one that holds on both builds: it watches the request limit at the seam of the build it runs on (the Java bridge's nextColumnBatch, or lib.call_frame) and asserts the same thing, every request carries `batch_size`, and a smaller `batch_size` makes more requests, with the ids intact. It fails with `batch_size` ignored on both builds (checked by mutating `_read_frames` on the native wheel of the PR run and `_read_columns` on the JVM overlay). The "Differences between the two builds" section of dataframes-and-columns.adoc said the native build reads every row into dicts and does not use `batch_size`; it now says both builds read in requests of at most `batch_size` rows, and the one difference left is the message of a missing pandas. Co-Authored-By: Claude Sonnet 5.5 --- .../pages/how-to/dataframes-and-columns.adoc | 7 +- tests/test_dataframe_columns_behavior.py | 66 ++++++++++++------- 2 files changed, 43 insertions(+), 30 deletions(-) diff --git a/docs/modules/ROOT/pages/how-to/dataframes-and-columns.adoc b/docs/modules/ROOT/pages/how-to/dataframes-and-columns.adoc index efa17431..eb0796a6 100644 --- a/docs/modules/ROOT/pages/how-to/dataframes-and-columns.adoc +++ b/docs/modules/ROOT/pages/how-to/dataframes-and-columns.adoc @@ -337,12 +337,7 @@ A missing library does not fail the same way everywhere. `to_columns()` returns == Differences between the two builds -The calls above, the types and the results are the same on `arcadedb` and `arcadedb-jvm`. Two things differ. The first is how the readers get the rows: - -* On `arcadedb-jvm`, `to_columns()` and `to_arrow()` read the result from the engine in batches of `batch_size` rows, as packed columns, so `batch_size` sets how many rows cross at a time. -* On `arcadedb` (native), they read every row into Python dicts first and build the columns from them. `batch_size` is accepted and not used, and the rows exist as Python objects while the columns are built. - -The second is the message of a missing pandas: the JVM package adds the `uv add` hint to the `ImportError` of `to_dataframe()`, and the native package raises the plain `ImportError`. +The calls above, the types and the results are the same on `arcadedb` and `arcadedb-jvm`, and so is how `to_columns()` and `to_arrow()` get the rows: both read the result from the engine as packed columns, in requests of at most `batch_size` rows, so `batch_size` sets how many rows cross at a time. One thing differs, the message of a missing pandas: the JVM package adds the `uv add` hint to the `ImportError` of `to_dataframe()`, and the native package raises the plain `ImportError`. == Related pages diff --git a/tests/test_dataframe_columns_behavior.py b/tests/test_dataframe_columns_behavior.py index 49b78969..fb602a7f 100644 --- a/tests/test_dataframe_columns_behavior.py +++ b/tests/test_dataframe_columns_behavior.py @@ -7,7 +7,7 @@ and refuses, and what the readers return for nulls, dates, ``batch_size`` and a missing library. ``test_dataframe_ingest.py``, ``test_insert_columns.py``, ``test_resultset_arrow.py`` and ``test_columnar_readers.py`` pin the same calls in more depth; the tests here are the ones the page quotes, so that a change to a sentence of the page has one place to fail. They use no Java object, so they run on the JVM -build and the native build alike (the two branches that differ test the backend they run on). +build and the native build alike (what differs between the builds is tested on the backend it runs on). """ import datetime as dt @@ -523,29 +523,24 @@ def test_the_columns_do_not_depend_on_batch_size(readings): assert columns["k"] == ["a", "b"] and columns["n"][0] == 7.0 -def test_how_the_rows_reach_the_columns_differs_between_the_builds( - readings, monkeypatch -): - """The JVM build reads the result in batches of ``batch_size`` rows from the engine's column buffer. The native build reads - every row into Python dicts first (``_drain_dicts``) and builds the columns from them, so ``batch_size`` is not used. +def _watch_the_engine_requests(monkeypatch): + """The list of row limits that the readers ask the engine for, one entry per request, on the build this runs on. The seam differs + (the Java bridge's ``nextColumnBatch`` on the JVM build, the library's column frame call on the native build), the list means the same. """ - from arcadedb import results - + requests = [] if arcadedb.__backend__ == "native": - resultset = type(readings.query("sql", QUERY)) - drained = [] - real = resultset._drain_dicts - - def spy(self, *args, **kwargs): - rows = real(self, *args, **kwargs) - drained.append(len(rows)) - return rows - - monkeypatch.setattr(resultset, "_drain_dicts", spy) - readings.query("sql", QUERY).to_columns(batch_size=1) - assert drained == [2] # one read of all the rows, whatever batch_size says + from arcadedb._native import lib + + real_frame = lib.call_frame + + def frame(rs_id, max_rows, *rest): + requests.append(int(max_rows)) + return real_frame(rs_id, max_rows, *rest) + + monkeypatch.setattr(lib, "call_frame", frame) else: - batches = [] + from arcadedb import results + real_bridge = results._bridge_class class Spy: @@ -555,7 +550,7 @@ def __init__(self, inner): def nextColumnBatch( self, java_rows, batch_size, *rest ): # noqa: N802 - the Java name - batches.append(int(batch_size)) + requests.append(int(batch_size)) return self._inner.nextColumnBatch(java_rows, batch_size, *rest) def bridge(name): @@ -563,8 +558,31 @@ def bridge(name): return Spy(inner) if name == "ColumnBatcher" else inner monkeypatch.setattr(results, "_bridge_class", bridge) - readings.query("sql", QUERY).to_columns(batch_size=1) - assert batches[:2] == [1, 1] + return requests + + +def test_batch_size_is_how_many_rows_cross_at_a_time_on_both_builds( + temp_db, monkeypatch +): + """``to_columns()`` and ``to_arrow()`` read the result from the engine as packed columns, ``batch_size`` rows per request, on the JVM + build and on the native build alike. The native build keeps in Python the rows that its first reply already holds (256 for a query), + so the result here has more rows than that: the request limit is what is asserted, not how many requests there are. + """ + rows = 600 + temp_db.command("sql", "CREATE DOCUMENT TYPE Wide") + assert temp_db.insert_columns("Wide", {"id": np.arange(rows)}) == rows + query = "SELECT id FROM Wide ORDER BY id" + requests = _watch_the_engine_requests(monkeypatch) + for reader in ("to_columns", "to_arrow"): + counts = {} + for size in (100, 1000): + requests.clear() + result = getattr(temp_db.query("sql", query), reader)(batch_size=size) + assert set(requests) == {size}, (reader, size, requests) + counts[size] = len(requests) + ids = result["id"] if reader == "to_columns" else result["id"].to_pylist() + assert list(ids) == list(range(rows)) + assert counts[100] > counts[1000] >= 1, (reader, counts) def test_an_empty_result_has_no_columns(temp_db):