diff --git a/src/arcadedb/core.py b/src/arcadedb/core.py index c1777814..a1024e91 100644 --- a/src/arcadedb/core.py +++ b/src/arcadedb/core.py @@ -6,6 +6,7 @@ import gc import itertools +import operator from collections.abc import Mapping from datetime import date from decimal import Decimal @@ -186,13 +187,61 @@ def _java_class(name): return cls +_NONE_TYPE = type(None) +_STR_OR_NONE = frozenset((str, _NONE_TYPE)) + + +def _string_column(values): + """A column of ``str`` and ``None`` as one ``StringColumn`` (the UTF-8 text of all rows in one ``byte[]``, one ``int[]`` of row + 1 + offsets, and one ``boolean[]`` of the null rows when there is a null), or None when the column is not that: empty, or holding any + other type (a mixed column converts per element), or too large for an int offset, or a ``str`` with no UTF-8 form (a lone + surrogate), which the per-element path keeps handling as before. The blob crosses through ``bytes`` into ``JArray(JByte)``, a + copy; nothing relies on a JPype view of Python memory. The type census, the lengths and the join run in C. + """ + n = len(values) + if _np is None or not n: + return None + kinds = set(map(type, values)) + if str not in kinds or not kinds <= _STR_OR_NONE: + return None + mask = None + if _NONE_TYPE in kinds: + mask = _np.fromiter( + map(operator.is_, values, itertools.repeat(None)), dtype=_np.bool_, count=n + ) + values = list(values) # the caller's sequence is not touched + for i in _np.flatnonzero(mask).tolist(): + values[i] = "" + if all(map(str.isascii, values)): + blob = "".join(values).encode("ascii") + lengths = _np.fromiter(map(len, values), dtype=_np.int64, count=n) + else: + try: + encoded = list(map(str.encode, values)) + except UnicodeEncodeError: + return None + blob = b"".join(encoded) + lengths = _np.fromiter(map(len, encoded), dtype=_np.int64, count=n) + del encoded + if len(blob) > _JAVA_INT_MAX: + return None + offsets = _np.zeros(n + 1, dtype=_np.int32) + _np.cumsum(lengths, dtype=_np.int32, out=offsets[1:]) + return _java_class("com.arcadedb.python.StringColumn")( + jpype.JArray(jpype.JByte)(blob), + jpype.JArray(jpype.JInt)(offsets), + None if mask is None else jpype.JArray(jpype.JBoolean)(mask), + ) + + def _column_to_java(name, values): """One column of ``Database.insert_columns`` as ``(java array, length)``. A numpy array of an integer, float, or bool kind crosses as ONE buffer copy into a ``long[]``, ``double[]``, or ``boolean[]``; a string or object array - and any other sequence convert per element into an ``Object[]`` (a ``str`` - reuses one Java String per distinct value, ``None`` is a null). A pandas + and any other sequence of mixed values convert per element into an ``Object[]`` (a ``str`` + reuses one Java String per distinct value, ``None`` is a null); a column of nothing but ``str`` and ``None`` crosses instead as one + ``StringColumn`` (``_string_column``). A pandas ``Series`` is read through ``to_numpy``, with a nullable dtype's ```` as ``None``. A 2-D ``float32`` array is a column of vectors: it crosses as one ``float[]`` and each document gets one row of it. Kinds that do not cross natively raise ``TypeError`` rather @@ -265,6 +314,9 @@ def _column_to_java(name, values): values = list(values.astype(object, copy=False)) elif not isinstance(values, (list, tuple)): values = list(values) + column = _string_column(values) + if column is not None: + return column, len(values) converted = [] seen = {} for value in values: @@ -1082,7 +1134,10 @@ def insert_columns( bool as ``boolean[]``, one buffer copy each) or any sequence of Python values (``str``, ``int``, ``float``, ``bool``, ``None``, and the types ``insert_many``'s per-row fallback accepts), which - converts per element. ``None`` is a null; a float ``NaN`` in a + converts per element, except a sequence of nothing but ``str`` + and ``None``, which crosses as one UTF-8 buffer with its offsets + (a string with no UTF-8 form, a lone surrogate, keeps the per-element + path). ``None`` is a null; a float ``NaN`` in a numpy float column is stored as NaN, not as null (use a sequence with ``None`` for nulls). A pandas nullable column (``Int64``, ``string``) converts with its ```` as null. @@ -1211,7 +1266,8 @@ def insert_dataframe( takes that table directly and has the type mapping, the declared-type rules, and the failure contract). Each column becomes a typed array that crosses the bridge once per batch: integers, floats, and booleans - without nulls as one buffer copy, anything else per element. A document + without nulls as one buffer copy, a string column as one UTF-8 buffer + with its offsets, anything else per element. A document type gets documents; a vertex type gets vertices. An edge type is refused: load the vertices, then buffer the edges with ``graph_batch().new_edges()``. diff --git a/src/java/com/arcadedb/python/DocumentBatcher.java b/src/java/com/arcadedb/python/DocumentBatcher.java index bbcae4f8..793e0c26 100644 --- a/src/java/com/arcadedb/python/DocumentBatcher.java +++ b/src/java/com/arcadedb/python/DocumentBatcher.java @@ -99,15 +99,17 @@ public static long insertManyJson(final Database db, final String typeName, fina /** * Columnar insert: the documents are built Java-side from whole columns, so each column crosses the FFI once (a long[] or - * double[] copied from a numpy buffer, a boolean[], or an Object[] of Strings, boxed numbers, and nulls) instead of one + * double[] copied from a numpy buffer, a boolean[], a StringColumn (one UTF-8 byte[] plus the int[] offsets), or an Object[] of + * boxed numbers, Strings of a mixed column, and nulls) instead of one * JSON text per batch (bindings issue #150: 2.24x over insertManyJson on the first * 2,000,000 TPC-H SF1 line items, nine typed properties, commit every 10,000, same sums and count). * * Same failure contract as insertManyJson (#7882): batch commits and the failure rollback apply only to a transaction * opened here; a caller's own transaction is the caller's to commit or roll back. * - * @param columns one array per name, each of length n: long[], double[], boolean[] or Object[]; a null element sets - * the property to null, as a JSON null does on the insertManyJson path + * @param columns one array per name, each of length n: long[], double[], boolean[], Object[], a FloatVectors.Matrix or a + * StringColumn; a null element (or null row of a StringColumn) sets the property to null, as a JSON null does + * on the insertManyJson path */ public static long insertColumns(final Database db, final String typeName, final String[] names, final Object[] columns, final int n, final int commitEvery) { @@ -195,9 +197,11 @@ else if (col instanceof Object[] a) len = a.length; else if (col instanceof FloatVectors.Matrix m) len = m.rows(); + else if (col instanceof StringColumn s) + len = s.rows(); else throw new IllegalArgumentException("column '" + names[c] + "' is a " + (col == null ? "null" : col.getClass().getName()) - + ", not a long[], double[], boolean[], Object[] or FloatVectors.Matrix"); + + ", not a long[], double[], boolean[], Object[], FloatVectors.Matrix or StringColumn"); if (len != n) throw new IllegalArgumentException("column '" + names[c] + "' has " + len + " values for " + n + " rows"); } @@ -215,6 +219,8 @@ else if (col instanceof boolean[] a) v = a[i]; else if (col instanceof FloatVectors.Matrix m) v = m.row(i); + else if (col instanceof StringColumn s) + v = s.get(i); else v = ((Object[]) col)[i]; doc.set(names[c], v); diff --git a/src/java/com/arcadedb/python/StringColumn.java b/src/java/com/arcadedb/python/StringColumn.java new file mode 100644 index 00000000..0946f125 --- /dev/null +++ b/src/java/com/arcadedb/python/StringColumn.java @@ -0,0 +1,61 @@ +/* + * Python-bindings bridge: a string column of Database.insert_columns that crosses as one UTF-8 blob plus an int32 offsets array. + * + * The JPype guidance names one conversion per element as the slow way across the boundary, and an Object[] of Strings costs one + * conversion per row (and one Java String per distinct value, all alive at once). Here the whole column is ONE byte[] (the UTF-8 + * text of every row, back to back), ONE int[] of row + 1 offsets into it, and optionally ONE boolean[] marking the null rows. + * Row i is blob[offsets[i], offsets[i + 1]), decoded when the document of that row is built, so the Strings are short-lived. + * + * The decoded String is the one Python's own str holds: UTF-8 round-trips every code point, so a character beyond the BMP becomes + * its surrogate pair, U+0000 stays a NUL and an empty row is the empty String. Python sends no column it cannot encode (a lone + * surrogate has no UTF-8 form): that column keeps the per-element path. + * + * Compiled into arcadedb-python-bridge.jar during the wheel build and consumed by DocumentBatcher.insertColumns() and + * insertColumnsParallel(). + */ +package com.arcadedb.python; + +import java.nio.charset.StandardCharsets; + +public final class StringColumn { + private final byte[] blob; + private final int[] offsets; + private final boolean[] nulls; + private final int rows; + + /** + * @param blob the UTF-8 text of all rows, back to back + * @param offsets rows + 1 non-decreasing offsets into {@code blob}, the first 0 and the last at most {@code blob.length} + * @param nulls null for a column without a null, else one flag per row (a flagged row has an empty range and reads as null) + */ + public StringColumn(final byte[] blob, final int[] offsets, final boolean[] nulls) { + if (offsets.length < 1) + throw new IllegalArgumentException("a string column needs rows + 1 offsets, got none"); + this.rows = offsets.length - 1; + if (offsets[0] != 0) + throw new IllegalArgumentException("the first offset of a string column must be 0, got " + offsets[0]); + for (int i = 0; i < rows; i++) + if (offsets[i + 1] < offsets[i]) + throw new IllegalArgumentException("the offsets of a string column must not decrease (row " + i + ")"); + if (offsets[rows] > blob.length) + throw new IllegalArgumentException( + "the last offset " + offsets[rows] + " of a string column is beyond its " + blob.length + " bytes"); + if (nulls != null && nulls.length != rows) + throw new IllegalArgumentException("a string column has " + nulls.length + " null flags for " + rows + " rows"); + this.blob = blob; + this.offsets = offsets; + this.nulls = nulls; + } + + public int rows() { + return rows; + } + + /** The String of row {@code i}, or null for a null row. */ + public String get(final int i) { + if (nulls != null && nulls[i]) + return null; + final int from = offsets[i]; + return new String(blob, from, offsets[i + 1] - from, StandardCharsets.UTF_8); + } +} diff --git a/tests/conftest.py b/tests/conftest.py index ec3147c7..bd2a40e6 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -119,6 +119,7 @@ "test_query_vector_params_jvm.py": "the java.util objects the JVM build's parameter conversion builds", "test_insert_many_gc_pause.py": "the cyclic collector pause is the JVM build's (the native insert_many allocates nothing the collector tracks)", "test_graph_batch_tagged_shapes.py": "the JVM wheel's graph batch bulk path (bridge VertexBatcher and EdgeBatcher, the property-matrix fallback)", + "test_insert_columns_string_blob.py": "the bridge's StringColumn and the JVM wheel's core._column_to_java (JPype arrays, a spy on the per-element conversion)", "test_async_proxy_cache.py": "the JPype proxy classes of the JVM build's async executor", "test_read_crossings.py": "crossing counts of the JVM build's read path (JPype proxies of the Java objects and the bridge classes)", "test_reader_crossings.py": "crossing counts of the JVM build's record and row readers (JPype proxies of the Java objects and the bridge classes)", diff --git a/tests/test_insert_columns.py b/tests/test_insert_columns.py index 34ca2ada..482d1228 100644 --- a/tests/test_insert_columns.py +++ b/tests/test_insert_columns.py @@ -79,7 +79,7 @@ def test_nulls_in_a_sequence_column_are_nulls(self, temp_db): assert _count(temp_db, "Col", "WHERE label IS NULL") == 2 assert _count(temp_db, "Col", "WHERE label = 'c'") == 1 - def test_a_numpy_string_array_and_an_object_array_with_none_cross_per_element( + def test_a_numpy_string_array_and_an_object_array_with_none_are_stored( self, temp_db ): _make_type(temp_db) diff --git a/tests/test_insert_columns_string_blob.py b/tests/test_insert_columns_string_blob.py new file mode 100644 index 00000000..40e9b306 --- /dev/null +++ b/tests/test_insert_columns_string_blob.py @@ -0,0 +1,183 @@ +"""The JVM wheel sends a column of str and None as one UTF-8 blob plus int32 offsets, not as one JPype conversion per element. + +`_column_to_java` used to convert every element of a string column (`convert_python_to_java`, then an `Object[]` of Java Strings built +element by element). Now a column of nothing but `str` and `None` is one `byte[]`, one `int[]` of row + 1 offsets and, when there is +a null, one `boolean[]`, wrapped in the bridge's `StringColumn`, which `DocumentBatcher.insertColumns` decodes row by row as it builds +each document. The blob crosses through `bytes` into `JArray(JByte)`, a copy; no JPype view of Python memory is relied on. A mixed +column, an empty column, a column with a lone surrogate (no UTF-8 form) and a column too large for an int offset keep the +per-element path. These tests spy on which path ran (they fail on the code before) and compare the stored table with the one the +per-element path stores for the same columns. +""" + +import jpype +import pytest + +np = pytest.importorskip("numpy") + +from arcadedb import core # noqa: E402 + +TEXTS = [ + "plain", + "", + "emoji \U0001f600 and \U00010000 and \U0010ffff", + "embedded\x00nul", + "\x00", + "line
separator
", + "中文 한글 ퟿ é", + "x" * 5_000, +] + + +def _string_column_class(): + return jpype.JClass("com.arcadedb.python.StringColumn") + + +def _make_type(db, name): + db.command("sql", f"CREATE DOCUMENT TYPE {name}") + db.command("sql", f"CREATE PROPERTY {name}.id LONG") + db.command("sql", f"CREATE PROPERTY {name}.label STRING") + + +def _stored(db, name): + rows = db.query("sql", f"SELECT FROM {name} ORDER BY id").to_list() # nosec B608 + return [(r["id"], r.get("label")) for r in rows] + + +@pytest.mark.parametrize( + "values", + [ + ["a", "bb", ""], + ["a", None, "c"], + ("tuple", "of", "str"), + np.array(["x", "yy"]), + np.array(["x", None, "z"], dtype=object), + TEXTS, + ], + ids=["list", "list_with_none", "tuple", "numpy_str", "object_array", "edge_texts"], +) +def test_a_column_of_str_and_none_is_one_string_column(values): + array, n = core._column_to_java("label", values) + assert n == len(values) + assert isinstance(array, _string_column_class()) + + +@pytest.mark.parametrize( + "values", + [ + ["a", 1], + ["a", b"bytes"], + ["a", 1.5, None], + [1, 2, 3], + [None, None], + [], + ], + ids=[ + "str_int", + "str_bytes", + "str_float", + "ints", + "only_none", + "empty", + ], +) +def test_any_other_column_keeps_the_per_element_path(values): + array, n = core._column_to_java("label", values) + assert n == len(values) + assert not isinstance(array, _string_column_class()) + + +def test_a_string_column_makes_no_per_element_conversion(temp_db, monkeypatch): + _make_type(temp_db, "S") + calls = [] + real = core.convert_python_to_java + monkeypatch.setattr( + core, "convert_python_to_java", lambda v: calls.append(v) or real(v) + ) + n = 2_000 + label = [TEXTS[i % len(TEXTS)] if i % 7 else None for i in range(n)] + temp_db.insert_columns("S", {"id": np.arange(n, dtype=np.int64), "label": label}) + assert calls == [], "a string column was converted element by element" + assert _stored(temp_db, "S") == [(i, label[i]) for i in range(n)] + + +@pytest.mark.parametrize("parallel", [False, True], ids=["sync", "parallel"]) +def test_the_blob_stores_the_table_the_per_element_path_stores( + temp_db, monkeypatch, parallel +): + n = 4_000 + label = [ + None if i % 5 == 3 else TEXTS[i % len(TEXTS)] + str(i % 3) for i in range(n) + ] + cols = {"id": np.arange(n, dtype=np.int64), "label": label} + _make_type(temp_db, "Blob") + _make_type(temp_db, "Ref") + temp_db.insert_columns("Blob", cols, parallel=parallel) + with monkeypatch.context() as mp: + mp.setattr(core, "_string_column", lambda values: None) # the path before + array, _n = core._column_to_java("label", label) + assert not isinstance(array, _string_column_class()) + temp_db.insert_columns("Ref", cols, parallel=parallel) + assert _stored(temp_db, "Blob") == _stored(temp_db, "Ref") + assert _stored(temp_db, "Blob") == [(i, label[i]) for i in range(n)] + + +@pytest.mark.parametrize("position", [0, 500, 999], ids=["first", "middle", "last"]) +def test_a_lone_surrogate_keeps_the_column_off_the_blob_and_is_refused_as_before( + temp_db, position +): + # a lone surrogate has no UTF-8 form: the blob builder gives up (it never raises), and the per-element path, which is what ran + # before, refuses the column (JPype cannot encode the str) before anything is written. The refusal is the contract of every bulk + # entry point: a lone surrogate raises instead of becoming a question mark (test_bulk_json_paths.py). + values = [f"row {i} \U0001f600" if i % 3 else f"row {i}" for i in range(1_000)] + values[position] = "lone \ud800 surrogate" + assert core._string_column(values) is None + with pytest.raises(UnicodeEncodeError): + core._column_to_java("label", values) + _make_type(temp_db, "S") + with pytest.raises(UnicodeEncodeError): + temp_db.insert_columns( + "S", {"id": np.arange(1_000, dtype=np.int64), "label": values} + ) + count = temp_db.query("sql", "SELECT count(*) AS c FROM S").first().get("c") + assert count == 0 + + +def test_a_non_bmp_character_still_takes_the_blob(): + # U+1F600 and U+10000 are one code point each and a surrogate PAIR in Java, but valid UTF-8: the blob carries them + values = ["a", "\U0001f600", "\U00010000\U0010ffff", None, "\ud7ff\ue000"] + assert core._string_column(values) is not None + array, n = core._column_to_java("label", values) + assert n == 5 and isinstance(array, _string_column_class()) + + +@pytest.mark.parametrize( + "blob, offsets, nulls", + [ + (b"ab", [1, 2], None), + (b"ab", [0, 2, 1], None), + (b"ab", [0, 3], None), + (b"ab", [0, 1, 2], [False]), + (b"ab", [], None), + ], + ids=[ + "first_not_zero", + "decreasing", + "past_the_end", + "short_null_flags", + "no_offsets", + ], +) +def test_the_java_side_refuses_a_malformed_column(blob, offsets, nulls): + with pytest.raises(Exception): # noqa: B017 - a Java IllegalArgumentException + _string_column_class()( + jpype.JArray(jpype.JByte)(blob), + jpype.JArray(jpype.JInt)(offsets), + None if nulls is None else jpype.JArray(jpype.JBoolean)(nulls), + ) + + +def test_a_column_beyond_an_int_offset_keeps_the_per_element_path(monkeypatch): + monkeypatch.setattr(core, "_JAVA_INT_MAX", 10) + array, n = core._column_to_java("label", ["0123456", "0123456"]) + assert n == 2 + assert not isinstance(array, _string_column_class()) diff --git a/tests/test_insert_columns_strings.py b/tests/test_insert_columns_strings.py new file mode 100644 index 00000000..fbaf6a23 --- /dev/null +++ b/tests/test_insert_columns_strings.py @@ -0,0 +1,87 @@ +"""insert_columns stores a string column exactly, on both wheels, whatever way it crosses. + +The JVM wheel sends a column of nothing but str and None as one UTF-8 blob plus int32 offsets (test_insert_columns_string_blob.py); +the native wheel reads the column in place. A user must not see a difference: a character beyond the BMP, a NUL, a line separator, +an empty string and a null are stored as they were given, in one column and in a mixed column, in the synchronous and the parallel +mode, from a list, a numpy string array and an object array. +""" + +import pytest + +np = pytest.importorskip("numpy") + +TEXTS = [ + "plain", + "", + "emoji \U0001f600 and \U00010000 and \U0010ffff", + "embedded\x00nul", + "\x00", + "line
separator
", + "中文 한글 ퟿ é", + "x" * 5_000, + 'tab\tnewline\nquote"backslash\\', +] + + +def _make_type(db, name): + db.command("sql", f"CREATE DOCUMENT TYPE {name}") + db.command("sql", f"CREATE PROPERTY {name}.id LONG") + db.command("sql", f"CREATE PROPERTY {name}.label STRING") + db.command("sql", f"CREATE PROPERTY {name}.other STRING") + + +def _stored(db, name): + rows = db.query("sql", f"SELECT FROM {name} ORDER BY id").to_list() # nosec B608 + return [(r["id"], r.get("label"), r.get("other")) for r in rows] + + +def _columns(n): + label = [TEXTS[i % len(TEXTS)] for i in range(n)] + other = [None if i % 4 == 1 else f"o{i}" for i in range(n)] + return {"id": np.arange(n, dtype=np.int64), "label": label, "other": other} + + +@pytest.mark.parametrize("parallel", [False, True], ids=["sync", "parallel"]) +def test_strings_are_stored_as_given(temp_db, parallel): + _make_type(temp_db, "S") + cols = _columns(3_000) + assert temp_db.insert_columns("S", cols, parallel=parallel) == 3_000 + assert _stored(temp_db, "S") == [ + (i, cols["label"][i], cols["other"][i]) for i in range(3_000) + ] + + +@pytest.mark.parametrize("kind", ["numpy_str", "object_array"]) +def test_other_containers_of_strings_are_stored_as_given(temp_db, kind): + _make_type(temp_db, "S") + # a numpy "U" array drops a trailing NUL by itself, so its texts have none + texts = [t for t in TEXTS if "\x00" not in t] + label = [texts[i % len(texts)] for i in range(40)] + column = np.array(label) if kind == "numpy_str" else np.array(label, dtype=object) + expected = label + temp_db.insert_columns("S", {"id": np.arange(40, dtype=np.int64), "label": column}) + assert [row[1] for row in _stored(temp_db, "S")] == expected + + +def test_a_mixed_column_keeps_every_value(temp_db): + _make_type(temp_db, "S") + temp_db.command("sql", "CREATE PROPERTY S.n LONG") + labels = ["a", "emoji \U0001f600", None, "", "z"] + temp_db.insert_columns( + "S", + { + "id": np.arange(5, dtype=np.int64), + "label": labels, + "n": [1, None, 3, 4, 5], + }, + ) + assert [row[1] for row in _stored(temp_db, "S")] == labels + assert _stored(temp_db, "S")[0][0] == 0 + + +def test_a_single_row_column(temp_db): + _make_type(temp_db, "S") + temp_db.insert_columns( + "S", {"id": np.array([10], dtype=np.int64), "label": ["one"]} + ) + assert _stored(temp_db, "S") == [(10, "one", None)]