JVM insert_columns: a column of str and None crosses as one UTF-8 blob plus int32 offsets (refs #324)

src/arcadedb/core.py

@@ -6,6 +6,7 @@
 
 import gc
 import itertools
+import operator
 from collections.abc import Mapping
 from datetime import date
 from decimal import Decimal
@@ -186,13 +187,61 @@ def _java_class(name):
     return cls
 
 
+_NONE_TYPE = type(None)
+_STR_OR_NONE = frozenset((str, _NONE_TYPE))
+
+
+def _string_column(values):
+    """A column of ``str`` and ``None`` as one ``StringColumn`` (the UTF-8 text of all rows in one ``byte[]``, one ``int[]`` of row + 1
+    offsets, and one ``boolean[]`` of the null rows when there is a null), or None when the column is not that: empty, or holding any
+    other type (a mixed column converts per element), or too large for an int offset, or a ``str`` with no UTF-8 form (a lone
+    surrogate), which the per-element path keeps handling as before. The blob crosses through ``bytes`` into ``JArray(JByte)``, a
+    copy; nothing relies on a JPype view of Python memory. The type census, the lengths and the join run in C.
+    """
+    n = len(values)
+    if _np is None or not n:
+        return None
+    kinds = set(map(type, values))
+    if str not in kinds or not kinds <= _STR_OR_NONE:
+        return None
+    mask = None
+    if _NONE_TYPE in kinds:
+        mask = _np.fromiter(
+            map(operator.is_, values, itertools.repeat(None)), dtype=_np.bool_, count=n
+        )
+        values = list(values)  # the caller's sequence is not touched
+        for i in _np.flatnonzero(mask).tolist():
+            values[i] = ""
+    if all(map(str.isascii, values)):
+        blob = "".join(values).encode("ascii")
+        lengths = _np.fromiter(map(len, values), dtype=_np.int64, count=n)
+    else:
+        try:
+            encoded = list(map(str.encode, values))
+        except UnicodeEncodeError:
+            return None
+        blob = b"".join(encoded)
+        lengths = _np.fromiter(map(len, encoded), dtype=_np.int64, count=n)
+        del encoded
+    if len(blob) > _JAVA_INT_MAX:
+        return None
+    offsets = _np.zeros(n + 1, dtype=_np.int32)
+    _np.cumsum(lengths, dtype=_np.int32, out=offsets[1:])
+    return _java_class("com.arcadedb.python.StringColumn")(
+        jpype.JArray(jpype.JByte)(blob),
+        jpype.JArray(jpype.JInt)(offsets),
+        None if mask is None else jpype.JArray(jpype.JBoolean)(mask),
+    )
+
+
 def _column_to_java(name, values):
     """One column of ``Database.insert_columns`` as ``(java array, length)``.
 
     A numpy array of an integer, float, or bool kind crosses as ONE buffer copy
     into a ``long[]``, ``double[]``, or ``boolean[]``; a string or object array
-    and any other sequence convert per element into an ``Object[]`` (a ``str``
-    reuses one Java String per distinct value, ``None`` is a null). A pandas
+    and any other sequence of mixed values convert per element into an ``Object[]`` (a ``str``
+    reuses one Java String per distinct value, ``None`` is a null); a column of nothing but ``str`` and ``None`` crosses instead as one
+    ``StringColumn`` (``_string_column``). A pandas
     ``Series`` is read through ``to_numpy``, with a nullable dtype's ``<NA>``
     as ``None``. A 2-D ``float32`` array is a column of vectors: it crosses as
     one ``float[]`` and each document gets one row of it. Kinds that do not cross natively raise ``TypeError`` rather
@@ -265,6 +314,9 @@ def _column_to_java(name, values):
         values = list(values.astype(object, copy=False))
     elif not isinstance(values, (list, tuple)):
         values = list(values)
+    column = _string_column(values)
+    if column is not None:
+        return column, len(values)
     converted = []
     seen = {}
     for value in values:
@@ -1082,7 +1134,10 @@ def insert_columns(
                 bool as ``boolean[]``, one buffer copy each) or any sequence of
                 Python values (``str``, ``int``, ``float``, ``bool``, ``None``,
                 and the types ``insert_many``'s per-row fallback accepts), which
-                converts per element. ``None`` is a null; a float ``NaN`` in a
+                converts per element, except a sequence of nothing but ``str``
+                and ``None``, which crosses as one UTF-8 buffer with its offsets
+                (a string with no UTF-8 form, a lone surrogate, keeps the per-element
+                path). ``None`` is a null; a float ``NaN`` in a
                 numpy float column is stored as NaN, not as null (use a
                 sequence with ``None`` for nulls). A pandas nullable column
                 (``Int64``, ``string``) converts with its ``<NA>`` as null.
@@ -1211,7 +1266,8 @@ def insert_dataframe(
         takes that table directly and has the type mapping, the declared-type
         rules, and the failure contract). Each column becomes a typed array
         that crosses the bridge once per batch: integers, floats, and booleans
-        without nulls as one buffer copy, anything else per element. A document
+        without nulls as one buffer copy, a string column as one UTF-8 buffer
+        with its offsets, anything else per element. A document
         type gets documents; a vertex type gets vertices. An edge type is
         refused: load the vertices, then buffer the edges with
         ``graph_batch().new_edges()``.

src/java/com/arcadedb/python/DocumentBatcher.java

@@ -99,15 +99,17 @@ public static long insertManyJson(final Database db, final String typeName, fina
 
   /**
    * Columnar insert: the documents are built Java-side from whole columns, so each column crosses the FFI once (a long[] or
-   * double[] copied from a numpy buffer, a boolean[], or an Object[] of Strings, boxed numbers, and nulls) instead of one
+   * double[] copied from a numpy buffer, a boolean[], a StringColumn (one UTF-8 byte[] plus the int[] offsets), or an Object[] of
+   * boxed numbers, Strings of a mixed column, and nulls) instead of one
    * JSON text per batch (bindings issue #150: 2.24x over insertManyJson on the first
    * 2,000,000 TPC-H SF1 line items, nine typed properties, commit every 10,000, same sums and count).
    *
    * Same failure contract as insertManyJson (#7882): batch commits and the failure rollback apply only to a transaction
    * opened here; a caller's own transaction is the caller's to commit or roll back.
    *
-   * @param columns one array per name, each of length n: long[], double[], boolean[] or Object[]; a null element sets
-   *                the property to null, as a JSON null does on the insertManyJson path
+   * @param columns one array per name, each of length n: long[], double[], boolean[], Object[], a FloatVectors.Matrix or a
+   *                StringColumn; a null element (or null row of a StringColumn) sets the property to null, as a JSON null does
+   *                on the insertManyJson path
    */
   public static long insertColumns(final Database db, final String typeName, final String[] names, final Object[] columns,
       final int n, final int commitEvery) {
@@ -195,9 +197,11 @@ else if (col instanceof Object[] a)
         len = a.length;
       else if (col instanceof FloatVectors.Matrix m)
         len = m.rows();
+      else if (col instanceof StringColumn s)
+        len = s.rows();
       else
         throw new IllegalArgumentException("column '" + names[c] + "' is a " + (col == null ? "null" : col.getClass().getName())
-            + ", not a long[], double[], boolean[], Object[] or FloatVectors.Matrix");
+            + ", not a long[], double[], boolean[], Object[], FloatVectors.Matrix or StringColumn");
       if (len != n)
         throw new IllegalArgumentException("column '" + names[c] + "' has " + len + " values for " + n + " rows");
     }
@@ -215,6 +219,8 @@ else if (col instanceof boolean[] a)
         v = a[i];
       else if (col instanceof FloatVectors.Matrix m)
         v = m.row(i);
+      else if (col instanceof StringColumn s)
+        v = s.get(i);
       else
         v = ((Object[]) col)[i];
       doc.set(names[c], v);

src/java/com/arcadedb/python/StringColumn.java

@@ -0,0 +1,61 @@
+/*
+ * Python-bindings bridge: a string column of Database.insert_columns that crosses as one UTF-8 blob plus an int32 offsets array.
+ *
+ * The JPype guidance names one conversion per element as the slow way across the boundary, and an Object[] of Strings costs one
+ * conversion per row (and one Java String per distinct value, all alive at once). Here the whole column is ONE byte[] (the UTF-8
+ * text of every row, back to back), ONE int[] of row + 1 offsets into it, and optionally ONE boolean[] marking the null rows.
+ * Row i is blob[offsets[i], offsets[i + 1]), decoded when the document of that row is built, so the Strings are short-lived.
+ *
+ * The decoded String is the one Python's own str holds: UTF-8 round-trips every code point, so a character beyond the BMP becomes
+ * its surrogate pair, U+0000 stays a NUL and an empty row is the empty String. Python sends no column it cannot encode (a lone
+ * surrogate has no UTF-8 form): that column keeps the per-element path.
+ *
+ * Compiled into arcadedb-python-bridge.jar during the wheel build and consumed by DocumentBatcher.insertColumns() and
+ * insertColumnsParallel().
+ */
+package com.arcadedb.python;
+
+import java.nio.charset.StandardCharsets;
+
+public final class StringColumn {
+  private final byte[]    blob;
+  private final int[]     offsets;
+  private final boolean[] nulls;
+  private final int       rows;
+
+  /**
+   * @param blob    the UTF-8 text of all rows, back to back
+   * @param offsets rows + 1 non-decreasing offsets into {@code blob}, the first 0 and the last at most {@code blob.length}
+   * @param nulls   null for a column without a null, else one flag per row (a flagged row has an empty range and reads as null)
+   */
+  public StringColumn(final byte[] blob, final int[] offsets, final boolean[] nulls) {
+    if (offsets.length < 1)
+      throw new IllegalArgumentException("a string column needs rows + 1 offsets, got none");
+    this.rows = offsets.length - 1;
+    if (offsets[0] != 0)
+      throw new IllegalArgumentException("the first offset of a string column must be 0, got " + offsets[0]);
+    for (int i = 0; i < rows; i++)
+      if (offsets[i + 1] < offsets[i])
+        throw new IllegalArgumentException("the offsets of a string column must not decrease (row " + i + ")");
+    if (offsets[rows] > blob.length)
+      throw new IllegalArgumentException(
+          "the last offset " + offsets[rows] + " of a string column is beyond its " + blob.length + " bytes");
+    if (nulls != null && nulls.length != rows)
+      throw new IllegalArgumentException("a string column has " + nulls.length + " null flags for " + rows + " rows");
+    this.blob = blob;
+    this.offsets = offsets;
+    this.nulls = nulls;
+  }
+
+  public int rows() {
+    return rows;
+  }
+
+  /** The String of row {@code i}, or null for a null row. */
+  public String get(final int i) {
+    if (nulls != null && nulls[i])
+      return null;
+    final int from = offsets[i];
+    return new String(blob, from, offsets[i + 1] - from, StandardCharsets.UTF_8);
+  }
+}

tests/conftest.py

@@ -119,6 +119,7 @@
     "test_query_vector_params_jvm.py": "the java.util objects the JVM build's parameter conversion builds",
     "test_insert_many_gc_pause.py": "the cyclic collector pause is the JVM build's (the native insert_many allocates nothing the collector tracks)",
     "test_graph_batch_tagged_shapes.py": "the JVM wheel's graph batch bulk path (bridge VertexBatcher and EdgeBatcher, the property-matrix fallback)",
+    "test_insert_columns_string_blob.py": "the bridge's StringColumn and the JVM wheel's core._column_to_java (JPype arrays, a spy on the per-element conversion)",
     "test_async_proxy_cache.py": "the JPype proxy classes of the JVM build's async executor",
     "test_read_crossings.py": "crossing counts of the JVM build's read path (JPype proxies of the Java objects and the bridge classes)",
     "test_reader_crossings.py": "crossing counts of the JVM build's record and row readers (JPype proxies of the Java objects and the bridge classes)",

tests/test_insert_columns.py

@@ -79,7 +79,7 @@ def test_nulls_in_a_sequence_column_are_nulls(self, temp_db):
         assert _count(temp_db, "Col", "WHERE label IS NULL") == 2
         assert _count(temp_db, "Col", "WHERE label = 'c'") == 1
 
-    def test_a_numpy_string_array_and_an_object_array_with_none_cross_per_element(
+    def test_a_numpy_string_array_and_an_object_array_with_none_are_stored(
         self, temp_db
     ):
         _make_type(temp_db)