create_vertices: Java float[] and float32 arrays cross as one matrix per property on both wheels (closes #337)

docs/modules/ROOT/pages/how-to/vectors.adoc

@@ -110,9 +110,9 @@ print(db.count_type("Chunk"))
 200
 ----
 
-`GraphBatch.create_vertices` rows and the properties of `new_edge` and `new_edges` take the same arrays. On the `arcadedb` (native) package, a non-empty one-dimensional `float32` array in a row or an edge property crosses in a binary side buffer, and the stored values are bit-identical to the ones the text path gave. A list or a `float64` array still crosses as number text, so make the array with `np.asarray(v, dtype=np.float32)` before the load. A laptop A/B on the native wheel (100,000 vertices for each load, `ab.py --noise`, ABBA order, 12 clean pairs for each figure, clean A/A control) gave this wall time of the new path against the old one (95 % interval of the median pair ratio): 0.517 [0.515, 0.520] at 128 dimensions, 0.218 [0.217, 0.219] at 96, and 0.271 [0.270, 0.272] at 64. A load without vectors was unchanged (1.009 [1.003, 1.012]).
+`GraphBatch.create_vertices` rows and the properties of `new_edge` and `new_edges` take the same arrays. On the `arcadedb` (native) package, a non-empty one-dimensional `float32` array in a row or an edge property crosses in a binary side buffer, and the stored values are bit-identical to the ones the text path gave. A list or a `float64` array still crosses as number text, so make the array with `np.asarray(v, dtype=np.float32)` before the load. When a property holds a `float32` array of one length in every row of a `create_vertices` call, the whole column crosses as one float32 matrix in that buffer, with no tag or Python call for each row. A laptop A/B on the native wheel (100,000 vertices for each load, `ab.py --noise`, ABBA order, 12 clean pairs for each figure, clean A/A control) gave this wall time of the new path against the old one (95 % interval of the median pair ratio): 0.517 [0.515, 0.520] at 128 dimensions, 0.218 [0.217, 0.219] at 96, and 0.271 [0.270, 0.272] at 64. A load without vectors was unchanged (1.009 [1.003, 1.012]).
 
-On the `arcadedb-jvm` package, `create_vertices` already sends the values of a property declared `ARRAY_OF_FLOATS` as one binary float32 matrix when every row holds a list or array of numbers of one length. So `float32` arrays in a property declared `ARRAY_OF_FLOATS` take the binary path on both packages.
+On the `arcadedb-jvm` package, `create_vertices` sends the values of a property declared `ARRAY_OF_FLOATS` as one binary float32 matrix when every row holds a list, a numpy array or a Java `float[]` or `double[]` of numbers of one length. A property that is not declared takes the same matrix when every row holds a `float32` numpy array or a Java `float[]` (what `to_java_float_array` returns on this package) of one length, and the stored value is a `float[]`, as before. A list or a `float64` array on an undeclared property keeps its doubles and the older path. So `float32` arrays and `to_java_float_array` values take the binary path on both packages, declared or not.
 
 == Search
 

native/src/main/java/org/humemai/nativelib/NativeExtras.java

@@ -73,6 +73,7 @@
 import org.graalvm.word.WordFactory;
 
 import java.util.ArrayList;
+import java.util.Arrays;
 import java.util.HashMap;
 import java.util.HashSet;
 import java.util.List;
@@ -250,6 +251,45 @@ static Object[] varargs(final Object props) {
     return out;
   }
 
+  /**
+   * The float32 vector columns of create_vertices rows: vecs is {"names": [..], "dims": [..], "offs": [..]} and the side buffer holds, for
+   * column c, one vector after the other (rows * dims[c] floats) from float offset offs[c]. Row i gets the property names[c] as a float[]
+   * of its slice, appended to its property array: the float[] that a {"\u0001":["B",offset,count]} tag per row gave, from one matrix and
+   * no tag per row. A null or empty vecs leaves the rows as they are.
+   */
+  private static void withVectors(final List<Object[]> flats, final Object vecs, final long buf, final long bufFloats) {
+    if (!(vecs instanceof Map<?, ?> v))
+      return;
+    final List<?> names = (List<?>) v.get("names"), dims = (List<?>) v.get("dims"), offs = (List<?>) v.get("offs");
+    final int columns = names.size(), rows = flats.size();
+    if (columns == 0)
+      return;
+    if (buf == 0 || dims.size() != columns || offs.size() != columns)
+      throw new IllegalArgumentException("vector columns without a matching side buffer");
+    final int[] dim = new int[columns];
+    final long[] off = new long[columns];
+    for (int c = 0; c < columns; c++) {
+      dim[c] = ((Number) dims.get(c)).intValue();
+      off[c] = ((Number) offs.get(c)).longValue();
+      if (dim[c] < 1 || off[c] < 0 || off[c] + (long) rows * dim[c] > bufFloats)
+        throw new IllegalArgumentException("vector column outside the side buffer");
+    }
+    final CFloatPointer fp = WordFactory.pointer(buf);
+    for (int i = 0; i < rows; i++) {
+      final Object[] row = flats.get(i);
+      final Object[] grown = Arrays.copyOf(row, row.length + 2 * columns);
+      for (int c = 0; c < columns; c++) {
+        final float[] f = new float[dim[c]];
+        final int base = (int) (off[c] + (long) i * dim[c]);
+        for (int k = 0; k < dim[c]; k++)
+          f[k] = fp.read(base + k);
+        grown[row.length + 2 * c] = names.get(c);
+        grown[row.length + 2 * c + 1] = f;
+      }
+      flats.set(i, grown);
+    }
+  }
+
   static String ridList(final RID[] rids) {
     final StringBuilder sb = new StringBuilder("[");
     for (int i = 0; i < rids.length; i++) {
@@ -488,8 +528,10 @@ else if (opName.equals("create_vertices"))
         final RID[] rids;
         if (!(a.get("rows") instanceof Integer))
           rids = batch.createVertices((String) a.get("type"), intOf(a.get("count"), 0));
-        else
+        else {
+          withVectors(flats, a.get("vecs"), buf, bufFloats);
           rids = batch.createVertices((String) a.get("type"), flats.toArray(new Object[0][]));
+        }
         return ArcadeNative.out("{\"ok\":true,\"rids\":" + ridList(rids) + "}");
       }
       case "new_edge" -> batch.newEdge(new RID((String) a.get("src")), (String) a.get("type"), new RID((String) a.get("dst")),

src/arcadedb/_native/codec.py

@@ -1,5 +1,7 @@
 """Python <-> native JSON: query parameters out, typed rows in."""
 
+import ctypes
+import itertools
 import json
 import os
 from collections.abc import Mapping
@@ -276,6 +278,64 @@ def encode_gb_args(args):
     return text, buf, state[1]
 
 
+_STR_TYPE = frozenset((str,))
+
+
+def _plain_rows(rows):
+    """True when every row is a dict whose keys are str and whose values are plain scalars: :func:`_prep` and the default hook would
+    change nothing, so ``json.dumps`` writes the rows as they are. The census runs in C (a set of types per level), not in Python per row.
+    """
+    if not set(map(type, set(itertools.chain.from_iterable(rows)))) <= _STR_TYPE:
+        return False
+    return (
+        set(map(type, itertools.chain.from_iterable(map(dict.values, rows)))) <= _PLAIN
+    )
+
+
+def _side_buffer(flat):
+    """The float32 array `flat` as the side-buffer argument of ``arcadedb_n_gb_bin``: a ctypes view of its memory (no copy), or its bytes
+    when the array is read-only or not contiguous."""
+    try:
+        return (ctypes.c_char * flat.nbytes).from_buffer(flat)
+    except (TypeError, ValueError):
+        return flat.tobytes()
+
+
+def encode_vertex_args(type_name, rows, names, matrices):
+    """The ``args`` of ``arcadedb_n_gb_bin`` for ``create_vertices`` whose vectors are float32 matrices: ``(text, side buffer, float count)``.
+
+    `rows` hold the properties other than `names`; matrices[c] is the ``(len(rows), dim)`` float32 array of the property names[c], one
+    row of it per vertex. The matrices are the front of the side buffer, back to back, and the text carries their names, dimensions and
+    float offsets (``"vecs"``): no tag and no Python call per row. A value of `rows` that needs the hook (a datetime, a float32 array that
+    did not qualify, a list of floats) is written as in :func:`encode_gb_args`, after the matrices in the buffer.
+    """
+    offs, total = [], 0
+    for matrix in matrices:
+        offs.append(total)
+        total += matrix.size
+    args = {
+        "type": type_name,
+        "rows": rows,
+        "vecs": {
+            "names": names,
+            "dims": [int(m.shape[1]) for m in matrices],
+            "offs": offs,
+        },
+    }
+    if _plain_rows(rows):
+        text, arrays = _dumps(args), list(matrices)
+    else:
+        state = [list(matrices), total]
+        text = json.dumps(_prep(args), default=lambda v: _gb_default(v, state))
+        arrays = state[0]
+    flat = (
+        arrays[0].reshape(-1)
+        if len(arrays) == 1
+        else _np.concatenate([a.reshape(-1) for a in arrays])
+    )
+    return text, _side_buffer(flat), flat.size
+
+
 _INT64_MAX = 2**63 - 1
 _NS_DATETIME = None if _np is None else _np.dtype("datetime64[ns]")
 

src/arcadedb/_native/extras.py

@@ -19,15 +19,26 @@
 from ..callbacks import check_event, check_function
 from ..exceptions import ArcadeDBError
 from ..graph_batch import GraphBatch as _BaseGraphBatch
+from ..type_conversion import take_vector_columns
 from . import interrupts, lib, not_yet
-from .codec import _PLAIN, _default, _np, _prep, encode_gb_args, loads, sample_array
+from .codec import (
+    _PLAIN,
+    _default,
+    _np,
+    _prep,
+    encode_gb_args,
+    encode_vertex_args,
+    loads,
+    sample_array,
+)
 
 # -------------------------------------------------------------------------------------------------------- upcall
 
 # (Java long id, char *json, void **out): the reply goes through *out, not through a return value (see _dispatch)
 _UPCALL_TYPE = ctypes.CFUNCTYPE(
     None, ctypes.c_longlong, ctypes.c_char_p, ctypes.c_void_p
 )
+_DICT_OR_NONE = frozenset((dict, type(None)))
 _callbacks: dict = {}
 _callback_ids = itertools.count(1)
 _callbacks_lock = threading.Lock()
@@ -180,6 +191,30 @@ def _op(self, prefix: str, op: str, **args) -> dict:
             return _call(prefix, "arcadedb_n_gb", self._gb, lib.b(op), lib.b(text))
         return _call(prefix, "arcadedb_n_gb", self._gb, lib.b(op), lib.b(_enc(args)))
 
+    def _create_chunk(self, prefix: str, type_name: str, rows: list) -> list:
+        """One crossing and one transaction for `rows` (dicts): the RIDs of the new vertices.
+
+        A property whose value in every row is a float32 numpy array of one length crosses as one float32 matrix in the side buffer
+        (``take_vector_columns``, ``encode_vertex_args``), with no tag, hook call or Python code per row; every other value is
+        written as :func:`encode_gb_args` writes it.
+        """
+        taken = take_vector_columns(rows, (), undeclared=True) if lib.GB_BIN else None
+        if taken is None:
+            return self._op(prefix, "create_vertices", type=type_name, rows=rows)[
+                "rids"
+            ]
+        names, matrices, rest = taken
+        text, buf, count = encode_vertex_args(type_name, rest, names, matrices)
+        return _call(
+            prefix,
+            "arcadedb_n_gb_bin",
+            self._gb,
+            lib.b("create_vertices"),
+            lib.b(text),
+            buf,
+            count,
+        )["rids"]
+
     def new_vertex(self, type_name: str):
         from .records import Vertex
 
@@ -229,27 +264,14 @@ def _to_java_varargs(self, properties: dict) -> dict:
                 raise TypeError(f"property names must be str, got {key!r}")
         return properties
 
-    def _create_vertices_json_bulk(self, type_name: str, rows):
-        """The bulk path (JSON rows in, RIDs out): the RIDs, or None for rows JSON would change (the caller falls back)."""
-        from ..type_conversion import json_bulk_scalar_ok
-
-        for row in rows:
-            for key, value in (row or {}).items():
-                if type(key) is not str or not json_bulk_scalar_ok(value):
-                    return None
-        return self._op(
-            f"Failed to create batch vertices for type '{type_name}'",
-            "create_vertices",
-            type=type_name,
-            rows=[r or {} for r in rows],
-        )["rids"]
-
     def create_vertices(self, type_name: str, count_or_properties) -> list:
         """The RIDs of the new vertices. Rows go in chunks of the batch's ``commit_every`` (1000 unless graph_batch() got another value;
         0: 10,000), one crossing and one transaction per chunk: chunks that already went in stay stored if a later chunk fails.
 
-        A non-empty one-dimensional ``float32`` numpy array in a row crosses in a binary side buffer; a list or a ``float64`` array
-        crosses as number text (``new_edge`` and ``new_edges`` props follow the same rule).
+        A property whose value in every row is a non-empty one-dimensional ``float32`` numpy array of one length crosses as one
+        float32 matrix in a binary side buffer, one buffer per property and chunk; any other ``float32`` array of a row crosses in the side
+        buffer one array at a time, and a list or a ``float64`` array as number text (``new_edge`` and ``new_edges`` props follow the
+        one-array rule).
         """
         self._check_not_closed()
         db = self._db
@@ -274,23 +296,24 @@ def create_vertices(self, type_name: str, count_or_properties) -> list:
                     ]
                 return rids
             rows = list(count_or_properties)
-            for row in rows:
-                if row is not None and not isinstance(row, dict):
-                    raise TypeError(
-                        "create_vertices() property rows must be dicts or None"
-                    )
+            kinds = set(map(type, rows))
+            if (
+                not kinds <= _DICT_OR_NONE
+            ):  # exact types in C; only an odd row pays the isinstance loop
+                for row in rows:
+                    if row is not None and not isinstance(row, dict):
+                        raise TypeError(
+                            "create_vertices() property rows must be dicts or None"
+                        )
             # chunked, as the JPype wrapper does: one crossing, one JSON text and one reply per chunk, so a 1M-vertex load
             # never holds the text, the parsed rows and the reply of all of it at once (the Serial GC copies what lives)
             rids = []
+            has_none = type(None) in kinds
             for start in range(0, len(rows), chunk):
-                rids.extend(
-                    self._op(
-                        prefix,
-                        "create_vertices",
-                        type=type_name,
-                        rows=[r or {} for r in rows[start : start + chunk]],
-                    )["rids"]
-                )
+                part = rows[start : start + chunk]
+                if has_none:
+                    part = [r or {} for r in part]
+                rids.extend(self._create_chunk(prefix, type_name, part))
             if not rows:
                 rids = self._op(prefix, "create_vertices", type=type_name, rows=[])[
                     "rids"