26.11.1.post1 prep: JVM to_json_list float parity (#320) and native columnar reads (#321) on the 26.11.1 tag

src/arcadedb/_native/columns.py

@@ -5,12 +5,16 @@
 
 
 def _names(rows, columns):
+    """The column names: ``columns``, or the keys of the rows in the order they are first seen."""
     if columns is not None:
         return list(columns)
     seen = {}
+    known = seen.keys()
     for row in rows:
-        for k in row:
-            seen.setdefault(k, None)
+        # a row whose keys are all known (nearly every row) costs one C-level subset test, not one Python step per key
+        if not row.keys() <= known:
+            for k in row:
+                seen.setdefault(k, None)
     return list(seen)
 
 
@@ -84,13 +88,13 @@ def to_columns(rows, columns=None):
     out = {}
     for name in _names(rows, columns):
         values = [row.get(name) for row in rows]
-        if _kind(values) == "obj":
+        kind = _kind(values)
+        if kind == "obj":
             vectors = _float_vectors(values)
             if vectors is not None:
                 out[name] = vectors
                 continue
         has_null = any(v is None for v in values)
-        kind = _kind(values)
         if kind == "i":
             out[name] = (
                 np.array([np.nan if v is None else v for v in values], dtype=np.float64)
@@ -124,12 +128,40 @@ def to_columns(rows, columns=None):
     return out
 
 
+def _inferred(values):
+    """``pa.array(values)`` when pyarrow's own inference gives the array the rules of :func:`to_arrow` would give, else None.
+
+    Inference replaces the per-value Python pass of :func:`_kind` for a column of ints, floats, bools, strings, lists and
+    other values it types on its own. Two inferred types differ from the rules: a temporal type (the rules write
+    ``timestamp[ms]`` for a datetime and a date, inference gives ``timestamp[us]`` or ``date32``) and an unsigned integer
+    (an int above int64 is an error under the rules). A column that inference cannot type is also left to the rules,
+    whatever the error (``ArrowInvalid``, ``ArrowTypeError``, and ``OverflowError`` for an int above int64, which the
+    explicit-type call turns into ``ArrowInvalid``), so a failed inference never decides the outcome: the rules have the
+    fallbacks (exact text for an int beyond 2**53 next to floats, for a Decimal wider than decimal256, for a value of mixed
+    types).
+    """
+    import pyarrow as pa
+
+    try:
+        array = pa.array(values)
+    except Exception:  # the rules decide the outcome of a column inference cannot type
+        return None
+    kind = array.type
+    if pa.types.is_temporal(kind) or pa.types.is_unsigned_integer(kind):
+        return None
+    return array
+
+
 def to_arrow(rows, columns=None):
     import pyarrow as pa
 
     arrays = {}
     for name in _names(rows, columns):
         values = [row.get(name) for row in rows]
+        inferred = _inferred(values)
+        if inferred is not None:
+            arrays[name] = inferred
+            continue
         kind = _kind(values)
         if kind == "i":
             arrays[name] = pa.array(values, type=pa.int64())

src/arcadedb/_native/database.py

@@ -248,8 +248,13 @@ def _readable(self) -> bool:
             "with block), so the rows it had not returned are gone. Run the query again to read them."
         )
 
-    def _fill(self) -> bool:
-        """Fetch the next batch into the buffer; False once the cursor is drained."""
+    def _fill(self, into: Optional[list] = None) -> bool:
+        """Fetch the next batch into the buffer; False once the cursor is drained.
+
+        ``into``: a list that takes the rows (without their meta) of the batch when the buffer is empty, so a drain that
+        wants only the rows does not build one tuple per row to take them apart again. The buffer then stays empty, and the
+        answer is the same: whether the batch had a row (or an error is pending).
+        """
         if self._pending_error is not None:
             err, self._pending_error = self._pending_error, None
             self._more = False
@@ -279,10 +284,16 @@ def _fill(self) -> bool:
             raise ArcadeDBError(
                 f"Reading the result set failed: {reply.get('cls')}: {reply.get('error')}"
             )
-        self._buf.extend(zip(reply["rows"], reply["meta"]))
+        if into is not None and not self._buf:
+            batch = reply["rows"]
+            into.extend(batch)
+            has_rows = bool(batch)
+        else:
+            self._buf.extend(zip(reply["rows"], reply["meta"]))
+            has_rows = bool(self._buf)
         self._more = not reply["drained"]
         self._pending_error = reply.get("rerr")
-        return bool(self._buf) or self._pending_error is not None
+        return has_rows or self._pending_error is not None
 
     def _finish(self) -> None:
         self._exhausted = True
@@ -334,11 +345,14 @@ def _drain_dicts(self) -> List[Dict[str, Any]]:
         out: List[Dict[str, Any]] = []
         if not self._readable():
             return out
-        while True:
-            out.extend(p for p, _ in self._buf)
-            self._buf.clear()
-            if not self._fill():
-                break
+        buf = self._buf
+        if (
+            buf
+        ):  # rows read ahead (the first reply, or rows a caller has not taken): the only rows that carry a tuple
+            out.extend([p for p, _ in buf])
+            buf.clear()
+        while self._fill(out):
+            pass
         self._finish()
         return out
 

src/java/com/arcadedb/python/RowBatcher.java

@@ -53,23 +53,36 @@ private RowBatcher() {
    * JSON has no NaN or infinity, and the engine's writers turn them into null. This returns {@code value} with every
    * NaN, Infinity and -Infinity (a Double or Float, or an element of a float[], a double[], a Collection or a Map, at any
    * depth) replaced by the tag {@code {"\u0001": ["f", "NaN"]}} (the text is {@code Double.toString}), which Python turns
-   * back into a float. A value with nothing non-finite is returned as it is, with no copy, so the common row costs one
-   * type test per value.
+   * back into a float. A Float, and each element of a float[], is also widened to a double (a float[] becomes a double[]),
+   * so the JSON text is the exact double and Python reads the same float that to_list() returns (#314). A value that is
+   * neither a Float nor a float[] and has nothing non-finite is returned as it is, with no copy, so the common row costs
+   * one type test per value.
    */
   static Object keepNonFinite(final Object value) {
-    if (value instanceof Double || value instanceof Float) {
-      final double d = ((Number) value).doubleValue();
+    if (value instanceof Double) {
+      final double d = (Double) value;
       return Double.isFinite(d) ? value : tag(d);
     }
+    if (value instanceof Float) {
+      // widened to a Double: the JSON writer prints a Float as the shortest float32 text ("0.1"), which Python reads as a
+      // different double than the exact widening of the float32 that to_list() and the native build return
+      final double d = ((Float) value).doubleValue();
+      return Double.isFinite(d) ? (Object) d : tag(d);
+    }
     if (value instanceof float[] floats) {
-      for (final float f : floats)
-        if (!Float.isFinite(f)) {
-          final List<Object> out = new ArrayList<>(floats.length);
-          for (final float x : floats)
-            out.add(Float.isFinite(x) ? (Object) x : tag(x));
-          return out;
-        }
-      return value;
+      final double[] wide = new double[floats.length];
+      boolean finite = true;
+      for (int i = 0; i < floats.length; i++) {
+        wide[i] = floats[i];
+        if (!Double.isFinite(wide[i]))
+          finite = false;
+      }
+      if (finite)
+        return wide;
+      final List<Object> out = new ArrayList<>(wide.length);
+      for (final double x : wide)
+        out.add(Double.isFinite(x) ? (Object) x : tag(x));
+      return out;
     }
     if (value instanceof double[] doubles) {
       for (final double d : doubles)

tests/conftest.py

@@ -83,6 +83,8 @@
     collect_ignore.append("test_native_point_op_glue.py")
     # the address table of append_samples (arcadedb_n_async_bin), observed on the native library
     collect_ignore.append("test_native_timeseries_binary.py")
+    # arcadedb._native.columns builds the columnar readers' tables from the rows; the file compares it with the rules it replaced (#312)
+    collect_ignore.append("test_native_columns_reference.py")
 if not os.path.isfile(
     os.path.join(os.path.dirname(os.path.dirname(__file__)), "docs", "index.md")
 ):