JVM insert_many: homogeneous chunks cross as typed columns, not as JSON text

docs/modules/ROOT/pages/how-to/bulk-loads.adoc

@@ -57,6 +57,8 @@ Because the two packages have different defaults, the same program can keep diff
 
 On both packages, `insert_many` keeps rows with `datetime`, `date`, `Decimal`, `bytes`, numpy scalar, or pandas `NA` and `NaT` values on the bulk path. On the `arcadedb` (native) package, a row with a value the bulk text cannot carry (NaN or infinity, an integer beyond 64 bits, a date-time or `Decimal` inside a list or dict, or a `datetime.time`) goes in with its own statement, in the order given.
 
+On the `arcadedb-jvm` package, a chunk of 10,000 rows crosses as typed columns, with no JSON text, when every row is a `dict` with the same keys in the same order and each column holds `None` and values of one type: `int` within 64 bits, `float` (NaN and infinity included), `bool`, `str`, `date`, or a `datetime` without a time zone. The stored values are the same as the JSON text stores: a whole number is an `Integer` when it fits an int and a `Long` beyond, and `None` is a null property. Any other chunk keeps the JSON text, as before: a row that lacks a key or lists the keys in another order, a column that mixes types (an `int` beside a `float`), a `Decimal`, `bytes`, numpy or pandas value, a `datetime` with a time zone, a list, tuple or dict, and a float vector for a property declared `ARRAY_OF_FLOATS`. A chunk is judged as a whole, so one such row sends its 10,000 rows down the JSON route. The `arcadedb` (native) package already sends the rows of one shape through its columnar entry.
+
 == Change the commit size
 
 [source,python]

native/src/main/java/org/humemai/nativelib/ArcadeNative.java

@@ -1,9 +1,10 @@
 /*
  * SPDX-License-Identifier: Apache-2.0
  *
- * Phase 2: the C entry points behind the Python native backend (arcadedb/_native). Every function returns a
+ * Phase 2: the C entry points behind the Python native backend (arcadedb/_native). Most functions return a
  * malloc'd UTF-8 JSON text that the caller releases with arcadedb_free: {"ok":true,...} or
  * {"ok":false,"error":msg,"cls":java class,"chain":[[class,msg],..],"retry":"need_retry"|"duplicated_key"|null,"retry_ms":n}.
+ * arcadedb_n_cmd and arcadedb_n_tx return a long (-1 for an error, whose text in that form arcadedb_n_last_error returns).
  * Databases, result sets and servers are long handles. Any thread attached to the isolate may call any function; a
  * transaction belongs to the (Java) thread that began it, as in the embedded API.
  */
@@ -184,9 +185,15 @@ public static CCharPointer info(final IsolateThread thread, final long h) {
     }
   }
 
-  /** op 0 begin, 1 commit, 2 rollback, 3 mark. Every reply carries {"active":b,"depth":n,"commits":n}. */
+  /**
+   * op 0 begin, 1 commit, 2 rollback, 3 mark. Returns the transaction state after the call as one non-negative long: bit 0 is
+   * isTransactionActive, bits 1 to 31 the nesting depth, bits 32 to 62 the low 31 bits of the commit count of the thread's
+   * transaction context (callers compare two counts for equality). -1 on error: the error text is then read with
+   * arcadedb_n_last_error on the same thread, as for arcadedb_n_cmd. No reply text, so no malloc, no arcadedb_free and no JSON
+   * decode in the caller.
+   */
   @CEntryPoint(name = "arcadedb_n_tx")
-  public static CCharPointer tx(final IsolateThread thread, final long h, final int op) {
+  public static long tx(final IsolateThread thread, final long h, final int op) {
     try {
       final Database db = db(h);
       switch (op) {
@@ -196,14 +203,15 @@ public static CCharPointer tx(final IsolateThread thread, final long h, final in
       default -> {
       }
       }
-      return out(txState(db));
+      return txState(db);
     } catch (final Throwable t) {
-      return err(t);
+      LAST_ERROR.set(errorJson(t));
+      return -1;
     }
   }
 
-  private static String txState(final Database db) {
-    int depth = 0;
+  private static long txState(final Database db) {
+    long depth = 0;
     long commits = 0;
     if (db instanceof LocalDatabase local)
       depth = local.getNestedTransactions();
@@ -212,7 +220,7 @@ private static String txState(final Database db) {
       if (ctx != null)
         commits = ctx.getCommitCount();
     }
-    return "{\"ok\":true,\"active\":" + db.isTransactionActive() + ",\"depth\":" + depth + ",\"commits\":" + commits + "}";
+    return (db.isTransactionActive() ? 1L : 0L) | ((depth & 0x7FFFFFFFL) << 1) | ((commits & 0x7FFFFFFFL) << 32);
   }
 
   // ---- statements

src/arcadedb/_native/database.py

@@ -74,12 +74,10 @@ def _call(prefix: str, name: str, *args) -> dict:
     return reply
 
 
-# Every successful ``arcadedb_n_tx`` reply is ``{"ok":true,"active":<bool>,"depth":<n>,"commits":<n>}`` (``ArcadeNative.txState``). A caller
-# that reads none of the state (begin, commit, rollback), or only ``active``, tells success and that bool from the text it already holds
-# instead of decoding a JSON object. An error reply is ``{"ok":false,...}``: it fails the prefix test and is decoded, as before.
-_TX_OK = '{"ok":true,'
-_TX_ACTIVE = '{"ok":true,"active":true,'
-_TX_IDLE = '{"ok":true,"active":false,'
+# ``arcadedb_n_tx`` answers with the transaction state after the call as one int (``ArcadeNative.tx``, no reply text to copy, free or
+# decode): bit 0 is ``active``, bits 1 to 31 the nesting ``depth``, bits 32 to 62 the ``commits`` of the transaction context (its low 31
+# bits: callers compare two of them for equality). Callers read the fields inline: ``state & 1``, ``(state >> 1) & 0x7FFFFFFF``,
+# ``state >> 32``. -1 is an error: its text, the ``{"ok":false,...}`` reply of every other entry point, is ``lib.last_error()``.
 
 
 def _first_row(database, reply: dict):
@@ -793,73 +791,63 @@ def _command_first_row(self, language: str, command: str, *args) -> NativeResult
 
     # -- transactions
 
-    def _tx_text(self, op: int, prefix: str) -> str:
-        """One ``arcadedb_n_tx`` crossing: the reply text of a success (not decoded), an ArcadeDBError for a failure."""
+    def _tx_state(self, op: int, prefix: str) -> int:
+        """One ``arcadedb_n_tx`` crossing, one foreign call: the transaction state after the operation as an int (its layout is
+        described at the top of this module), an ArcadeDBError for a failure.
+        """
         if self._closed:
             raise ArcadeDBError("Database is closed")
         if (op == 1 or op == 2) and lib.CLOSE_BATCH:
             lib.flush_closes()
-        text = lib.call("arcadedb_n_tx", self._h, op)
-        if text.startswith(_TX_OK):
-            return text
-        reply = json.loads(text)
-        if not reply.get("ok"):
-            raise _error(prefix, reply)
-        return text
-
-    def _tx(self, op: int, prefix: str) -> dict:
-        """The decoded state reply: for a caller that reads ``depth`` or ``commits``."""
-        return json.loads(self._tx_text(op, prefix))
+        state = lib.tx(self._h, op)
+        if state < 0:
+            raise _error(prefix, json.loads(lib.last_error()))
+        return state
 
     def begin(self):
-        self._tx_text(0, "Failed to begin transaction")
+        self._tx_state(0, "Failed to begin transaction")
 
-    def _begin_state(self) -> dict:
-        """begin(), and the state right after it: the reply of ``arcadedb_n_tx`` carries ``active``, ``depth`` and ``commits``."""
-        return self._tx(0, "Failed to begin transaction")
+    def _begin_state(self) -> int:
+        """begin(), and the state right after it: the int of ``arcadedb_n_tx`` carries ``active``, ``depth`` and ``commits``."""
+        return self._tx_state(0, "Failed to begin transaction")
 
     def commit(self):
-        self._tx_text(1, "Failed to commit transaction")
+        self._tx_state(1, "Failed to commit transaction")
 
     def rollback(self):
-        self._tx_text(2, "Failed to rollback transaction")
+        self._tx_state(2, "Failed to rollback transaction")
 
-    def _rollback_state(self) -> dict:
-        """rollback(), and the state right after it (the reply carries it, as begin's does): the unwind reads it instead of asking."""
-        return self._tx(2, "Failed to rollback transaction")
+    def _rollback_state(self) -> int:
+        """rollback(), and the state right after it (the int carries it, as begin's does): the unwind reads it instead of asking."""
+        return self._tx_state(2, "Failed to rollback transaction")
 
     def transaction(self) -> TransactionContext:
         return TransactionContext(self)
 
     def is_transaction_active(self) -> bool:
-        text = self._tx_text(3, "Failed to check transaction status")
-        if text.startswith(_TX_ACTIVE):
-            return True
-        if text.startswith(_TX_IDLE):
-            return False
-        return json.loads(text)["active"]
+        return bool(self._tx_state(3, "Failed to check transaction status") & 1)
 
     def _transaction_stack_mark(self):
-        state = self._tx(3, "Failed to read the transaction state")
-        return state["active"], state["depth"]
+        state = self._tx_state(3, "Failed to read the transaction state")
+        return bool(state & 1), (state >> 1) & 0x7FFFFFFF
 
     def _transaction_commit_mark(self):
         """The commit count of the current transaction (the witness that a BATCH boundary published part of its work)."""
-        return self._tx(3, "Failed to read the transaction state")["commits"]
+        return self._tx_state(3, "Failed to read the transaction state") >> 32
 
     def _unwind_opened_since(self, mark):
         """Roll back every level opened since ``mark``, and no other; returns ``(state, first_error)``.
 
-        ``state`` is the transaction state after the last crossing (every rollback reply carries it, so no read sits between two
+        ``state`` is the transaction state int after the last crossing (every rollback carries it, so no read sits between two
         steps), or None when that crossing failed. A level the caller held at the mark is theirs, as in the JPype ``Database``.
         """
         was_active, depth_before = mark
         first_error = None
-        state = self._tx(3, "Failed to read the transaction state")
-        for _ in range(state["depth"] - depth_before):
+        state = self._tx_state(3, "Failed to read the transaction state")
+        for _ in range(((state >> 1) & 0x7FFFFFFF) - depth_before):
             if state is None:
-                state = self._tx(3, "Failed to check transaction status")
-            if not state["active"]:
+                state = self._tx_state(3, "Failed to check transaction status")
+            if not state & 1:
                 break
             try:
                 state = self._rollback_state()
@@ -868,8 +856,8 @@ def _unwind_opened_since(self, mark):
                 first_error = first_error or e
         if not was_active:
             if state is None:
-                state = self._tx(3, "Failed to check transaction status")
-            if state["active"]:
+                state = self._tx_state(3, "Failed to check transaction status")
+            if state & 1:
                 try:
                     state = self._rollback_state()
                 except Exception as e:
@@ -893,7 +881,7 @@ def run_in_transaction(self, fn, retries: int = 12, backoff_s: float = 0.005):
             mark = self._transaction_stack_mark()
             commits_at_start = None
             try:
-                commits_at_start = self._begin_state()["commits"]
+                commits_at_start = self._begin_state() >> 32
                 result = fn()
                 self.commit()
                 return result
@@ -919,7 +907,7 @@ def run_in_transaction(self, fn, retries: int = 12, backoff_s: float = 0.005):
                     # retry, as before); the raise sits outside that try, or its own except would swallow it.
                     try:
                         commits_now = (
-                            state["commits"]
+                            state >> 32
                             if state is not None
                             else self._transaction_commit_mark()
                         )

src/arcadedb/_native/lib.py

@@ -259,6 +259,9 @@ class _Params(ctypes.Structure):
             "arcadedb_n_set_upcall": [P, P],
             "arcadedb_free": [P, P],
         }
+        # the entry points that answer with a number, not with the address of a reply text: a cursor id (arcadedb_n_cmd) and the
+        # transaction state (arcadedb_n_tx); -1 is an error for both, its text is read with arcadedb_n_last_error
+        returns_long = ("arcadedb_n_cmd", "arcadedb_n_tx")
         for name, argtypes in sigs.items():
             fn = getattr(lib, name, None)
             if fn is None:  # a library built before this entry point existed
@@ -272,7 +275,7 @@ class _Params(ctypes.Structure):
                     "arcadedb_n_rs_close_many",
                     "arcadedb_n_set_upcall",
                 )
-                else ctypes.c_longlong if name == "arcadedb_n_cmd" else ctypes.c_void_p
+                else ctypes.c_longlong if name in returns_long else ctypes.c_void_p
             )
         global BINARY_PARAMS, CMD_PATH, CLOSE_BATCH, INSERT_ROWS, EXEC_MANY, NEXT_BIN, GB_BIN, TS_BIN
         if not hasattr(
@@ -488,7 +491,19 @@ def cmd(handle, flags, language, command, params, buf, count, first) -> int:
     )
 
 
+def tx(handle, op) -> int:
+    """``arcadedb_n_tx``: op 0 begin, 1 commit, 2 rollback, 3 mark. Returns the transaction state after the call as one int (bit 0
+    ``active``, bits 1 to 31 ``depth``, bits 32 to 62 ``commits``, as ``ArcadeNative.tx`` writes it), or -1 for an error (read it
+    with :func:`last_error`). One foreign call: no reply text to copy, free or decode.
+    """
+    lib = _lib or _load()
+    t = _local.__dict__.get("thread")
+    th = t.handle if t is not None else _thread_handle()
+    return lib.arcadedb_n_tx(th, handle, op)
+
+
 def last_error() -> str:
+    """The error of the last failed :func:`cmd` or :func:`tx` on this thread, as the JSON text of an error reply."""
     return call("arcadedb_n_last_error")
 
 

src/arcadedb/_native/records.py

@@ -359,8 +359,11 @@ def _save_update(self, db) -> None:
 
         Like every write (and like the JVM build), it needs an active transaction while auto-transaction is off, the engine default:
         outside one it raises ArcadeDBError ("Transaction not begun") before it reads or writes anything. This method opens no
-        transaction of its own."""
-        if not db.is_transaction_active() and not db._auto_transaction:
+        transaction of its own. With auto-transaction on the check cannot refuse, so the state is not asked for (one crossing less).
+        The engine's own refusal cannot stand in for the check with it off: the engine runs the before-update callbacks and the
+        validation before it looks for a transaction, and an UPDATE of a record that is gone raises nothing.
+        """
+        if not db._auto_transaction and not db.is_transaction_active():
             raise ArcadeDBError(
                 "Failed to save record: com.arcadedb.exception.TransactionException: Transaction not begun "
                 "(wrap the write in `with db.transaction():` or call begin() first)"