Native point reads: shorter call path (one-parameter text, flat exec call, first row, record wrap)

bindings/python/docs/api/database.md

@@ -863,6 +863,13 @@ Lookup a record by an indexed key (index-based: O(1) for a hash index, O(log n)
 - Python record wrapper (`Document`, `Vertex`, or `Edge`, matching the underlying
   record type) or `None` if not found
 
+**Raises:** `ArcadeDBError` when no `LSM_TREE` or `HASH` index covers exactly `keys`, in
+this order, on the type or a supertype (`No index has been created on type 'User'
+properties [email]`). The JVM wheel and the native wheel both refuse it: the call never
+scans the type. A `FULL_TEXT` index on the keys is refused with its own text (`No key
+index has been created ...`). An index dropped with `DROP INDEX` stops serving the call
+on the next call.
+
 **Example:**
 
 ```python

bindings/python/src/arcadedb_embedded/_native/codec.py

@@ -66,6 +66,8 @@ def _default(value, _bin=None):
 
 _PLAIN = frozenset((str, int, float, bool, type(None)))
 _dumps = json.dumps
+# the C function json.dumps writes a str with
+_enc_str = json.encoder.encode_basestring_ascii
 _fast = os.environ.get("ARCADEDB_NATIVE_FASTENC", "1") != "0"
 
 
@@ -95,6 +97,16 @@ def encode_params(args, binary=False):
         return ("", None, 0) if binary else ""
     values = args[0] if len(args) == 1 and isinstance(args[0], (list, tuple)) else args
     plain = _PLAIN
+    if _fast and len(values) == 1:
+        # one int or str parameter (the point read, WHERE id = ?): the text is written directly, the bytes json.dumps({"pos": [v]}) gives, without the set of types and the encoder call
+        v = values[0]
+        t = type(v)
+        if t is int:
+            text = '{"pos": [%d]}' % v
+            return (text, None, 0) if binary else text
+        if t is str:
+            text = '{"pos": [%s]}' % _enc_str(v)
+            return (text, None, 0) if binary else text
     if _fast and set(map(type, values)) <= plain:
         # all plain scalars (the per-row INSERT of a load): the cached default encoder, no default hook, no per-call encoder object
         text = _dumps({"pos": values if type(values) is list else list(values)})

bindings/python/src/arcadedb_embedded/_native/database.py

@@ -25,6 +25,22 @@
 _EPOCH_DAY_ORDINAL = date(1970, 1, 1).toordinal()
 _NEXT_BATCH = 2048
 _KIND = {"v": "v", "e": "e", "d": "d"}
+_wrap_record = None
+_LOOKUP_SQL = (
+    {}
+)  # (type name, key names) -> the statement text of lookup_by_key (the text only: it holds for every database)
+_EXACT_KEY_INDEX_TYPES = (
+    "LSM_TREE",
+    "HASH",
+)  # Schema.INDEX_TYPE.isExactKeyLookup(): the kinds that answer "value equals key"
+
+
+def _may_drop_index(language: str, command: str) -> bool:
+    """Whether a command can take an index away (DROP or ALTER in SQL, anything in a script), so lookup_by_key has to look again."""
+    lang = language.lower()
+    return lang == "sqlscript" or (
+        lang == "sql" and command.lstrip()[:5].upper() in ("DROP ", "ALTER")
+    )
 
 
 def _error(prefix: str, reply: dict) -> ArcadeDBError:
@@ -86,12 +102,14 @@ def get_rid(self) -> Optional[str]:
         return self._meta[0] if self._meta else None
 
     def _record(self, kinds):
-        from .records import _wrap
-
+        global _wrap_record
+        # records.py imports this module: looked up on first use, then kept (an import statement per call costs about 0.4 us)
+        if _wrap_record is None:
+            from .records import _wrap as _wrap_record
         self._check_open()
         if not self._meta or self._meta[2] not in kinds:
             return None
-        return _wrap(self._database, self)
+        return _wrap_record(self._database, self)
 
     def get_vertex(self):
         return self._record("v")
@@ -340,6 +358,14 @@ def count(self) -> int:
         return n
 
     def first(self) -> Optional[NativeResult]:
+        buf = self._buf
+        database = self._database
+        if buf and not self._closed and (database is None or not database._closed):
+            # the usual point read: the first row is already here and the result is readable (what __next__ checks): take it and close
+            props, meta = buf.popleft()
+            row = NativeResult(props, meta, database)
+            self.close()
+            return row
         try:
             return next(iter(self))
         except StopIteration:
@@ -386,6 +412,9 @@ def __init__(self, handle: int, owned_by_server: bool = False):
         self._vector_meta: dict = {}
         self._vector_dimensions: dict = {}
         self._vector_id_properties: dict = {}
+        # (type name, key names) -> the statement of lookup_by_key, for the pairs this database has an index for. Per handle, not
+        # module-wide like _LOOKUP_SQL: whether an index exists is a fact about one database (and DROP INDEX clears it: command()).
+        self._lookup_sql: dict = {}
 
     # -- lifecycle
 
@@ -504,12 +533,11 @@ def _exec(
                     count,
                 )
             else:
-                raw, floats = lib.call_bin(
-                    "arcadedb_n_exec",
+                raw, floats = lib.exec_query(
                     self._h,
                     flags,
-                    lib.b(language),
-                    lib.b(command),
+                    lib.bc(language),
+                    lib.bc(command),
                     lib.b(text),
                     _FIRST_BATCH,
                 )
@@ -534,6 +562,8 @@ def query(self, language: str, command: str, *args) -> NativeResultSet:
 
     def command(self, language: str, command: str, *args) -> NativeResultSet:
         """Run a statement that may write."""
+        if self._lookup_sql and _may_drop_index(language, command):
+            self._lookup_sql.clear()  # the next lookup_by_key reads the schema again
         return self._exec(1, "Command failed", language, command, args)
 
     # -- transactions
@@ -649,14 +679,66 @@ def lookup_by_rid(self, rid: str):
 
     def lookup_by_key(self, type_name: str, keys: List[str], values: List[Any]):
         self._check_not_closed()
-        where = " AND ".join(f"`{k}` = ?" for k in keys)
         try:
-            lookup_sql = f"SELECT FROM `{type_name}` WHERE {where} LIMIT 1"  # nosec B608 - backtick-quoted identifiers; values go through ? parameters
+            pair = (type_name, tuple(keys))
+            lookup_sql = self._lookup_sql.get(pair)
+            if lookup_sql is None:
+                # the first lookup of this pair on this handle: the index check the JVM engine makes on every call (see _has_key_index)
+                has_index = self._has_key_index(type_name, pair[1])
+                lookup_sql = _LOOKUP_SQL.get(pair)
+                if lookup_sql is None:
+                    where = " AND ".join(f"`{k}` = ?" for k in keys)
+                    lookup_sql = f"SELECT FROM `{type_name}` WHERE {where} LIMIT 1"  # nosec B608 - backtick-quoted identifiers; values go through ? parameters
+                    if len(_LOOKUP_SQL) < 1024:
+                        _LOOKUP_SQL[pair] = lookup_sql
+                if has_index and len(self._lookup_sql) < 1024:
+                    self._lookup_sql[pair] = lookup_sql
             row = self.query("sql", lookup_sql, list(values)).first()
             return row.get_element() if row is not None else None
         except ArcadeDBError as e:
             raise ArcadeDBError(f"Failed to lookup by key in '{type_name}': {e}") from e
 
+    def _has_key_index(self, type_name: str, keys: tuple) -> bool:
+        """Whether an exact-key index (LSM_TREE or HASH) covers exactly ``keys``, in this order, on the type or one of its supertypes.
+
+        ``Database.lookupByKey`` of the engine refuses a lookup no index covers (``IllegalArgumentException: No index has been created
+        on type ... properties [...]``); the statement lookup_by_key runs here would scan the whole type instead, silently (P-04). One read of
+        ``schema:indexes`` per (type, keys) and handle, remembered by the caller. A subtype lists its own bucket indexes for the indexes
+        it inherits, so the type's own rows are enough. Returns False for a type that does not exist: the statement then fails with the
+        schema error, as before. Raises ArcadeDBError with the engine's wording when the type has no such index.
+        """
+        wanted = list(keys)
+        rows = self.query(
+            "sql", "SELECT FROM schema:indexes WHERE typeName = ?", type_name
+        ).to_list()
+        other = None  # an index on these properties of a kind that cannot look up by exact key (FULL_TEXT, vector, geospatial)
+        for r in rows:
+            props = r.get("properties") or []
+            if props and isinstance(props[0], list):
+                props = props[0]
+            if list(props) != wanted:
+                continue
+            if r.get("indexType") in _EXACT_KEY_INDEX_TYPES:
+                return True
+            if other is None or ("[" in r["name"] and "[" not in other["name"]):
+                other = r
+        if (
+            not rows
+            and not self.query(
+                "sql", "SELECT FROM schema:types WHERE name = ?", type_name
+            ).to_list()
+        ):
+            return False
+        shown = "[" + ", ".join(wanted) + "]"  # Arrays.toString
+        if other is not None:
+            raise ArcadeDBError(
+                f"java.lang.IllegalArgumentException: No key index has been created on type '{type_name}' properties {shown}: "
+                f"index '{other['name']}' is {other.get('indexType')} and cannot look up a record by exact key"
+            )
+        raise ArcadeDBError(
+            f"java.lang.IllegalArgumentException: No index has been created on type '{type_name}' properties {shown}"
+        )
+
     def insert_many(
         self, type_name: str, rows, commit_every: int = 10_000, parallel: bool = False
     ) -> int:

bindings/python/src/arcadedb_embedded/_native/lib.py

@@ -268,6 +268,32 @@ def call_bin(name: str, *args):
     return raw.decode("utf-8"), floats
 
 
+def exec_query(handle, flags, language, command, params, count):
+    """``arcadedb_n_exec`` for a statement whose parameters need no float side buffer: the same ``(text, floats)`` as
+    :func:`call_bin` returns for it. The entry point is looked up once per library, not by name per call, the isolate thread is read
+    without a function call and no argument tuple is built; the reply is read as :func:`call_bin` reads it.
+    """
+    lib = _lib or _load()
+    t = _local.__dict__.get("thread")
+    th = t.handle if t is not None else _thread_handle()
+    ptr = lib.arcadedb_n_exec(th, handle, flags, language, command, params, count)
+    try:
+        raw = ctypes.string_at(ptr)
+        floats = None
+        # a negative start counts from the end and stops at 0 for a short reply
+        i = raw.rfind(b',"bin":', -40)
+        if i >= 0 and raw.endswith(b"}"):
+            n = int(raw[i + 7 : -1])
+            import numpy as np
+
+            off = (len(raw) + 1 + 3) & ~3
+            floats = np.frombuffer(ctypes.string_at(ptr + off, 4 * n), dtype=np.float32)
+            raw = raw[:i] + b"}"
+    finally:
+        lib.arcadedb_free(th, ptr)
+    return raw.decode("utf-8"), floats
+
+
 def call_void(name: str, *args) -> None:
     """Run an entry point that returns nothing (``arcadedb_n_set_upcall``)."""
     lib = _lib or _load()

bindings/python/src/arcadedb_embedded/_native/records.py

@@ -14,11 +14,17 @@
 from . import not_yet
 from .codec import RID
 
+_JAVA_OBJECT = None
+
 
 def _java_object():
-    from .database import _JavaObject
+    """The one stand-in for the Java object of a record (it has no state: any use of it says so); made on first use, as database.py imports this module."""
+    global _JAVA_OBJECT
+    if _JAVA_OBJECT is None:
+        from .database import _JavaObject
 
-    return _JavaObject()
+        _JAVA_OBJECT = _JavaObject()
+    return _JAVA_OBJECT
 
 
 _UNSAVED = "#-1:-1"
@@ -186,14 +192,16 @@ def get_out(self) -> Vertex:
         return self._end("outV")
 
 
+_CLASS_OF = {"v": Vertex, "e": Edge}
+
+
 def _wrap(database, row):
     """The wrapper for a result row that is a record (vertex, edge or document); None for a projection."""
     meta = row._meta
     if not meta:
         return None
     rid, type_name, kind = meta
-    cls = {"v": Vertex, "e": Edge}.get(kind, Document)
-    return cls(database, type_name, rid, row._props)
+    return _CLASS_OF.get(kind, Document)(database, type_name, rid, row._props)
 
 
 def _wrap_dict(database, record: dict, kind: str = "d"):

bindings/python/tests/conftest.py

@@ -29,6 +29,8 @@
     collect_ignore.append("test_native_udf.py")
     # the native library's start-up output (#477) can only be checked with the native library loaded
     collect_ignore.append("test_native_startup_runtime.py")
+    # the helpers of the native point-read path (parameter codec, ctypes entry points, statement-text and schema-read caches)
+    collect_ignore.append("test_native_point_call_native.py")
 if not os.path.isdir(os.path.join(os.path.dirname(os.path.dirname(__file__)), "docs")):
     # the upstream pull request branch has no docs/ directory
     collect_ignore.append("test_docs_examples.py")