From d9159f2a026892a0d44a6b4910cbfbe12c5618df Mon Sep 17 00:00:00 2001 From: Taewoon Kim Date: Fri, 6 Nov 2026 15:26:20 +0900 Subject: [PATCH 1/2] Docs: a Python functions and record events page, with tests that pin each statement first (refs #267) Row 8 of the docs audit: register_function and db.on were only in example 28. The existing tests already pin the basics (names, argument counts, return types, a failing function, each event, the veto), so the new behavior tests pin what the page adds: what a function receives from a stored value (datetime, date, Decimal, vertex, edge, document), that a raising function or callback undoes its statement and leaves the transaction going (26.11.1), that a function belongs to the process and an event to one database, which thread runs a function or an event (caller, engine thread for async writes), that a waiting function blocks only its own statement, when an after event fires relative to the commit, and what a callback may do (read, write into the same transaction, fire further events, change the record). A docs-page test runs every Python block in order and checks the printed output. The tests were written from the sources without a local run (the guard hook blocks pytest); CI is the first run on both backends. Co-Authored-By: Claude Sonnet 5.5 --- docs/modules/ROOT/nav.adoc | 1 + .../pages/how-to/databases-and-queries.adoc | 2 +- .../pages/how-to/functions-and-events.adoc | 357 +++++++++++++ .../ROOT/pages/how-to/transactions.adoc | 3 +- docs/modules/ROOT/pages/installation.adoc | 2 +- docs/modules/ROOT/pages/quickstart.adoc | 1 + tests/test_docs_functions_page.py | 62 +++ tests/test_functions_and_events_behavior.py | 467 ++++++++++++++++++ 8 files changed, 892 insertions(+), 3 deletions(-) create mode 100644 docs/modules/ROOT/pages/how-to/functions-and-events.adoc create mode 100644 tests/test_docs_functions_page.py create mode 100644 tests/test_functions_and_events_behavior.py diff --git a/docs/modules/ROOT/nav.adoc b/docs/modules/ROOT/nav.adoc index 0af52aa..a1c4a58 100644 --- a/docs/modules/ROOT/nav.adoc +++ b/docs/modules/ROOT/nav.adoc @@ -11,6 +11,7 @@ ** xref:how-to/bulk-loads.adoc[Bulk loads and commit size] ** xref:how-to/import-export.adoc[Importing and exporting data] ** xref:how-to/dataframes-and-columns.adoc[DataFrames, Arrow and columns] +** xref:how-to/functions-and-events.adoc[Python functions and record events] ** xref:how-to/vectors.adoc[Vectors] * Development ** xref:development/ci.adoc[Continuous integration] diff --git a/docs/modules/ROOT/pages/how-to/databases-and-queries.adoc b/docs/modules/ROOT/pages/how-to/databases-and-queries.adoc index be580e7..71854a3 100644 --- a/docs/modules/ROOT/pages/how-to/databases-and-queries.adoc +++ b/docs/modules/ROOT/pages/how-to/databases-and-queries.adoc @@ -3,7 +3,7 @@ // Sources: bindings docs guide/core/database.md and guide/core/queries.md. -This guide covers the life of a database directory and the two calls that talk to it. Prefer SQL or openCypher through `db.command(...)` and `db.query(...)` for schema, inserts and graph work. The bindings also expose record objects and a `Schema` API; they are on xref:how-to/values-and-records.adoc[Values and records]. The wrappers for the rest of the Java API are not covered here. +This guide covers the life of a database directory and the two calls that talk to it. Prefer SQL or openCypher through `db.command(...)` and `db.query(...)` for schema, inserts and graph work. The bindings also expose record objects and a `Schema` API; they are on xref:how-to/values-and-records.adoc[Values and records]. A Python function that a statement calls, and a callback on record events, are on xref:how-to/functions-and-events.adoc[Python functions and record events]. The wrappers for the rest of the Java API are not covered here. Every script on this page ran against the published `{jvm-pkg-name}` wheel. Run each one in an empty directory. diff --git a/docs/modules/ROOT/pages/how-to/functions-and-events.adoc b/docs/modules/ROOT/pages/how-to/functions-and-events.adoc new file mode 100644 index 0000000..cca92d3 --- /dev/null +++ b/docs/modules/ROOT/pages/how-to/functions-and-events.adoc @@ -0,0 +1,357 @@ += Python functions and record events +:description: Call a Python function from SQL and openCypher with register_function, what crosses in and out, what the caller sees when it raises, which thread runs it, and run Python when a record is created, updated or deleted with db.on. + +// Sources: src/arcadedb/callbacks.py and core.py (register_function, unregister_function, on), _native/extras.py (the native build). Every statement below is pinned by tests/test_functions_and_events_behavior.py. The basics are pinned by tests/test_python_callbacks.py (names, argument counts, replace, return types, record events, veto), the native upcall by tests/test_native_udf.py, and a real Ctrl-C by tests/test_python_function_interrupt.py and tests/test_transaction_context_interrupt.py. + +Use `register_function` when a statement needs something that SQL and openCypher cannot compute, such as a slug, a hash or a call into your own library. Use `db.on` when Python code must run each time a record is written, for an audit trail or a rule that can refuse a write. Both are part of the Python bindings, not of the engine, and both work the same on `arcadedb` and `arcadedb-jvm`, with one difference that is called out below. + +The short version: + +* `db.register_function("slugify", fn)` makes `slugify(...)` a function of SQL and openCypher. The function runs once per row, inside the statement. +* A function belongs to the process, not to one database. It lasts until `unregister_function` or until the `Database` that registered it closes. +* If your function raises, the statement fails with an `ArcadeDBError` that names the function and the Python exception. Inside a transaction the statement is undone and the transaction goes on. +* `db.on("after_create", fn, "Person")` calls `fn(record)` when a `Person` is created. An event belongs to one database. +* A `before_*` callback that returns `False` refuses the write. An `after_*` callback runs before the commit, so a rollback does not take its call back. + +The statements stay SQL and openCypher, as on xref:how-to/databases-and-queries.adoc[Databases and queries]. What a transaction guarantees is in the main manual: {arcadedb-docs-url}/concepts/transactions[Transactions]. + +The blocks on this page run one after the other against one empty database, and a test in the repository runs them that way (`tests/test_docs_functions_page.py`). Each block that prints something is followed by its output. Every block assumes that `db` is an open database. The first block creates the types that the rest use. + +[source,python] +---- +import re + +from arcadedb import ArcadeDBError + +db.command("sql", "CREATE DOCUMENT TYPE Person") +db.command("sql", "CREATE DOCUMENT TYPE Audit") +db.command("sql", "CREATE VERTEX TYPE City") +with db.transaction(): + for name in ("Ann Lee", "Bob Ray", None): + db.command("sql", "INSERT INTO Person SET name = ?", name) + db.command("sql", "CREATE VERTEX City SET name = 'New York'") +---- + +== Python functions in statements + +`register_function(name, fn)` takes a name and a callable. After that, a statement calls the function by name, in a projection or in a `WHERE` clause. SQL passes `NULL` as `None`, so a function that may see a `NULL` has to check for it. + +openCypher accepts a function name only when it has a namespace, such as `py.slugify`. Register the function under that name for openCypher. A bare name such as `slugify` is refused with `Unknown function`. Register the same callable under both names when you need both languages. Names are case-insensitive in SQL. + +[source,python] +---- +def slugify(text): + if text is None: + return None + return re.sub(r"[^a-z0-9]+", "-", text.lower()).strip("-") + + +db.register_function("slugify", slugify) +db.register_function("py.slugify", slugify) + +rows = db.query("sql", "SELECT name, slugify(name) AS slug FROM Person").to_list() +for row in sorted(rows, key=lambda r: r["name"] or ""): + print(row["name"], row["slug"]) + +hits = db.query("sql", "SELECT name FROM Person WHERE slugify(name) = 'bob-ray'") +print([row["name"] for row in hits.to_list()]) + +cypher = "MATCH (c:City) RETURN py.slugify(c.name) AS slug" +print(db.query("opencypher", cypher).to_list()) +---- + +[source,text] +---- +None None +Ann Lee ann-lee +Bob Ray bob-ray +['Bob Ray'] +[{'slug': 'new-york'}] +---- + +The function is called once for each row that the statement reads. Two things that `register_function` checks for you: the name must be letters, digits and underscores (dots are allowed between parts), and a name that is already a function of the engine, such as `abs` or `count`, is refused. The number of arguments comes from the signature of your function. Pass `min_args` and `max_args` to set it by hand. A function cannot be an aggregate: it returns one value per call, not one per group. + +=== What crosses + +An argument reaches your function as a Python value, and the return value is converted back. The conversion is the one on xref:how-to/values-and-records.adoc[Values and records]. + +[cols="2,3"] +|=== +| Argument in the statement | What your function gets + +| a number, a string, `true`, `null` +| `int` or `float`, `str`, `bool`, `None` + +| a list or a map +| `list` or `dict` + +| a stored `DATETIME` or `DATE` property +| `datetime` without a time zone, or `date` + +| a stored `DECIMAL` property +| `Decimal` + +| a record, such as `@this` +| a `Document`, `Vertex` or `Edge`, the same objects as on xref:how-to/values-and-records.adoc#records[Values and records] +|=== + +You can return `None`, a number, a string, a `bool`, a `Decimal`, a `datetime`, a `list`, a `tuple` (it comes back as a list) or a `dict`. A value that cannot be stored, such as an `int` beyond 64 bits or a plain `object()`, fails the statement with the Python error name in the message. + +[source,python] +---- +from datetime import date, datetime +from decimal import Decimal + +db.register_function("kinds", lambda *args: [type(a).__name__ for a in args]) +row = db.query( + "sql", "SELECT kinds(1, 2.5, 'a', true, null, [1, 2], {'k': 'v'}) AS k" +).first() +print(row.get("k")) + +db.command("sql", "CREATE DOCUMENT TYPE Booking") +db.command("sql", "CREATE PROPERTY Booking.at DATETIME_MICROS") +db.command("sql", "CREATE PROPERTY Booking.day DATE") +db.command("sql", "CREATE PROPERTY Booking.price DECIMAL") +with db.transaction(): + db.command( + "sql", + "INSERT INTO Booking SET at = ?, day = ?, price = ?", + datetime(2026, 10, 1, 12, 34, 56, 789123), + date(2026, 10, 1), + Decimal("19.99"), + ) +row = db.query("sql", "SELECT kinds(at, day, price) AS k FROM Booking").first() +print(row.get("k")) +---- + +[source,text] +---- +['int', 'float', 'str', 'bool', 'NoneType', 'list', 'dict'] +['datetime', 'date', 'Decimal'] +---- + +A function can read the database it runs in with `db.query`. + +[[raises]] +=== When the function raises + +Any exception that leaves your function, `SystemExit` included, fails the statement with an `ArcadeDBError`. The message names the function and the Python exception, as `ValueError: cannot handle Ann Lee`. The database and the other functions work as before. + +[source,python] +---- +def explode(value): + raise ValueError(f"cannot handle {value}") + + +db.register_function("explode", explode) +try: + db.query("sql", "SELECT explode(name) AS x FROM Person LIMIT 1").to_list() +except ArcadeDBError as error: + print("explode" in str(error), "ValueError: cannot handle" in str(error)) +---- + +[source,text] +---- +True True +---- + +Inside a transaction, the failed statement is undone and the transaction goes on. This is the engine behavior of 26.11.1 and later, described on xref:how-to/transactions.adoc#failed-statement[Transactions]. An `UPDATE` that changed rows before the function raised leaves none of them changed. + +[source,python] +---- +def refuse_bob(name): + if name == "Bob Ray": + raise ValueError("no Bob") + return name.upper() + + +db.register_function("refuse_bob", refuse_bob) +with db.transaction(): + db.command("sql", "INSERT INTO Audit SET note = 'before'") + try: + db.command( + "sql", "UPDATE Person SET shout = refuse_bob(name) WHERE name IS NOT NULL" + ) + except ArcadeDBError: + print("refused, still active:", db.is_transaction_active()) + db.command("sql", "INSERT INTO Audit SET note = 'after'") + +changed = db.query("sql", "SELECT FROM Person WHERE shout IS NOT NULL").to_list() +notes = db.query("sql", "SELECT note FROM Audit").to_list() +print(len(changed), sorted(row["note"] for row in notes)) +---- + +[source,text] +---- +refused, still active: True +0 ['after', 'before'] +---- + +A `KeyboardInterrupt` raised in your function is the one exception that does not become a plain `ArcadeDBError`. The statement stops, and the caller gets the `KeyboardInterrupt`, with the `ArcadeDBError` as its `+__cause__+`. A real Ctrl-C during such a statement reaches the caller the same way, wherever the signal lands. The tests that send the signal do not run on Windows, which cannot signal its own process like this. + +=== Scope and lifetime + +The engine keeps functions for the whole process. A function that one `Database` registered is visible to every open database in the process, and a second `register_function` with the same name from another database is refused unless you pass `replace=True`. Only the `Database` that registered a function can call `unregister_function` for it. When that `Database` closes, the function goes with it, and a statement that calls it fails from then on. A statement that names an unregistered function fails with an unknown-function error that names it. + +Register your functions once, at start-up, before other threads run statements. + +=== Threads + +A statement that you run in a `threading.Thread` calls your function on that thread. A write that you send through `db.async_executor()` runs in an engine thread, so a function that the statement calls runs there too, not on the thread that queued the write. + +A function that waits, for a lock, a socket or an `Event`, blocks only its own statement. Other threads go on running statements on the same `Database`. Write the function as if two threads could call it at once: the GIL does not protect it on a free-threaded build of Python, and a function that waits gives up the GIL anyway. + +== Record events + +`db.on(event, fn, type_name=None)` registers `fn`. The engine calls `fn(record)` with the record as a `Document`, `Vertex` or `Edge` when a record of that kind changes. It returns an `EventHandle`; `handle.remove()` stops the callback, and the handle is a context manager that removes the callback when the block ends. + +[cols="2,3"] +|=== +| Event | When it fires + +| `before_create`, `before_update`, `before_delete` +| Before the engine writes the record. The callback can refuse the write. + +| `after_create`, `after_update`, `after_delete` +| Right after the write, inside the transaction and before the commit. +|=== + +A `before_update` callback gets the record with its new values. An `after_delete` callback gets the record that was deleted, and its properties are readable. A statement, the record API (`save()`, `delete()`) and an openCypher write fire the same events. An edge fires the events of its own type, with an `Edge`. + +The block registers one callback for three events on `Person`, then creates, updates and deletes a person: + +[source,python] +---- +audit = [] + + +def record_event(event): + def callback(record): + audit.append((event, record.get_type_name(), record.get("name"))) + + return callback + + +handles = [ + db.on(event, record_event(event), "Person") + for event in ("after_create", "after_update", "after_delete") +] +with db.transaction(): + db.command("sql", "INSERT INTO Person SET name = 'Cy Dunn'") +with db.transaction(): + db.command("sql", "UPDATE Person SET name = 'Cy Dunne' WHERE name = 'Cy Dunn'") +with db.transaction(): + db.command("sql", "DELETE FROM Person WHERE name = 'Cy Dunne'") +for entry in audit: + print(entry) +for handle in handles: + handle.remove() +---- + +[source,text] +---- +('after_create', 'Person', 'Cy Dunn') +('after_update', 'Person', 'Cy Dunne') +('after_delete', 'Person', 'Cy Dunne') +---- + +=== Type, database and scope + +`type_name` limits the callback to records of exactly that type. The records of a subtype do not fire it. Leave `type_name` out and the callback fires for every record of the database. An event belongs to the database object that registered it. Another open database does not fire it, and it is gone when the database closes, so register it again after you reopen. `type_name` must exist, or `db.on` raises `ArcadeDBError`. + +=== Events and the transaction + +An `after_*` event fires before the commit. Another thread cannot see the record yet, and when the transaction rolls back, the callback has already run. If the callback sent a message or called a service, a rollback does not take that back. Do that work after the commit instead, from the code that owns the `with db.transaction():` block. + +[source,python] +---- +db.command("sql", "CREATE DOCUMENT TYPE Ticket") +log = [] +log_handles = [ + db.on(event, lambda record, event=event: log.append(event), "Ticket") + for event in ("before_create", "after_create") +] +try: + with db.transaction(): + db.command("sql", "INSERT INTO Ticket SET name = 'x'") + print("inside:", log) + raise RuntimeError("stop") +except RuntimeError: + pass +print("after the rollback:", log, db.count_type("Ticket")) +for handle in log_handles: + handle.remove() +---- + +[source,text] +---- +inside: ['before_create', 'after_create'] +after the rollback: ['before_create', 'after_create'] 0 +---- + +=== Refusing a write + +A `before_*` callback that returns `False` refuses the write. The record is not written, and a statement or `save()` skips it without an error. Only the value `False` refuses: `None`, `True`, `0` and an empty string let the write go on. The result of an `after_*` callback is ignored. + +[source,python] +---- +veto = db.on( + "before_create", lambda record: record.get("name") != "blocked", "Ticket" +) +with db.transaction(): + db.command("sql", "INSERT INTO Ticket SET name = 'ok'") + db.command("sql", "INSERT INTO Ticket SET name = 'blocked'") +rows = db.query("sql", "SELECT name FROM Ticket").to_list() +print(sorted(row["name"] for row in rows)) +veto.remove() +---- + +[source,text] +---- +['ok'] +---- + +=== When a callback raises + +An exception in a callback fails the statement or the `save()` that fired the event, with an `ArcadeDBError` that names the event and the Python exception. As for a function, a statement that fails this way is undone and the transaction goes on, so the other statements of the transaction commit. If you let the error leave the `with` block, the whole transaction rolls back. A `KeyboardInterrupt` in a callback reaches the caller as a `KeyboardInterrupt` chained from that `ArcadeDBError`. + +=== What a callback may do + +A callback runs inside the operation that fired it, on the thread that runs it. For a write that goes through `db.async_executor()`, that is an engine thread. + +* *Read.* A callback can run `db.query`. A `before_create` callback does not see the new record in a query. An `after_create` callback does. +* *Write.* A callback can run `db.command`. The write joins the transaction of the operation that fired the event: it commits with it and rolls back with it. +* *Fire more events.* The write of a callback fires events too. A callback registered without `type_name` fires for its own writes. Guard against that, or it never ends. +* *Change the record.* Do not rely on it. On `arcadedb-jvm` a `before_create` callback can call `record.set(...)`, and the value is stored. On `arcadedb` the change is lost. Put the value in the statement instead. +* *Wait.* Keep it quick, and do not wait in it for another thread's transaction. + +[source,python] +---- +db.command("sql", "CREATE DOCUMENT TYPE Journal") + + +def note_creation(record): + if record.get_type_name() != "Journal": + note = "created " + record.get_type_name() + db.command("sql", "INSERT INTO Journal SET note = ?", note) + + +with db.on("after_create", note_creation): + with db.transaction(): + db.command("sql", "INSERT INTO Ticket SET name = 'third'") +rows = db.query("sql", "SELECT note FROM Journal").to_list() +print(sorted(row["note"] for row in rows)) +---- + +[source,text] +---- +['created Ticket'] +---- + +The guard `!= "Journal"` is the one that stops the callback from answering its own write. Without it, every `Journal` row would fire the callback again. + +== Where to go next + +* xref:how-to/transactions.adoc[Transactions]: what a failed statement and a rollback do. +* xref:how-to/values-and-records.adoc[Values and records]: the conversion of values and the record objects that a function or a callback receives. +* `examples/28_python_functions_and_record_events.py` in the repository runs both features end to end and measures the cost of one call. diff --git a/docs/modules/ROOT/pages/how-to/transactions.adoc b/docs/modules/ROOT/pages/how-to/transactions.adoc index e072ad9..3286ed0 100644 --- a/docs/modules/ROOT/pages/how-to/transactions.adoc +++ b/docs/modules/ROOT/pages/how-to/transactions.adoc @@ -158,6 +158,7 @@ after the commit, other thread: 4 Share one `Database` between threads, as on xref:how-to/concurrency.adoc#threads[Concurrency, processes and crashes]. That page also covers what happens when two transactions change the same record. +[[failed-statement]] == A failed statement inside a transaction A statement that fails inside your transaction is undone on its own, and the transaction goes on. This is the behavior of ArcadeDB 26.11.1 and later. Before it, a statement that had written part of its rows left them in the transaction, and the transaction could no longer be committed. @@ -184,7 +185,7 @@ still active: True The second statement writes `frank` once, and then the unique index refuses the second `frank`. The first `frank` is gone with it, while `erin` from the statement before and `grace` from the statement after are committed. The engine keeps a journal of what each statement changes in order to undo it. When a statement changes more entries than `arcadedb.txStatementSavepointMaxEntries` (100,000 by default), the engine does not undo it, and the transaction can no longer be committed: roll it back. The setting is listed in the main manual: {arcadedb-docs-url}/reference/settings[Settings]. -A failed statement is one thing, and an exception that you let leave the `with` block is another: that one rolls the whole transaction back, as above. +A failed statement is one thing, and an exception that you let leave the `with` block is another: that one rolls the whole transaction back, as above. A Python function or a record event callback that raises fails its statement the same way, see xref:how-to/functions-and-events.adoc#raises[Python functions and record events]. == A second begin diff --git a/docs/modules/ROOT/pages/installation.adoc b/docs/modules/ROOT/pages/installation.adoc index 4305c22..8dfca6b 100644 --- a/docs/modules/ROOT/pages/installation.adoc +++ b/docs/modules/ROOT/pages/installation.adoc @@ -77,7 +77,7 @@ Each native wheel is about 38 MB. These platforms are not built, and the install Engine modules the bindings do not use are left out to keep the wheel small. These include the gRPC and MongoDB wire protocols, Raft HA, and Gremlin. -NOTE: *JavaScript is not available.* The GraalJS jars are left out, so `DEFINE FUNCTION ... LANGUAGE js` fails. Define functions in SQL or openCypher, or register a Python function with `Database.register_function` (it works on both packages). +NOTE: *JavaScript is not available.* The GraalJS jars are left out, so `DEFINE FUNCTION ... LANGUAGE js` fails. Define functions in SQL or openCypher, or register a Python function with `Database.register_function` (it works on both packages, see xref:how-to/functions-and-events.adoc[Python functions and record events]). // REMOVE WHEN: the old arcadedb-embedded* shims are retired and two releases have shipped with the two-package layout allow-old-name NOTE: *The Studio web UI is in the JVM wheels from 26.11.1.* Studio used to bundle ApexCharts, which is no longer open source (https://github.com/ArcadeData/arcadedb/issues/10579[ArcadeData/arcadedb#10579]). https://github.com/ArcadeData/arcadedb/pull/10627[ArcadeData/arcadedb#10627] replaced it with Apache ECharts. The earlier 26.10.x wheels on PyPI do not contain Studio. The HTTP API of the in-process server works in all of them. diff --git a/docs/modules/ROOT/pages/quickstart.adoc b/docs/modules/ROOT/pages/quickstart.adoc index 7615794..1958864 100644 --- a/docs/modules/ROOT/pages/quickstart.adoc +++ b/docs/modules/ROOT/pages/quickstart.adoc @@ -186,3 +186,4 @@ In the `{jvm-pkg-name}` wheels from 26.11.1 the server also serves the Studio we * xref:how-to/values-and-records.adoc[Values and records]: which Python types go in and what comes back, datetimes, record objects and the `Schema` API. * xref:how-to/import-export.adoc[Importing and exporting data]: load a file, export a database or a query. * xref:how-to/dataframes-and-columns.adoc[DataFrames, Arrow and columns]: load and read pandas, Polars, Arrow and numpy data. +* xref:how-to/functions-and-events.adoc[Python functions and record events]: call a Python function from SQL and openCypher, and run Python when a record is written. diff --git a/tests/test_docs_functions_page.py b/tests/test_docs_functions_page.py new file mode 100644 index 0000000..8f74818 --- /dev/null +++ b/tests/test_docs_functions_page.py @@ -0,0 +1,62 @@ +"""Every Python block of the Antora page ``docs/modules/ROOT/pages/how-to/functions-and-events.adoc`` runs, and prints what the page says. + +Same pattern as ``test_docs_vectors_page.py``: the blocks are chained on purpose, they run in the order of the page, in one +namespace, against one fresh temporary database that the namespace holds as ``db``. A block that is followed by a +``[source,text]`` block must print exactly that text. What each statement of the page claims is pinned on its own in +``test_functions_and_events_behavior.py``. The test uses no Java object, so it runs on the JVM build and the native build alike. +""" + +import contextlib +import io +import re +from pathlib import Path + +PAGE = ( + Path(__file__).resolve().parents[1] + / "docs" + / "modules" + / "ROOT" + / "pages" + / "how-to" + / "functions-and-events.adoc" +) + +# a Python listing, then (optionally, directly after it) the text listing with its output +BLOCK_RE = re.compile( + r"\[source,python[^\]\n]*\]\n----\n(?P.*?)\n----\n" + r"(?:\n\[source,text\]\n----\n(?P.*?)\n----\n)?", + re.DOTALL, +) + + +def _blocks(): + text = PAGE.read_text(encoding="utf-8") + found = [(m.group("code"), m.group("out")) for m in BLOCK_RE.finditer(text)] + # no Python listing may escape the extraction (a different spelling of the block header would run nowhere) + assert text.count("[source,python") == len(found) + return found + + +def test_the_page_has_its_blocks(): + blocks = _blocks() + assert len(blocks) >= 9 + assert sum(1 for _, out in blocks if out is not None) >= 8 + + +def test_every_block_runs_in_order_and_prints_what_the_page_says(temp_db): + namespace = {"db": temp_db, "__name__": "functions_page"} + for number, (code, expected) in enumerate(_blocks(), start=1): + captured = io.StringIO() + compiled = compile( + code, f"functions-and-events.adoc python block {number}", "exec" + ) + with contextlib.redirect_stdout(captured): + exec(compiled, namespace) # nosec B102 - the page of this repository + if expected is not None: + assert ( + captured.getvalue().strip() == expected.strip() + ), f"block {number} printed other text than the page shows:\n{code}" + else: + assert ( + captured.getvalue() == "" + ), f"block {number} prints, the page shows no output" diff --git a/tests/test_functions_and_events_behavior.py b/tests/test_functions_and_events_behavior.py new file mode 100644 index 0000000..24b469a --- /dev/null +++ b/tests/test_functions_and_events_behavior.py @@ -0,0 +1,467 @@ +"""What Python functions in statements and record events do, as the how-to page +``docs/modules/ROOT/pages/how-to/functions-and-events.adoc`` says it (ArcadeData/arcadedb-python#267, row 8). + +``test_python_callbacks.py`` already pins the basics: a function in SELECT and WHERE, the names and argument counts, replace and +unregister, the types a function returns, a failing function as a query error, a record as a wrapper, each event with its record, +the type filter, the veto, ``remove()``, a failing callback and the registration errors. ``test_native_udf.py`` and +``test_python_function_interrupt.py`` pin the native upcall and a real Ctrl-C. This file pins what the page adds: what a function +receives from a stored value, what an exception does to a transaction, the scope of a function (the process) and of an event (one +database), the threads, and what a callback may do (read, write, re-enter, change the record). The tests use SQL, openCypher and the +wrapper objects only, no Java object, so they run on the JVM build and the native build alike. +""" + +import threading +from datetime import date, datetime +from decimal import Decimal + +import pytest + +import arcadedb +from arcadedb import ArcadeDBError + + +@pytest.fixture +def db(temp_db): + temp_db.command("sql", "CREATE DOCUMENT TYPE Person") + temp_db.command("sql", "CREATE DOCUMENT TYPE Audit") + temp_db.command("sql", "CREATE VERTEX TYPE City") + temp_db.command("sql", "CREATE EDGE TYPE Road") + return temp_db + + +def _insert(db, *names, type_name="Person"): + with db.transaction(): + statement = ( + f"INSERT INTO {type_name} SET name = ?" # nosec B608 - literal type names + ) + for name in names: + db.command("sql", statement, name) + + +def _names(db, type_name="Person"): + statement = f"SELECT name FROM {type_name}" # nosec B608 - literal type names + rows = db.query("sql", statement).to_list() + return sorted(row["name"] for row in rows) + + +def _count_in_another_thread(db, type_name): + seen = [] + statement = f"SELECT FROM {type_name}" # nosec B608 - literal type names + + def read(): + seen.append(len(db.query("sql", statement).to_list())) + + thread = threading.Thread(target=read) + thread.start() + thread.join() + return seen[0] + + +# --- what a function receives and returns ---------------------------------------------------------- + + +def test_the_literals_of_a_statement_arrive_as_the_python_types_they_look_like(db): + db.register_function("kinds", lambda *args: [type(a).__name__ for a in args]) + row = db.query( + "sql", "SELECT kinds(1, 2.5, 'a', true, null, [1, 2], {'k': 'v'}) AS k" + ).first() + assert row.get("k") == [ + "int", + "float", + "str", + "bool", + "NoneType", + "list", + "dict", + ] + + +def test_a_stored_datetime_date_and_decimal_arrive_as_python_values(db): + db.command("sql", "CREATE DOCUMENT TYPE Booking") + db.command("sql", "CREATE PROPERTY Booking.at DATETIME_MICROS") + db.command("sql", "CREATE PROPERTY Booking.day DATE") + db.command("sql", "CREATE PROPERTY Booking.price DECIMAL") + with db.transaction(): + db.command( + "sql", + "INSERT INTO Booking SET at = ?, day = ?, price = ?, tags = ?, meta = ?", + datetime(2026, 10, 1, 12, 34, 56, 789123), + date(2026, 10, 1), + Decimal("19.99"), + ["a", "b"], + {"k": 1}, + ) + seen = [] + db.register_function("seen", lambda *args: seen.append(args) or 0) + db.query( + "sql", "SELECT seen(at, day, price, tags, meta) AS s FROM Booking" + ).to_list() + assert len(seen) == 1 + at, day, price, tags, meta = seen[0] + # what the page says about values: a datetime has no time zone, a date is a date, a Decimal keeps its digits + assert type(at) is datetime and at == datetime(2026, 10, 1, 12, 34, 56, 789123) + assert type(day) is date and day == date(2026, 10, 1) + assert type(price) is Decimal and price == Decimal("19.99") + assert tags == ["a", "b"] and meta == {"k": 1} + + +def test_a_vertex_an_edge_and_a_document_arrive_as_their_own_wrapper(db): + with db.transaction(): + db.command("sql", "INSERT INTO Person SET name = 'Ann'") + db.command("sql", "CREATE VERTEX City SET name = 'A'") + db.command("sql", "CREATE VERTEX City SET name = 'B'") + db.command( + "sql", + "CREATE EDGE Road FROM (SELECT FROM City WHERE name = 'A') " + "TO (SELECT FROM City WHERE name = 'B') SET km = 5", + ) + db.register_function( + "kind", lambda record: [type(record).__name__, record.get_type_name()] + ) + for type_name, expected in ( + ("Person", ["Document", "Person"]), + ("City", ["Vertex", "City"]), + ("Road", ["Edge", "Road"]), + ): + statement = f"SELECT kind(@this) AS k FROM {type_name} LIMIT 1" # nosec B608 - literal type names + assert db.query("sql", statement).first().get("k") == expected + + +def test_a_tuple_result_is_a_list(db): + db.register_function("pair", lambda: (1, "a")) + assert db.query("sql", "SELECT pair() AS p").first().get("p") == [1, "a"] + + +def test_opencypher_passes_and_returns_the_same_python_values(db): + with db.transaction(): + db.command("opencypher", "CREATE (:City {name: 'Oslo'})") + seen = [] + + def describe(*args): + seen.append(args) + return [len(args), "ok"] + + db.register_function("py.describe", describe) + rows = db.query( + "opencypher", + "MATCH (c:City) RETURN py.describe(1, 2.5, 'a', true, null, [1, 2]) AS d", + ).to_list() + assert seen == [(1, 2.5, "a", True, None, [1, 2])] + assert [type(a) for a in seen[0]] == [int, float, str, bool, type(None), list] + assert rows == [{"d": [6, "ok"]}] + + +def test_the_function_runs_once_per_row(db): + _insert(db, "a", "b", "c") + calls = [] + db.register_function("note", lambda name: calls.append(name) or name) + db.query("sql", "SELECT note(name) AS n FROM Person").to_list() + assert sorted(calls) == ["a", "b", "c"] + + +def test_a_function_can_query_the_database_it_runs_in(db): + _insert(db, "a", "b") + + def people(): + return len(db.query("sql", "SELECT FROM Person").to_list()) + + db.register_function("people", people) + assert db.query("sql", "SELECT people() AS n").first().get("n") == 2 + + +# --- what an exception does ------------------------------------------------------------------------ + + +def test_a_function_that_raises_undoes_its_statement_and_the_transaction_goes_on(db): + _insert(db, "Ann", "Bob", "Cy") + + def refuse_bob(name): + if name == "Bob": + raise ValueError("no Bob") + return name.upper() + + db.register_function("refuse_bob", refuse_bob) + with db.transaction(): + db.command("sql", "INSERT INTO Audit SET note = 'before'") + with pytest.raises(ArcadeDBError, match="ValueError: no Bob"): + db.command("sql", "UPDATE Person SET shout = refuse_bob(name)") + assert db.is_transaction_active() is True + db.command("sql", "INSERT INTO Audit SET note = 'after'") + # the rows that the failed UPDATE had reached before Bob are not changed, and the other statements are committed + shouted = db.query("sql", "SELECT FROM Person WHERE shout IS NOT NULL").to_list() + assert shouted == [] + notes = db.query("sql", "SELECT note FROM Audit").to_list() + assert sorted(row["note"] for row in notes) == ["after", "before"] + + +@pytest.mark.parametrize("event", ["before_create", "after_create"]) +def test_a_callback_that_raises_undoes_its_statement_and_the_transaction_goes_on( + db, event +): + def refuse_bad(record): + if record.get("name") == "bad": + raise RuntimeError("no bad") + + db.on(event, refuse_bad, "Person") + with db.transaction(): + db.command("sql", "INSERT INTO Person SET name = 'first'") + with pytest.raises(ArcadeDBError, match=f"{event}.*RuntimeError: no bad"): + db.command("sql", "INSERT INTO Person SET name = 'bad'") + assert db.is_transaction_active() is True + db.command("sql", "INSERT INTO Person SET name = 'second'") + assert _names(db) == ["first", "second"] + + +def test_keyboard_interrupt_in_a_callback_reaches_the_caller_as_one(db): + def interrupt(record): + raise KeyboardInterrupt("stop") + + db.on("before_create", interrupt, "Person") + with pytest.raises(KeyboardInterrupt) as error: + with db.transaction(): + db.command("sql", "INSERT INTO Person SET name = 'x'") + assert isinstance(error.value.__cause__, ArcadeDBError) + assert "before_create" in str(error.value.__cause__) + assert db.is_transaction_active() is False + assert _names(db) == [] + + +# --- scope: a function belongs to the process, an event to one database ---------------------------- + + +def test_a_function_is_seen_by_every_open_database_and_goes_with_the_one_that_registered_it( + db, tmp_path +): + other = arcadedb.create_database(str(tmp_path / "other")) + try: + db.register_function("shared_fn", lambda: "hi") + assert other.query("sql", "SELECT shared_fn() AS x").first().get("x") == "hi" + # the name is taken for the process, and only the database that registered it can remove it + with pytest.raises(ArcadeDBError, match="already registered"): + other.register_function("shared_fn", lambda: "no") + with pytest.raises(ArcadeDBError, match="No Python function"): + other.unregister_function("shared_fn") + db.close() + with pytest.raises(ArcadeDBError, match="shared_fn"): + other.query("sql", "SELECT shared_fn() AS x").to_list() + finally: + other.close() + + +def test_an_event_callback_belongs_to_the_database_that_registered_it(db, tmp_path): + other = arcadedb.create_database(str(tmp_path / "other")) + try: + other.command("sql", "CREATE DOCUMENT TYPE Person") + seen = [] + db.on("after_create", lambda record: seen.append(record.get("name"))) + _insert(other, "elsewhere") + _insert(db, "here") + assert seen == ["here"] + finally: + other.close() + + +def test_closing_a_database_drops_its_callbacks(temp_db_path): + seen = [] + with arcadedb.create_database(temp_db_path) as first: + first.command("sql", "CREATE DOCUMENT TYPE Person") + first.on("after_create", lambda record: seen.append("type"), "Person") + first.on("after_create", lambda record: seen.append("database")) + with arcadedb.open_database(temp_db_path) as second: + _insert(second, "after reopen") + assert _names(second) == ["after reopen"] + assert seen == [] + + +@pytest.mark.parametrize("answer", [None, True, 0, "", []]) +def test_only_false_vetoes_a_before_callback(db, answer): + db.on("before_create", lambda record, answer=answer: answer, "Person") + _insert(db, "kept") + assert _names(db) == ["kept"] + + +def test_the_type_filter_matches_the_exact_type_and_not_its_subtypes(db): + db.command("sql", "CREATE DOCUMENT TYPE Animal") + db.command("sql", "CREATE DOCUMENT TYPE Dog EXTENDS Animal") + on_animal, on_everything = [], [] + db.on("after_create", lambda r: on_animal.append(r.get_type_name()), "Animal") + db.on("after_create", lambda r: on_everything.append(r.get_type_name())) + with db.transaction(): + db.command("sql", "INSERT INTO Dog SET name = 'rex'") + db.command("sql", "INSERT INTO Animal SET name = 'generic'") + assert on_animal == ["Animal"] + assert sorted(on_everything) == ["Animal", "Dog"] + + +def test_an_edge_fires_the_event_of_its_own_type(db): + seen = [] + db.on( + "after_create", + lambda r: seen.append((type(r).__name__, r.get_type_name())), + "Road", + ) + with db.transaction(): + db.command("sql", "CREATE VERTEX City SET name = 'A'") + db.command("sql", "CREATE VERTEX City SET name = 'B'") + db.command( + "sql", + "CREATE EDGE Road FROM (SELECT FROM City WHERE name = 'A') " + "TO (SELECT FROM City WHERE name = 'B')", + ) + assert seen == [("Edge", "Road")] + + +# --- when an event fires --------------------------------------------------------------------------- + + +def test_after_events_fire_before_the_commit_and_a_rollback_does_not_unfire_them(db): + log = [] + for event in ("before_create", "after_create"): + db.on(event, lambda record, event=event: log.append(event), "Person") + with pytest.raises(ValueError, match="stop"): + with db.transaction(): + db.command("sql", "INSERT INTO Person SET name = 'x'") + # both events have fired, and no other thread sees the row yet + assert log == ["before_create", "after_create"] + assert _count_in_another_thread(db, "Person") == 0 + raise ValueError("stop") + assert _names(db) == [] + assert log == ["before_create", "after_create"] + + +def test_a_before_callback_sees_the_table_without_the_record_and_an_after_callback_with_it( + db, +): + counts = {} + + def count(label): + def callback(record): + counts[label] = len(db.query("sql", "SELECT FROM Person").to_list()) + + return callback + + db.on("before_create", count("before"), "Person") + db.on("after_create", count("after"), "Person") + _insert(db, "first") + assert counts == {"before": 0, "after": 1} + + +# --- what a callback may do ------------------------------------------------------------------------ + + +def test_a_write_in_a_callback_belongs_to_the_transaction_that_fired_it(db): + db.on( + "after_create", + lambda record: db.command( + "sql", "INSERT INTO Audit SET note = ?", "created " + record.get("name") + ), + "Person", + ) + _insert(db, "ann") + notes = db.query("sql", "SELECT note FROM Audit").to_list() + assert [row["note"] for row in notes] == ["created ann"] + + with pytest.raises(ValueError, match="stop"): + with db.transaction(): + db.command("sql", "INSERT INTO Person SET name = 'bob'") + raise ValueError("stop") + notes = db.query("sql", "SELECT note FROM Audit").to_list() + assert [row["note"] for row in notes] == ["created ann"] + assert _names(db) == ["ann"] + + +def test_a_callback_for_every_record_fires_for_the_writes_that_callbacks_make(db): + log = [] + + def audit(record): + log.append(record.get_type_name()) + if ( + record.get_type_name() != "Audit" + ): # without this guard the callback would never end + db.command("sql", "INSERT INTO Audit SET note = 'seen'") + + db.on("after_create", audit) + _insert(db, "ann") + assert log == ["Person", "Audit"] + + +def test_changing_the_record_in_before_create_is_stored_on_the_jvm_build_only(db): + def stamp(record): + record.set("stamp", "x") + + db.on("before_create", stamp, "Person") + _insert(db, "ann") + row = db.query("sql", "SELECT FROM Person").first() + expected = "x" if arcadedb.__backend__ == "jvm" else None + assert row.get("stamp") == expected + + +# --- threads --------------------------------------------------------------------------------------- + + +def test_a_function_runs_on_the_thread_of_its_statement_and_on_an_engine_thread_for_async_writes( + db, +): + _insert(db, "a") + seen = [] + db.register_function("who", lambda x: seen.append(threading.get_ident()) or x) + + own = [] + + def run(): + own.append(threading.get_ident()) + db.query("sql", "SELECT who(name) AS w FROM Person").to_list() + + thread = threading.Thread(target=run) + thread.start() + thread.join() + assert seen == own + db.query("sql", "SELECT who(name) AS w FROM Person").to_list() + assert seen[-1] == threading.get_ident() + + executor = db.async_executor() + executor.command("sql", "INSERT INTO Audit SET note = who('async')") + executor.wait_completion() + assert len(seen) == 3 and seen[-1] != threading.get_ident() + + +def test_a_function_that_waits_does_not_block_other_threads(db): + _insert(db, "a") + entered, release = threading.Event(), threading.Event() + + def park(name): + entered.set() + release.wait(30) + return name + + db.register_function("park", park) + outcome = [] + + def run(): + outcome.append( + db.query("sql", "SELECT park(name) AS p FROM Person").first().get("p") + ) + + thread = threading.Thread(target=run) + thread.start() + try: + assert entered.wait(30) + # the statement of the other thread is inside the function, and this thread still runs statements + assert db.query("sql", "SELECT 1 + 1 AS two").first().get("two") == 2 + assert outcome == [] + finally: + release.set() + thread.join(30) + assert outcome == ["a"] + + +def test_the_event_of_an_async_write_runs_on_an_engine_thread(db): + idents = [] + db.on("after_create", lambda record: idents.append(threading.get_ident()), "Person") + executor = db.async_executor() + document = db.new_document("Person") + document.set("name", "async") + executor.create_record(document) + executor.wait_completion() + assert len(idents) == 1 and idents[0] != threading.get_ident() + assert _names(db) == ["async"] From e11394a59896e4625cb2bf6ad312ab03dd7f4569 Mon Sep 17 00:00:00 2001 From: Taewoon Kim Date: Fri, 6 Nov 2026 15:41:33 +0900 Subject: [PATCH 2/2] Docs: get_type_name() is a Java string on arcadedb-jvm, so the page converts it with str() (refs #267) The JVM run of the docs-page test failed in the Journal block: record.get_type_name() returns the Java string on arcadedb-jvm (graph.py returns getTypeName() as is), and "created " + java_string raises TypeError, while the native wrapper returns a str. record.get(...), to_dict() values and get_rid() are Python str on both builds. New behavior test pins the type per backend, inside a callback and outside one; the page says once, in the callback section, to write str(record.get_type_name()), and the block does. Co-Authored-By: Claude Sonnet 5.5 --- .../pages/how-to/functions-and-events.adoc | 5 ++-- tests/test_functions_and_events_behavior.py | 27 +++++++++++++++++++ 2 files changed, 30 insertions(+), 2 deletions(-) diff --git a/docs/modules/ROOT/pages/how-to/functions-and-events.adoc b/docs/modules/ROOT/pages/how-to/functions-and-events.adoc index cca92d3..0df9935 100644 --- a/docs/modules/ROOT/pages/how-to/functions-and-events.adoc +++ b/docs/modules/ROOT/pages/how-to/functions-and-events.adoc @@ -3,7 +3,7 @@ // Sources: src/arcadedb/callbacks.py and core.py (register_function, unregister_function, on), _native/extras.py (the native build). Every statement below is pinned by tests/test_functions_and_events_behavior.py. The basics are pinned by tests/test_python_callbacks.py (names, argument counts, replace, return types, record events, veto), the native upcall by tests/test_native_udf.py, and a real Ctrl-C by tests/test_python_function_interrupt.py and tests/test_transaction_context_interrupt.py. -Use `register_function` when a statement needs something that SQL and openCypher cannot compute, such as a slug, a hash or a call into your own library. Use `db.on` when Python code must run each time a record is written, for an audit trail or a rule that can refuse a write. Both are part of the Python bindings, not of the engine, and both work the same on `arcadedb` and `arcadedb-jvm`, with one difference that is called out below. +Use `register_function` when a statement needs something that SQL and openCypher cannot compute, such as a slug, a hash or a call into your own library. Use `db.on` when Python code must run each time a record is written, for an audit trail or a rule that can refuse a write. Both are part of the Python bindings, not of the engine, and both work the same on `arcadedb` and `arcadedb-jvm`, with two differences that are called out below. The short version: @@ -322,6 +322,7 @@ A callback runs inside the operation that fired it, on the thread that runs it. * *Read.* A callback can run `db.query`. A `before_create` callback does not see the new record in a query. An `after_create` callback does. * *Write.* A callback can run `db.command`. The write joins the transaction of the operation that fired the event: it commits with it and rolls back with it. * *Fire more events.* The write of a callback fires events too. A callback registered without `type_name` fires for its own writes. Guard against that, or it never ends. +* *Read the type name with `str(...)`.* On `arcadedb-jvm`, `record.get_type_name()` returns a Java string, not a Python `str`. It compares equal to a `str`, but adding it to a `str`, as in `"created " + record.get_type_name()`, raises `TypeError`. Write `str(record.get_type_name())`. On `arcadedb` it is already a `str`. A property read with `record.get(...)`, a value of `to_dict()` and `get_rid()` are Python `str` on both. * *Change the record.* Do not rely on it. On `arcadedb-jvm` a `before_create` callback can call `record.set(...)`, and the value is stored. On `arcadedb` the change is lost. Put the value in the statement instead. * *Wait.* Keep it quick, and do not wait in it for another thread's transaction. @@ -332,7 +333,7 @@ db.command("sql", "CREATE DOCUMENT TYPE Journal") def note_creation(record): if record.get_type_name() != "Journal": - note = "created " + record.get_type_name() + note = "created " + str(record.get_type_name()) db.command("sql", "INSERT INTO Journal SET note = ?", note) diff --git a/tests/test_functions_and_events_behavior.py b/tests/test_functions_and_events_behavior.py index 24b469a..840ad1d 100644 --- a/tests/test_functions_and_events_behavior.py +++ b/tests/test_functions_and_events_behavior.py @@ -465,3 +465,30 @@ def test_the_event_of_an_async_write_runs_on_an_engine_thread(db): executor.wait_completion() assert len(idents) == 1 and idents[0] != threading.get_ident() assert _names(db) == ["async"] + + +def test_the_type_name_of_a_record_is_a_python_str_on_the_native_build_and_a_java_string_on_the_jvm_build( + db, +): + seen = {} + + def look(record): + seen["type_name"] = record.get_type_name() + seen["name"] = record.get("name") + seen["dict_name"] = record.to_dict()["name"] + seen["rid"] = record.get_rid() + + db.on("after_create", look, "Person") + _insert(db, "x") + # a property, a to_dict value and the RID are Python str on both builds + for key in ("name", "dict_name", "rid"): + assert type(seen[key]) is str, key + assert seen["name"] == "x" and seen["dict_name"] == "x" + # the type name is a Java string on the JVM build: it compares equal, and str() makes it a Python str + value = seen["type_name"] + assert value == "Person" and type(str(value)) is str + assert (type(value) is str) == (arcadedb.__backend__ != "jvm") + # the same holds for a record that is not in a callback + outside = db.query("sql", "SELECT FROM Person").first().get_element() + assert outside.get_type_name() == "Person" + assert (type(outside.get_type_name()) is str) == (arcadedb.__backend__ != "jvm")