diff --git a/.github/workflows/release-native.yml b/.github/workflows/release-native.yml index a4fcf29..fb3b4ed 100644 --- a/.github/workflows/release-native.yml +++ b/.github/workflows/release-native.yml @@ -25,7 +25,8 @@ name: Release native (arcadedb) # # Nothing here stores a token: the publish job authenticates with PyPI trusted publishing (OIDC), and only that job has the id-token # permission. The guard in the publish job refuses to publish anything that is not `arcadedb`, so this workflow cannot publish the -# JVM `arcadedb-jvm` by accident. +# JVM `arcadedb-jvm` by accident. The test call carries issues: write too, for the one job of test-native.yml that logs a crash retry +# on issue 295 (see the comment at that call). on: push: @@ -97,6 +98,12 @@ jobs: name: Native build and parity needs: validate-version uses: ./.github/workflows/test-native.yml + # issues: write is for the `crash-report` job of test-native.yml alone (one comment on issue 295 when a Windows test file crashed and + # passed on its retry). A called workflow cannot hold more permission than its caller gives it, so without this line the whole release + # run would refuse to start; the called workflow's own `permissions: contents: read` keeps every other job read-only. + permissions: + contents: read + issues: write with: version: ${{ needs.validate-version.outputs.version }} release: true diff --git a/.github/workflows/test-native.yml b/.github/workflows/test-native.yml index 02a5f6e..3515050 100644 --- a/.github/workflows/test-native.yml +++ b/.github/workflows/test-native.yml @@ -33,6 +33,12 @@ # # A leg that passes uploads a marker (native-passed--py) and the wheel (native-wheel-); release-native.yml # reads them to decide what may be published. Do not rename these artifacts without changing release-native.yml. +# +# Crashes (ArcadeData/arcadedb-python#295): a test file whose process dies without a result is a CRASH row with its exit code, and fails the leg. +# On the Windows legs only, the harness runs such a file once more in a fresh process (--retry-crash): a retry that produces a result is a +# FLAKY-CRASH row, the leg may pass, and the `crash-report` job makes it loud (the job summary, and on a release or a dispatch one comment on +# issue 295); a second crash, a failing test, a timeout and every Linux or macOS crash stay failures. Each Windows test process also writes a +# faulthandler trace into the parity artifact (parity-wheel-xml/.attemptN.fault.txt). name: Test native (arcadedb) on: @@ -650,12 +656,16 @@ jobs: cores="$(nproc 2>/dev/null || sysctl -n hw.ncpu 2>/dev/null || echo 2)" limit=() if [ "$FINAL" = true ]; then limit=(--max-notyet 0); fi + # A file whose process dies without a result is run once more on Windows only (the crashes of issue 295 are Windows only); + # Linux and macOS stay strict. The harness marks a passing retry FLAKY-CRASH and the crash-report job logs it. + retry=() + if [ "$RUNNER_OS" = Windows ]; then retry=(--retry-crash); fi # macOS runs bash 3.2: under set -u an empty "${limit[@]}" is "unbound variable", so expand it only when set uv run --no-project --python "$PY" --with "$w" --with pytest --with setuptools --with numpy --with requests --with jpype1 --with pyarrow \ --with polars --with pandas --with duckdb --with networkx --with "psycopg[binary]" --with adbc-driver-postgresql \ --with redis --with neo4j --with ladybug \ python native/wheel/parity_wheel.py --repo "$GITHUB_WORKSPACE" \ - --out "$T/parity/parity-wheel.tsv" --cores "0-$((cores - 1))" ${limit[@]+"${limit[@]}"} | tee "$T/parity/parity-wheel.run.log" + --out "$T/parity/parity-wheel.tsv" --cores "0-$((cores - 1))" ${limit[@]+"${limit[@]}"} ${retry[@]+"${retry[@]}"} | tee "$T/parity/parity-wheel.run.log" - name: "Gate: no failure, error, crash, timeout or uncounted skip" run: | @@ -665,10 +675,13 @@ jobs: awk -F'\t' ' NR == 1 { next } $1 == "TOTAL" { passed = $2; failed = $3; err = $4; notyet = $5; na = $6; skipped = $7; tot = 1; next } + $8 ~ /^FLAKY-CRASH/ { flaky = flaky " " $1 "(" $8 ")"; next } $8 != "" { bad = bad " " $1 "(" $8 ")" } END { if (!tot) { print "no TOTAL row" > "/dev/stderr"; exit 1 } printf "parity %s py%s: passed=%s failed=%s error=%s notyet=%s na_jvm=%s skipped=%s\n", ENVIRON["KEY"], ENVIRON["PY"], passed, failed, err, notyet, na, skipped + # a file that crashed and passed when run once more: the leg may pass, the summary says it did only because of the retry + if (flaky != "") printf "FLAKY-CRASH (died without a result, passed on the one retry; first exit code in brackets):%s\n", flaky if (bad != "") { print "files with a status:" bad > "/dev/stderr"; exit 1 } if (passed + 0 == 0 || failed + 0 != 0 || err + 0 != 0) exit 1 # this repository accepts no skips; NOTYET and NA-JVM are the two counted kinds of the native build @@ -696,11 +709,44 @@ jobs: name: native-parity-${{ matrix.key }}-py${{ matrix.python }} path: | ${{ runner.temp }}/parity/parity-wheel.tsv + ${{ runner.temp }}/parity/parity-wheel.meta.json ${{ runner.temp }}/parity/parity-wheel.run.log ${{ runner.temp }}/parity/parity-wheel-xml/ if-no-files-found: warn retention-days: 30 + # The one job that may write to an issue: after the parity legs it reads their meta files (parity-wheel.meta.json in each native-parity-* + # artifact) and lists every FLAKY-CRASH and CRASH in the job summary; on a release (a tag push) or a dispatch it also posts ONE comment on + # issue 295 (native/wheel/report_flaky_crash.py decides: never on a pull request, only in ArcadeData/arcadedb-python). A log is not a gate: a token + # that cannot write issues, a missing artifact or a gh failure print a warning and the job still ends green. release-native.yml grants the + # issues permission to its call of this workflow, because a called workflow cannot hold more permission than its caller gives it. + crash-report: + name: Report parity crashes + needs: [plan, test] + if: ${{ !cancelled() && needs.plan.outputs.run == 'true' }} + runs-on: ubuntu-24.04 + timeout-minutes: 10 + continue-on-error: true + permissions: + contents: read + issues: write + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + + - uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333 # v8.0.2 + continue-on-error: true + with: + pattern: native-parity-* + path: ${{ runner.temp }}/parity-artifacts + + - name: List the crashes and log them on issue 295 + continue-on-error: true + env: + GH_TOKEN: ${{ github.token }} + run: python3 -I native/wheel/report_flaky_crash.py --artifacts "$RUNNER_TEMP/parity-artifacts" + # The examples of examples/ against the INSTALLED native wheel, so the scripts a new user copies are run on what `uv add arcadedb` installs. # Nothing is built here: the wheel is the one the build job uploaded (native-wheel-), the file the parity job installs. The platforms are # those of the parity job and the Pythons are 3.12 and 3.14 (scripts/select_native_matrix.py, `examples`). The runner is scripts/run_examples.sh, the @@ -1031,10 +1077,13 @@ jobs: awk -F'\t' ' NR == 1 { next } $1 == "TOTAL" { passed = $2; failed = $3; err = $4; notyet = $5; na = $6; skipped = $7; tot = 1; next } + $8 ~ /^FLAKY-CRASH/ { flaky = flaky " " $1 "(" $8 ")"; next } $8 != "" { bad = bad " " $1 "(" $8 ")" } END { if (!tot) { print "no TOTAL row" > "/dev/stderr"; exit 1 } printf "parity %s py%s: passed=%s failed=%s error=%s notyet=%s na_jvm=%s skipped=%s\n", ENVIRON["KEY"], ENVIRON["PY"], passed, failed, err, notyet, na, skipped + # a file that crashed and passed when run once more: the leg may pass, the summary says it did only because of the retry + if (flaky != "") printf "FLAKY-CRASH (died without a result, passed on the one retry; first exit code in brackets):%s\n", flaky if (bad != "") { print "files with a status:" bad > "/dev/stderr"; exit 1 } if (passed + 0 == 0 || failed + 0 != 0 || err + 0 != 0) exit 1 # this repository accepts no skips; NOTYET and NA-JVM are the two counted kinds of the native build diff --git a/docs/modules/ROOT/pages/development/ci.adoc b/docs/modules/ROOT/pages/development/ci.adoc index 57faa66..39dbcac 100644 --- a/docs/modules/ROOT/pages/development/ci.adoc +++ b/docs/modules/ROOT/pages/development/ci.adoc @@ -100,6 +100,7 @@ The workflow `test-native.yml` builds the native image of the `arcadedb` package * A pull request runs it only when native paths change: `native/**`, `src/arcadedb/**`, `tests/**`, `scripts/select_native_matrix.py` or `test-native.yml` itself. Changes under `tests/` and `src/arcadedb/` are included because both backends share those files, so a shared test or source can break native parity without touching a native file; docs outside those paths do not run it. It then runs on `linux-x86_64` with Python 3.12 only. * A push to `main`, the nightly run, a manual run (dispatch) run all four platforms: `linux-x86_64`, `linux-aarch64`, `macos-arm64` and `windows-x64` (Python 3.12 and 3.14). A release (`release-native.yml` calls the workflow with `release: true`) runs the same four platforms on all four Python versions, 16 legs. * The `pyver` job installs the already built `linux-x86_64` wheel (no rebuild) into the Pythons that the full parity runs of the event leave out (3.11, 3.13 and 3.14 on a pull request; 3.11 and 3.13 on a push to `main`, the nightly run and a dispatch; none on a release) into 3.15 and into the free-threaded 3.14t, and runs 13 test files there (14 where polars installs; 3.15 has no jpype1 wheel, so it leaves out the two files that need jpype1), about 20 seconds of tests per Python. It is informational: `release-native.yml` reads only the `native-passed-*` markers and the `native-wheel-*` wheels, so a red `pyver` leg never blocks a release. The 3.14t leg has no polars (no free-threaded wheel exists), so it skips `test_dataframe_ingest.py`. `scripts/select_native_matrix.py` decides its legs. +* Every test file runs in its own process (`native/wheel/parity_wheel.py`). A process that dies without writing its result is a `CRASH` row in `parity-wheel.tsv` with its exit code, for example `CRASH(rc=0xC0000005,3221225477)`, and it fails the leg. On the Windows legs only (the crashes of issue 295 happen there), the harness runs such a file once more in a fresh process (`--retry-crash`). A retry that produces a result is a `FLAKY-CRASH` row: the gate lets the leg pass, the job summary lists the file with the first exit code, and on a release or a dispatch the `crash-report` job adds one comment to issue 295 (run URL, Python, wheel version, exit codes). A second crash, a failing test, a timeout and every crash on Linux or macOS stay failures. A process that wrote its result and then died in shutdown is a `DIRTY-EXIT` record (exit code, file, attempt) in the metadata, the job summary and the issue comment: it is not retried and does not fail the leg. Each Windows test process also writes a faulthandler trace, `parity-wheel-xml/.attemptN.fault.txt`, into the parity artifact of the leg. Only a retry hides a crash, so treat a `FLAKY-CRASH` as an open bug: read the trace in the artifact before a release is announced. * A native image build takes about 5 to 30 minutes for each platform (macOS is the slowest). The organisation shares one runner limit with upstream ArcadeDB, so do not add runs lightly. Change the paths or the matrix of `test-native.yml` only for a reason that you can write down. == Change the selection diff --git a/docs/modules/ROOT/pages/how-to/dataframes-and-columns.adoc b/docs/modules/ROOT/pages/how-to/dataframes-and-columns.adoc index 4e70c87..eb0796a 100644 --- a/docs/modules/ROOT/pages/how-to/dataframes-and-columns.adoc +++ b/docs/modules/ROOT/pages/how-to/dataframes-and-columns.adoc @@ -1,7 +1,7 @@ = DataFrames, Arrow and columns :description: Load a pandas or Polars frame, an Arrow table or whole columns with insert_dataframe, insert_arrow and insert_columns, what each Arrow type becomes, how nulls and NaN are stored, what a declared property accepts, how many rows are converted at a time, what a failed load leaves, and what to_columns, to_arrow, to_dataframe and to_polars return, including the difference between the two builds. -// Sources: src/arcadedb/core.py (insert_columns, insert_dataframe, insert_arrow), dataframe.py, results.py and _native/columns.py (to_columns, to_arrow, to_dataframe, to_polars). The statements below are pinned by tests/test_dataframe_columns_behavior.py, and for the Arrow types, the declared types, the column checks, the readers and the dates also by tests/test_dataframe_ingest.py, tests/test_insert_columns.py, tests/test_resultset_arrow.py, tests/test_columnar_readers.py and tests/test_resultset.py. +// Sources: src/arcadedb/core.py (insert_columns, insert_dataframe, insert_arrow), dataframe.py, results.py, _columnar.py and _native/colframes.py (to_columns, to_arrow, to_dataframe, to_polars). The statements below are pinned by tests/test_dataframe_columns_behavior.py, and for the Arrow types, the declared types, the column checks, the readers and the dates also by tests/test_dataframe_ingest.py, tests/test_insert_columns.py, tests/test_resultset_arrow.py, tests/test_columnar_readers.py and tests/test_resultset.py. This page covers column-shaped data in and out of a database. Load it with `insert_dataframe` (a pandas or Polars frame), `insert_arrow` (an Arrow table) or `insert_columns` (numpy arrays and lists). Read a result as columns with `to_columns`, `to_arrow`, `to_dataframe` or `to_polars`. For a list of Python dicts use `insert_many`, and for a file use the importer: xref:how-to/bulk-loads.adoc[Bulk loads and commit size] and xref:how-to/import-export.adoc[Importing and exporting data]. The calls here are bindings calls and have no statement in SQL, so the main manual has nothing to say about them. The types that a stored value can have are the engine's: {arcadedb-docs-url}/reference/datatypes[Data types]. @@ -337,12 +337,7 @@ A missing library does not fail the same way everywhere. `to_columns()` returns == Differences between the two builds -The calls above, the types and the results are the same on `arcadedb` and `arcadedb-jvm`. Two things differ. The first is how the readers get the rows: - -* On `arcadedb-jvm`, `to_columns()` and `to_arrow()` read the result from the engine in batches of `batch_size` rows, as packed columns, so `batch_size` sets how many rows cross at a time. -* On `arcadedb` (native), they read every row into Python dicts first and build the columns from them. `batch_size` is accepted and not used, and the rows exist as Python objects while the columns are built. - -The second is the message of a missing pandas: the JVM package adds the `uv add` hint to the `ImportError` of `to_dataframe()`, and the native package raises the plain `ImportError`. +The calls above, the types and the results are the same on `arcadedb` and `arcadedb-jvm`, and so is how `to_columns()` and `to_arrow()` get the rows: both read the result from the engine as packed columns, in requests of at most `batch_size` rows, so `batch_size` sets how many rows cross at a time. One thing differs, the message of a missing pandas: the JVM package adds the `uv add` hint to the `ImportError` of `to_dataframe()`, and the native package raises the plain `ImportError`. == Related pages diff --git a/docs/modules/ROOT/pages/how-to/vectors.adoc b/docs/modules/ROOT/pages/how-to/vectors.adoc index 23c4362..23df5dc 100644 --- a/docs/modules/ROOT/pages/how-to/vectors.adoc +++ b/docs/modules/ROOT/pages/how-to/vectors.adoc @@ -110,9 +110,9 @@ print(db.count_type("Chunk")) 200 ---- -`GraphBatch.create_vertices` rows and the properties of `new_edge` and `new_edges` take the same arrays. On the `arcadedb` (native) package, a non-empty one-dimensional `float32` array in a row or an edge property crosses in a binary side buffer, and the stored values are bit-identical to the ones the text path gave. A list or a `float64` array still crosses as number text, so make the array with `np.asarray(v, dtype=np.float32)` before the load. A laptop A/B on the native wheel (100,000 vertices for each load, `ab.py --noise`, ABBA order, 12 clean pairs for each figure, clean A/A control) gave this wall time of the new path against the old one (95 % interval of the median pair ratio): 0.517 [0.515, 0.520] at 128 dimensions, 0.218 [0.217, 0.219] at 96, and 0.271 [0.270, 0.272] at 64. A load without vectors was unchanged (1.009 [1.003, 1.012]). +`GraphBatch.create_vertices` rows and the properties of `new_edge` and `new_edges` take the same arrays. On the `arcadedb` (native) package, a non-empty one-dimensional `float32` array in a row or an edge property crosses in a binary side buffer, and the stored values are bit-identical to the ones the text path gave. A list or a `float64` array still crosses as number text, so make the array with `np.asarray(v, dtype=np.float32)` before the load. When a property holds a `float32` array of one length in every row of a `create_vertices` call, the whole column crosses as one float32 matrix in that buffer, with no tag or Python call for each row. A laptop A/B on the native wheel (100,000 vertices for each load, `ab.py --noise`, ABBA order, 12 clean pairs for each figure, clean A/A control) gave this wall time of the new path against the old one (95 % interval of the median pair ratio): 0.517 [0.515, 0.520] at 128 dimensions, 0.218 [0.217, 0.219] at 96, and 0.271 [0.270, 0.272] at 64. A load without vectors was unchanged (1.009 [1.003, 1.012]). -On the `arcadedb-jvm` package, `create_vertices` already sends the values of a property declared `ARRAY_OF_FLOATS` as one binary float32 matrix when every row holds a list or array of numbers of one length. So `float32` arrays in a property declared `ARRAY_OF_FLOATS` take the binary path on both packages. +On the `arcadedb-jvm` package, `create_vertices` sends the values of a property declared `ARRAY_OF_FLOATS` as one binary float32 matrix when every row holds a list, a numpy array or a Java `float[]` or `double[]` of numbers of one length. A property that is not declared takes the same matrix when every row holds a `float32` numpy array or a Java `float[]` (what `to_java_float_array` returns on this package) of one length, and the stored value is a `float[]`, as before. A list or a `float64` array on an undeclared property keeps its doubles and the older path. So `float32` arrays and `to_java_float_array` values take the binary path on both packages, declared or not. == Search diff --git a/docs/modules/ROOT/pages/installation.adoc b/docs/modules/ROOT/pages/installation.adoc index a8ce4b5..0f2e67d 100644 --- a/docs/modules/ROOT/pages/installation.adoc +++ b/docs/modules/ROOT/pages/installation.adoc @@ -157,6 +157,11 @@ WARNING: *One JVM configuration per process.* JVM options are locked after the J Each Python process starts its own JVM, and each JVM sees the whole container limit. With several Python processes in one container, give each process its own `heap_size`. +[[jvm-cds]] +=== Class-data sharing + +The bundled JRE ships the JVM's default class-data-sharing archive, dumped with `-XX:+UseCompactObjectHeaders`, the flag that `start_jvm` sets. The JVM finds the archive by itself and maps the JDK's classes from it instead of loading and verifying them at every start. You do nothing to use it. The JVM ignores the archive, and starts as it did before, when your `jvm_args` change a setting that the archive depends on (`-XX:-UseCompactObjectHeaders`, `-Xshare:off`) or when the heap is larger than 32 GB (the default is half of the RAM, so a machine with more than 64 GB of RAM starts without the archive unless you set `heap_size`). The archive holds the JDK's classes only, not the engine's jars. + [[jvm-aot]] === Faster start-up with the AOT cache diff --git a/native/src/main/java/org/humemai/nativelib/ArcadeNative.java b/native/src/main/java/org/humemai/nativelib/ArcadeNative.java index 7a6e31e..3d6eb6c 100644 --- a/native/src/main/java/org/humemai/nativelib/ArcadeNative.java +++ b/native/src/main/java/org/humemai/nativelib/ArcadeNative.java @@ -317,6 +317,52 @@ public static CCharPointer nextBin(final IsolateThread thread, final long rsId, } } + /** + * The next rows of an open cursor as one column frame (see ColumnFrames), written as an int64 LE byte count and the frame; a failure is + * the negated byte count of the error JSON and that JSON. max: rows at most (a frame also ends at about 1 MiB of values). columns: a + * JSON array of the column names to read, or "" for the union of the names of the rows. flags bit 0: datetimes as microseconds (the + * to_dataframe form). The rows the cursor already gave out in a JSON reply are not in the frame. The frame that ends the rows closes the + * cursor. The reply is freed with arcadedb_free. + */ + @CEntryPoint(name = "arcadedb_n_next_col") + @SuppressWarnings("unchecked") + public static CCharPointer nextCol(final IsolateThread thread, final long rsId, final int max, final CCharPointer columns, + final int flags) { + ResultSet rs = null; + try { + rs = RESULT_SETS.get(rsId); + if (rs == null) + throw new IllegalArgumentException("unknown result set " + rsId); + String[] pinned = null; + final String spec = str(columns); + if (spec != null && !spec.isEmpty()) { + final List names = (List) Json.parse(spec); + pinned = new String[names.size()]; + for (int i = 0; i < pinned.length; i++) + pinned[i] = (String) names.get(i); + } + final ColumnFrames.Frame frame = ColumnFrames.next(rs, max, pinned, (flags & 1) != 0); + if (frame.done) { + RESULT_SETS.remove(rsId); + closeQuietly(rs); + } + return frameOut(frame.bytes.length, frame.bytes); + } catch (final Throwable t) { + RESULT_SETS.remove(rsId); + closeQuietly(rs); + final byte[] e = errorJson(t).getBytes(StandardCharsets.UTF_8); + return frameOut(-e.length, e); + } + } + + private static CCharPointer frameOut(final int length, final byte[] b) { + final CCharPointer p = UnmanagedMemory.malloc(8 + b.length); + final ByteBuffer bb = CTypeConversion.asByteBuffer(p, 8 + b.length).order(ByteOrder.LITTLE_ENDIAN); + bb.putLong(length); + bb.put(b, 0, b.length); + return p; + } + @CEntryPoint(name = "arcadedb_n_rs_close") public static CCharPointer rsClose(final IsolateThread thread, final long rsId) { closeQuietly(RESULT_SETS.remove(rsId)); @@ -605,7 +651,8 @@ public static CCharPointer insertCols(final IsolateThread thread, final long h, * Columnar insert by pointer: no copy of the columns into one buffer. spec points at 6 longs per column (data address and bytes, aux * address and bytes, null-bitmap address and bytes, see BulkOps.insertColumns) and nspec is their count. Kinds q d b f as * arcadedb_n_insert_cols, plus i (int32), e (float32) and s (Arrow UTF-8 string: int32 offsets and the bytes they index), and null bitmaps. - * The caller keeps every buffer alive and unchanged until the call returns. Returns {"ok":true,"n":rows}. + * The caller keeps every buffer alive and unchanged until the call returns. Returns {"ok":true,"n":rows}; with "parallel":true in the + * header the records go to the async executor and the reply also carries "fid" (see BulkOps.insertColumnsReply). */ @CEntryPoint(name = "arcadedb_n_insert_colp") public static CCharPointer insertColp(final IsolateThread thread, final long h, final CCharPointer type, final CCharPointer header, @@ -616,7 +663,7 @@ public static CCharPointer insertColp(final IsolateThread thread, final long h, final long[] table = new long[nspec]; for (int i = 0; i < nspec; i++) table[i] = ((Pointer) spec).readLong(WordFactory.signed(8L * i)); - return out("{\"ok\":true,\"n\":" + BulkOps.insertColumns(db(h), str(type), str(header), table) + "}"); + return out(BulkOps.insertColumnsReply(db(h), str(type), str(header), table)); } catch (final Throwable t) { return err(t); } diff --git a/native/src/main/java/org/humemai/nativelib/BulkOps.java b/native/src/main/java/org/humemai/nativelib/BulkOps.java index 37cedf9..c703d87 100644 --- a/native/src/main/java/org/humemai/nativelib/BulkOps.java +++ b/native/src/main/java/org/humemai/nativelib/BulkOps.java @@ -10,16 +10,25 @@ import com.arcadedb.database.Database; import com.arcadedb.database.MutableDocument; +import com.arcadedb.database.async.DatabaseAsyncExecutor; +import com.arcadedb.database.async.ErrorCallback; +import com.arcadedb.exception.NeedRetryException; +import com.arcadedb.schema.EdgeType; import com.arcadedb.schema.VertexType; import org.graalvm.nativeimage.c.type.CTypeConversion; import org.graalvm.word.Pointer; import org.graalvm.word.WordFactory; +import java.math.BigDecimal; import java.nio.ByteBuffer; import java.nio.charset.StandardCharsets; +import java.time.LocalDate; +import java.time.LocalDateTime; +import java.time.ZoneOffset; import java.util.List; import java.util.Map; +import java.util.function.IntConsumer; final class BulkOps { private BulkOps() { @@ -55,26 +64,45 @@ static long insertColumns(final Database db, final String type, final String hea spec[SLOTS * c + 1] = bytes; pos += bytes; } - return insertColumns(db, type, h, spec); + return insertColumns(db, type, h, spec, null); } - /** The pointer form (arcadedb_n_insert_colp): the table holds SLOTS longs per column, see {@link #insertColumns(Database, String, Map, long[])}. */ + /** + * The pointer form (arcadedb_n_insert_colp): the table holds SLOTS longs per column, see {@link #insertColumns(Database, String, Map, long[], ErrorCallback)}. + * Returns the reply text {"ok":true,"n":rows}. A header with "parallel":true must also carry "fid", a failure token that the caller made + * with the "token" operation of arcadedb_n_async: the records go to the database's async executor, the call does not wait for the writers + * and neither opens, commits nor rolls back a transaction ("commit" does not apply), exactly as insert_parallel does for rows. The + * caller waits for the writers and reads (and so removes) the token with the "failures" operation, also when this call throws or never + * returns to it: the token is the caller's, so this call never drops it. + */ @SuppressWarnings("unchecked") - static long insertColumns(final Database db, final String type, final String header, final long[] spec) { - return insertColumns(db, type, (Map) Json.parse(header), spec); + static String insertColumnsReply(final Database db, final String type, final String header, final long[] spec) { + final Map h = (Map) Json.parse(header); + if (!Boolean.TRUE.equals(h.get("parallel"))) + return "{\"ok\":true,\"n\":" + insertColumns(db, type, h, spec, null) + "}"; + if (!(h.get("fid") instanceof Number fid)) + throw new IllegalArgumentException("a parallel columnar insert needs a failure token (header \"fid\", from the \"token\" operation)"); + return "{\"ok\":true,\"n\":" + insertColumns(db, type, h, spec, NativeExtras.FailureToken.of(fid.longValue()).onError) + "}"; } /** * header: {"n":rows,"commit":every,"cols":[[name,kind,dim],..]}. Kinds: q (int64), d (float64), b (bool byte), i (int32), e (float32), * f (float32 matrix, dim floats per row), s (UTF-8 string: int32 offsets[n+1] in the data slot, the bytes they index in the aux slot, as - * an Arrow string array; offsets[0] need not be 0). spec: per column six longs, [data address, data bytes, aux address, aux bytes, null + * an Arrow string array; offsets[0] need not be 0). The kinds that the JSON rows carry as tagged values (Json.decodeTag), read as the same + * objects: T (int64 ticks since 1970-01-01T00:00:00 of the UTC wall clock, dim = ticks per second: 1, 1000, 10^6 or 10^9, read as a + * LocalDateTime), D (int32 days since the epoch, a LocalDate), n (a decimal as text in the string layout, a BigDecimal) and x (bytes in + * the string layout, a byte[]), so a column of these is stored as the row path stores it. spec: per column six longs, [data address, data bytes, aux address, aux bytes, null * bitmap address, null bitmap bytes]; the bitmap is Arrow's (bit i of byte i/8, least significant first; 1 = present), an address of 0 * means no nulls. The memory belongs to the caller for the duration of the call; nothing is kept. Every length is checked here before * the first document is built, so a wrong table is an exception and nothing is written. Same transaction contract as * DocumentBatcher.insertColumns: batch commits and the rollback on failure apply only to a transaction opened here. + * onError null: the synchronous mode above. Not null (header "parallel":true): every record goes to db.async().createRecord with that + * callback and no transaction is touched, as DocumentBatcher.insertColumnsParallel does; an edge type is refused there, as it is by + * insert_parallel. The columns are validated before the first record is handed over in both modes. */ @SuppressWarnings("unchecked") - private static long insertColumns(final Database db, final String type, final Map h, final long[] spec) { + private static long insertColumns(final Database db, final String type, final Map h, final long[] spec, + final ErrorCallback onError) { final long nRows = ((Number) h.get("n")).longValue(); if (nRows < 0 || nRows > Integer.MAX_VALUE) throw new IllegalArgumentException("row count out of range: " + nRows); @@ -102,13 +130,18 @@ private static long insertColumns(final Database db, final String type, final Ma final long need = switch (kinds[c]) { case 'q', 'd' -> 8L * n; case 'b' -> n; - case 'i', 'e' -> 4L * n; + case 'i', 'e', 'D' -> 4L * n; + case 'T' -> { + if (dims[c] != 1 && dims[c] != 1_000 && dims[c] != 1_000_000 && dims[c] != 1_000_000_000) + throw new IllegalArgumentException("column '" + names[c] + "': a date-time column has 1, 1000, 10^6 or 10^9 ticks per second, not " + dims[c]); + yield 8L * n; + } case 'f' -> { if (dims[c] <= 0) throw new IllegalArgumentException("column '" + names[c] + "': a vector column needs a dimension above 0"); yield 4L * n * dims[c]; } - case 's' -> 4L * (n + 1L); + case 's', 'n', 'x' -> 4L * (n + 1L); default -> throw new IllegalArgumentException("unknown column kind " + kinds[c]); }; if (dataBytes < need || (need > 0 && dataAddr == 0)) @@ -119,7 +152,7 @@ private static long insertColumns(final Database db, final String type, final Ma throw new IllegalArgumentException("column '" + names[c] + "' has a null bitmap of " + nullBytes + " bytes, " + (n + 7L) / 8 + " needed"); nulls[c] = nullAddr; } - if (kinds[c] == 's') { + if (kinds[c] == 's' || kinds[c] == 'n' || kinds[c] == 'x') { if (auxBytes < 0 || auxBytes > Integer.MAX_VALUE || (auxBytes > 0 && auxAddr == 0)) throw new IllegalArgumentException("column '" + names[c] + "': the string bytes are missing or beyond 2 GiB"); blobs[c] = auxBytes == 0 ? ByteBuffer.allocate(0) : CTypeConversion.asByteBuffer(WordFactory.pointer(auxAddr), (int) auxBytes); @@ -137,64 +170,157 @@ private static long insertColumns(final Database db, final String type, final Ma } } final byte[] scratch = new byte[longest]; - final boolean vertex = db.getSchema().getType(type) instanceof VertexType; - final boolean wasActive = db.isTransactionActive(); - if (!wasActive) - db.begin(); - try { - for (int i = 0; i < n; i++) { - final MutableDocument doc = vertex ? db.newVertex(type) : db.newDocument(type); - for (int c = 0; c < k; c++) { - final Pointer base = WordFactory.pointer(data[c]); - if (nulls[c] != 0 && ((WordFactory.pointer(nulls[c]).readByte(WordFactory.signed(i >> 3)) >> (i & 7)) & 1) == 0) { - doc.set(names[c], null); - continue; - } - switch (kinds[c]) { - case 'q' -> doc.set(names[c], base.readLong(WordFactory.signed(8L * i))); - case 'd' -> doc.set(names[c], base.readDouble(WordFactory.signed(8L * i))); - case 'b' -> doc.set(names[c], base.readByte(WordFactory.signed(i)) != 0); - case 'i' -> doc.set(names[c], base.readInt(WordFactory.signed(4L * i))); - case 'e' -> doc.set(names[c], base.readFloat(WordFactory.signed(4L * i))); - case 's' -> { - final int from = base.readInt(WordFactory.signed(4L * i)); - final int len = base.readInt(WordFactory.signed(4L * (i + 1L))) - from; - if (len == 0) - doc.set(names[c], ""); - else { - blobs[c].get(from, scratch, 0, len); - doc.set(names[c], new String(scratch, 0, len, StandardCharsets.UTF_8)); - } - } - default -> { - final int dim = dims[c]; - final float[] v = new float[dim]; - final long at = 4L * i * dim; - for (int j = 0; j < dim; j++) - v[j] = base.readFloat(WordFactory.signed(at + 4L * j)); - doc.set(names[c], v); - } - } + final Object schemaType = db.getSchema().getType(type); + final boolean vertex = schemaType instanceof VertexType; + if (onError != null) { + if (schemaType instanceof EdgeType) + throw new IllegalArgumentException("Cannot bulk-insert into '" + type + + "' because it is an edge type; load the vertices, then create the edges (graph_batch().new_edges() or CREATE EDGE)"); + final DatabaseAsyncExecutor async = db.async(); + int handed = 0; + try { + for (; handed < n; handed++) { + final MutableDocument doc = vertex ? db.newVertex(type) : db.newDocument(type); + setRow(doc, handed, names, kinds, dims, data, nulls, blobs, scratch); + async.createRecord(doc, null, onError); } - doc.save(); - if (!wasActive && commitEvery > 0 && (i + 1) % commitEvery == 0) { - db.commit(); + } catch (final Throwable t) { + // the records handed over before the failure are the writers' now, whatever this call does + throw new IllegalStateException("after handing " + handed + " of " + n + " rows to the async writers (those may have been stored): " + t, t); + } + return n; + } + writeChunks(db, n, commitEvery, i -> { + final MutableDocument doc = vertex ? db.newVertex(type) : db.newDocument(type); + setRow(doc, i, names, kinds, dims, data, nulls, blobs, scratch); + doc.save(); + }); + return n; + } + + /** + * The retries of a chunk and the back-off before them are the engine's GraphBatch defaults (commitRetries 10, commitRetryDelayMs 1000, + * doubling up to 10 s), so a load that went through createVertices before keeps the same patience. + */ + private static final int COMMIT_RETRIES = 10; + private static final long COMMIT_RETRY_DELAY_MS = 1_000L; + private static final long MAX_COMMIT_RETRY_DELAY_MS = 10_000L; + + /** + * Writes rows [0, n) with writeRow (build and save row i), in transactions opened here: one per commitEvery rows (0: one for all). A + * chunk whose save or commit raises a NeedRetryException (an MVCC conflict with another writer, a lock timeout) is rolled back and + * written again from its first row after a back-off, up to COMMIT_RETRIES times; after the last attempt the error is thrown unchanged. + * Any other failure rolls the open chunk back and is thrown; the chunks committed before it stay. A transaction the caller opened is the + * caller's: the rows join it, nothing is committed, rolled back or retried here. + */ + private static void writeChunks(final Database db, final int n, final int commitEvery, final IntConsumer writeRow) { + if (db.isTransactionActive()) { + for (int i = 0; i < n; i++) + writeRow.accept(i); + return; + } + for (int from = 0; from < n;) { + final int to = commitEvery > 0 ? (int) Math.min((long) n, (long) from + commitEvery) : n; + for (int attempt = 0;;) { + try { db.begin(); + for (int i = from; i < to; i++) + writeRow.accept(i); + db.commit(); + break; + } catch (final NeedRetryException e) { + if (!rollbackQuietly(db, e) || ++attempt > COMMIT_RETRIES) + throw e; + backoffBeforeRetry(attempt); + } catch (final Throwable e) { + rollbackQuietly(db, e); + throw e; } } - if (!wasActive) - db.commit(); - } catch (final Throwable e) { - if (!wasActive && db.isTransactionActive()) { - try { - db.rollback(); - } catch (final Throwable rollbackError) { - e.addSuppressed(rollbackError); + from = to; + } + } + + /** Rolls back the open transaction, if any; false when the rollback itself failed (the failure is added to e as suppressed). */ + private static boolean rollbackQuietly(final Database db, final Throwable e) { + if (!db.isTransactionActive()) + return true; + try { + db.rollback(); + return true; + } catch (final Throwable rollbackError) { + e.addSuppressed(rollbackError); + return false; + } + } + + private static void backoffBeforeRetry(final int attempt) { + long delay = COMMIT_RETRY_DELAY_MS; + for (int i = 1; i < attempt && delay < MAX_COMMIT_RETRY_DELAY_MS; i++) + delay = Math.min(delay * 2, MAX_COMMIT_RETRY_DELAY_MS); + try { + Thread.sleep(delay); + } catch (final InterruptedException ie) { + Thread.currentThread().interrupt(); + throw new NeedRetryException("Interrupted while waiting to retry the commit of a bulk-insert chunk"); + } + } + + /** Sets the properties of row i on doc from the columns: the one place that reads a value out of a buffer. */ + private static void setRow(final MutableDocument doc, final int i, final String[] names, final char[] kinds, final int[] dims, + final long[] data, final long[] nulls, final ByteBuffer[] blobs, final byte[] scratch) { + final int k = names.length; + for (int c = 0; c < k; c++) { + final Pointer base = WordFactory.pointer(data[c]); + if (nulls[c] != 0 && ((WordFactory.pointer(nulls[c]).readByte(WordFactory.signed(i >> 3)) >> (i & 7)) & 1) == 0) { + doc.set(names[c], null); + continue; + } + switch (kinds[c]) { + case 'q' -> doc.set(names[c], base.readLong(WordFactory.signed(8L * i))); + case 'd' -> doc.set(names[c], base.readDouble(WordFactory.signed(8L * i))); + case 'b' -> doc.set(names[c], base.readByte(WordFactory.signed(i)) != 0); + case 'i' -> doc.set(names[c], base.readInt(WordFactory.signed(4L * i))); + case 'e' -> doc.set(names[c], base.readFloat(WordFactory.signed(4L * i))); + case 'T' -> { + final long ticks = base.readLong(WordFactory.signed(8L * i)); + final long perSecond = dims[c]; + doc.set(names[c], LocalDateTime.ofEpochSecond(Math.floorDiv(ticks, perSecond), + (int) (Math.floorMod(ticks, perSecond) * (1_000_000_000L / perSecond)), ZoneOffset.UTC)); + } + case 'D' -> doc.set(names[c], LocalDate.ofEpochDay(base.readInt(WordFactory.signed(4L * i)))); + case 'n' -> { + final int from = base.readInt(WordFactory.signed(4L * i)); + final int len = base.readInt(WordFactory.signed(4L * (i + 1L))) - from; + blobs[c].get(from, scratch, 0, len); + doc.set(names[c], new BigDecimal(new String(scratch, 0, len, StandardCharsets.US_ASCII))); + } + case 'x' -> { + final int from = base.readInt(WordFactory.signed(4L * i)); + final byte[] bytes = new byte[base.readInt(WordFactory.signed(4L * (i + 1L))) - from]; + blobs[c].get(from, bytes, 0, bytes.length); + doc.set(names[c], bytes); + } + case 's' -> { + final int from = base.readInt(WordFactory.signed(4L * i)); + final int len = base.readInt(WordFactory.signed(4L * (i + 1L))) - from; + if (len == 0) + doc.set(names[c], ""); + else { + blobs[c].get(from, scratch, 0, len); + doc.set(names[c], new String(scratch, 0, len, StandardCharsets.UTF_8)); + } + } + default -> { + final int dim = dims[c]; + final float[] v = new float[dim]; + final long at = 4L * i * dim; + for (int j = 0; j < dim; j++) + v[j] = base.readFloat(WordFactory.signed(at + 4L * j)); + doc.set(names[c], v); } } - throw e; } - return n; } /** diff --git a/native/src/main/java/org/humemai/nativelib/ColumnFrames.java b/native/src/main/java/org/humemai/nativelib/ColumnFrames.java new file mode 100644 index 0000000..02d9106 --- /dev/null +++ b/native/src/main/java/org/humemai/nativelib/ColumnFrames.java @@ -0,0 +1,380 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * Result rows as one column frame for the Python native backend (arcadedb_n_next_col): the layout the JVM wheel's ColumnBatcher + * writes, so one Python decoder (arcadedb/_columnar.py) reads both. A frame is + * + * int32 LE header length, header JSON, then for each column its null bitmap and its buffer + * + * The header is {"count":n,"done":bool,"cols":[{"name":..,"type":..,"nulls":bytes,"bytes":bytes[,"dim":d]},..]} ("done" is an addition + * to ColumnBatcher's header, which the decoder ignores). A null bitmap has bit i of byte i/8 set for a null row. A column type is + * + * i8 int64 LE (Long, Integer, Short, Byte) f8 float64 LE (Double, Float; a Float is its exact double) + * b1 one byte per row dt int64 epoch milliseconds, dtu: epoch microseconds (see micros) + * f4v dim float32 LE per row (float[]) f8v dim float64 LE per row (double[]) + * str int32 LE offsets (n + 1) then UTF-8 bytes dec a BigDecimal as text in the str layout + * json one JSON array of the column in the tagged encoding of NativeCodec: what no fixed layout carries without changing a value + * (nested lists and maps, records and RIDs, BigInteger, a column that mixes kinds, a DATE or a time-zone-aware temporal next to + * datetimes with micros, a ragged vector, a Long beyond 2**53 next to a floating value) + * + * A frame holds at most {@code max} rows and stops once the rows read add up to about FRAME_BYTES, so the working set of a read is a frame + * and not the table. + */ +package org.humemai.nativelib; + +import com.arcadedb.query.sql.executor.Result; +import com.arcadedb.query.sql.executor.ResultSet; + +import java.io.ByteArrayOutputStream; +import java.math.BigDecimal; +import java.nio.ByteBuffer; +import java.nio.ByteOrder; +import java.nio.charset.StandardCharsets; +import java.time.Instant; +import java.time.LocalDate; +import java.time.LocalDateTime; +import java.time.OffsetDateTime; +import java.time.ZoneOffset; +import java.time.ZonedDateTime; +import java.util.ArrayList; +import java.util.Calendar; +import java.util.Collection; +import java.util.Date; +import java.util.LinkedHashSet; +import java.util.List; +import java.util.Map; +import java.util.Set; + +final class ColumnFrames { + /** The rows of a frame add up to about this many bytes of values. */ + static final long FRAME_BYTES = 1L << 20; + private static final long TWO53 = 1L << 53; + + private ColumnFrames() { + } + + /** One frame and whether the rows are over (the cursor was read to its end). */ + static final class Frame { + final byte[] bytes; + final boolean done; + + Frame(final byte[] bytes, final boolean done) { + this.bytes = bytes; + this.done = done; + } + } + + private static long toEpochMillis(final Object v) { + if (v instanceof Date d) + return d.getTime(); + if (v instanceof Calendar c) + return c.getTimeInMillis(); + if (v instanceof LocalDateTime ldt) + return ldt.toInstant(ZoneOffset.UTC).toEpochMilli(); + if (v instanceof LocalDate ld) + return ld.atStartOfDay().toInstant(ZoneOffset.UTC).toEpochMilli(); + if (v instanceof Instant i) + return i.toEpochMilli(); + if (v instanceof ZonedDateTime z) + return z.toInstant().toEpochMilli(); + if (v instanceof OffsetDateTime o) + return o.toInstant().toEpochMilli(); + throw new IllegalArgumentException("not a temporal: " + v.getClass()); + } + + private static long toEpochMicros(final Object v) { + if (v instanceof LocalDateTime ldt) + return Math.addExact(Math.multiplyExact(ldt.toEpochSecond(ZoneOffset.UTC), 1_000_000L), ldt.getNano() / 1000); + if (v instanceof Date d) + return Math.multiplyExact(d.getTime(), 1000L); + if (v instanceof Calendar c) + return Math.multiplyExact(c.getTimeInMillis(), 1000L); + throw new IllegalArgumentException("not a naive datetime: " + v.getClass()); + } + + private static boolean yearFits(final LocalDateTime ldt) { + return ldt.getYear() >= 1 && ldt.getYear() <= 9999; + } + + /** About how many bytes the values of a row take in a frame (a stop rule, not an exact size). */ + private static long estimate(final Result row) { + long n = 0; + for (final String name : row.getPropertyNames()) { + final Object v = row.getProperty(name); + if (v == null) + n += 1; + else if (v instanceof String s) + n += s.length() + 4; + else if (v instanceof float[] a) + n += 4L * a.length; + else if (v instanceof double[] a) + n += 8L * a.length; + else if (v instanceof Number || v instanceof Boolean || v instanceof Date || v instanceof LocalDate || v instanceof LocalDateTime + || v instanceof Instant) + n += 8; + else if (v instanceof Collection c) + n += 16 + 16L * c.size(); + else if (v instanceof Map m) + n += 16 + 32L * m.size(); + else + n += 32; + } + return n; + } + + /** + * The next frame of the cursor. {@code pinned} is the column set, or null to take the union of the names of the rows of this frame, in + * order of first appearance. With {@code micros} a naive datetime column is epoch microseconds ("dtu") and a column with a DATE, a + * time-zone-aware temporal or a year Python cannot hold is "json". + * + * The frame ends at {@code max} rows or when what it will hold reaches FRAME_BYTES: the larger of the sizes of the values read and of the + * fixed-width buffers (rows times columns times 9 bytes, which a schemaless result with a different property in every row makes much + * bigger than its values), so the allocation of a frame stays near its budget whatever the shape of the rows. A frame has one row at least. + */ + static Frame next(final ResultSet rs, final int max, final String[] pinned, final boolean micros) { + final int cap = Math.max(max, 1); + final List results = new ArrayList<>(Math.min(cap, 1 << 14)); + final Set names = new LinkedHashSet<>(); + String[] previous = null; + long budget = 0; + boolean done = false; + while (results.size() < cap + && Math.max(budget, (long) results.size() * (pinned != null ? pinned.length : names.size()) * 9L) < FRAME_BYTES) { + if (!rs.hasNext()) { + done = true; + break; + } + final Result row = rs.next(); + results.add(row); + budget += estimate(row); + if (pinned == null) { + // rows of one type almost always list the same names in the same order: a row is compared with the one before it name by name + // and only a row that differs is hashed into the set + final Set rowNames = row.getPropertyNames(); + if (previous != null && rowNames.size() == previous.length) { + int i = 0; + for (final String name : rowNames) + if (!name.equals(previous[i++])) { + i = -1; + break; + } + if (i >= 0) + continue; + } + names.addAll(rowNames); + previous = rowNames.toArray(new String[0]); + } + } + if (!done) + done = !rs.hasNext(); + + final String[] columns = pinned != null ? pinned : names.toArray(new String[0]); + final int n = columns.length; + final int count = results.size(); + final List rows = new ArrayList<>(count); + for (final Result row : results) { + final Object[] vals = new Object[n]; + for (int c = 0; c < n; c++) + vals[c] = row.getProperty(columns[c]); + rows.add(vals); + } + + final StringBuilder hdr = new StringBuilder("{\"count\":").append(count).append(",\"done\":").append(done).append(",\"cols\":["); + final ByteArrayOutputStream body = new ByteArrayOutputStream(Math.max(count * n * 8, 16)); + + for (int c = 0; c < n; c++) { + final byte[] nulls = new byte[(count + 7) / 8]; + String type = null; + boolean sawInt = false, sawFloating = false, bigInt = false; + for (int r = 0; r < count; r++) { + final Object v = rows.get(r)[c]; + if (v == null) + continue; + final String t; + if (v instanceof Long || v instanceof Integer || v instanceof Short || v instanceof Byte) { + t = "i8"; + sawInt = true; + final long l = ((Number) v).longValue(); + if (l > TWO53 || l < -TWO53) + bigInt = true; + } else if (v instanceof Double || v instanceof Float) { + t = "f8"; + sawFloating = true; + } else if (v instanceof Boolean) + t = "b1"; + else if (v instanceof Date || v instanceof Calendar || v instanceof LocalDateTime || v instanceof LocalDate + || v instanceof Instant || v instanceof ZonedDateTime || v instanceof OffsetDateTime) { + if (!micros) + t = "dt"; + else if (v instanceof LocalDateTime ldt ? yearFits(ldt) : v instanceof Date || v instanceof Calendar) + t = "dtu"; + else + t = "json"; + } else if (v instanceof String || v instanceof Character) + t = "str"; + else if (v instanceof BigDecimal) + t = "dec"; + else if (v instanceof float[]) + t = "f4v"; + else if (v instanceof double[]) + t = "f8v"; + else + t = "json"; + if (type == null) + type = t; + else if (!type.equals(t)) { + if ((type.equals("i8") && t.equals("f8")) || (type.equals("f8") && t.equals("i8"))) + type = "f8"; + else { + type = "json"; + break; + } + } + } + if (type == null) + type = "str"; // an all-null column + // an integer beyond 2**53 next to a floating value has no exact double: the values stay what they are, in a json column + if (type.equals("f8") && sawInt && sawFloating && bigInt) + type = "json"; + + int vdim = -1; + if (type.equals("f4v") || type.equals("f8v")) { + for (int r = 0; r < count && vdim != -2; r++) { + final Object v = rows.get(r)[c]; + if (v == null) + continue; + final int len = type.equals("f4v") ? ((float[]) v).length : ((double[]) v).length; + if (vdim == -1) + vdim = len; + else if (vdim != len) + vdim = -2; + } + if (vdim < 0 && vdim != -1) + type = "json"; // ragged + else if (vdim == -1) + vdim = 0; + } + + final byte[] colBuf; + if (type.equals("i8") || type.equals("dt") || type.equals("dtu")) { + final ByteBuffer bb = ByteBuffer.allocate(count * 8).order(ByteOrder.LITTLE_ENDIAN); + for (int r = 0; r < count; r++) { + final Object v = rows.get(r)[c]; + if (v == null) { + nulls[r >> 3] |= (1 << (r & 7)); + bb.putLong(0); + } else if (type.equals("dt")) + bb.putLong(toEpochMillis(v)); + else if (type.equals("dtu")) + bb.putLong(toEpochMicros(v)); + else + bb.putLong(((Number) v).longValue()); + } + colBuf = bb.array(); + } else if (type.equals("f8")) { + final ByteBuffer bb = ByteBuffer.allocate(count * 8).order(ByteOrder.LITTLE_ENDIAN); + for (int r = 0; r < count; r++) { + final Object v = rows.get(r)[c]; + if (v == null) { + nulls[r >> 3] |= (1 << (r & 7)); + bb.putDouble(Double.NaN); + } else + bb.putDouble(((Number) v).doubleValue()); + } + colBuf = bb.array(); + } else if (type.equals("b1")) { + final ByteBuffer bb = ByteBuffer.allocate(count); + for (int r = 0; r < count; r++) { + final Object v = rows.get(r)[c]; + if (v == null) { + nulls[r >> 3] |= (1 << (r & 7)); + bb.put((byte) 0); + } else + bb.put((byte) (((Boolean) v) ? 1 : 0)); + } + colBuf = bb.array(); + } else if (type.equals("f4v")) { + final ByteBuffer bb = ByteBuffer.allocate(count * vdim * 4).order(ByteOrder.LITTLE_ENDIAN); + for (int r = 0; r < count; r++) { + final Object v = rows.get(r)[c]; + if (v == null) { + nulls[r >> 3] |= (1 << (r & 7)); + for (int k = 0; k < vdim; k++) + bb.putFloat(Float.NaN); + } else + for (final float x : (float[]) v) + bb.putFloat(x); + } + colBuf = bb.array(); + } else if (type.equals("f8v")) { + final ByteBuffer bb = ByteBuffer.allocate(count * vdim * 8).order(ByteOrder.LITTLE_ENDIAN); + for (int r = 0; r < count; r++) { + final Object v = rows.get(r)[c]; + if (v == null) { + nulls[r >> 3] |= (1 << (r & 7)); + for (int k = 0; k < vdim; k++) + bb.putDouble(Double.NaN); + } else + for (final double x : (double[]) v) + bb.putDouble(x); + } + colBuf = bb.array(); + } else if (type.equals("json")) { + final StringBuilder arr = new StringBuilder(); + arr.append('['); + for (int r = 0; r < count; r++) { + final Object v = rows.get(r)[c]; + if (r > 0) + arr.append(','); + if (v == null) + nulls[r >> 3] |= (1 << (r & 7)); + NativeCodec.append(arr, v); + } + arr.append(']'); + colBuf = arr.toString().getBytes(StandardCharsets.UTF_8); + } else { // str and dec: int32 offsets (count + 1) then the UTF-8 bytes + final ByteArrayOutputStream chars = new ByteArrayOutputStream(Math.max(count * 16, 16)); + final ByteBuffer offs = ByteBuffer.allocate((count + 1) * 4).order(ByteOrder.LITTLE_ENDIAN); + int pos = 0; + offs.putInt(0); + for (int r = 0; r < count; r++) { + final Object v = rows.get(r)[c]; + if (v == null) + nulls[r >> 3] |= (1 << (r & 7)); + else { + final String text = v instanceof BigDecimal d ? d.toPlainString() : v.toString(); + final byte[] b = text.getBytes(StandardCharsets.UTF_8); + chars.write(b, 0, b.length); + pos += b.length; + } + offs.putInt(pos); + } + final byte[] charBytes = chars.toByteArray(); + final ByteBuffer bb = ByteBuffer.allocate(offs.capacity() + charBytes.length); + bb.put(offs.array()); + bb.put(charBytes); + colBuf = bb.array(); + } + + if (c > 0) + hdr.append(','); + hdr.append("{\"name\":"); + NativeCodec.quote(hdr, columns[c]); + hdr.append(",\"type\":\"").append(type).append("\",\"nulls\":").append(nulls.length).append(",\"bytes\":").append(colBuf.length); + if (vdim > 0 && (type.equals("f4v") || type.equals("f8v"))) + hdr.append(",\"dim\":").append(vdim); + hdr.append('}'); + body.write(nulls, 0, nulls.length); + body.write(colBuf, 0, colBuf.length); + } + hdr.append("]}"); + + final byte[] headerBytes = hdr.toString().getBytes(StandardCharsets.UTF_8); + final byte[] bodyBytes = body.toByteArray(); + final ByteBuffer out = ByteBuffer.allocate(4 + headerBytes.length + bodyBytes.length).order(ByteOrder.LITTLE_ENDIAN); + out.putInt(headerBytes.length); + out.put(headerBytes); + out.put(bodyBytes); + return new Frame(out.array(), done); + } +} diff --git a/native/src/main/java/org/humemai/nativelib/NativeCodec.java b/native/src/main/java/org/humemai/nativelib/NativeCodec.java index 59b7c94..b3bbab0 100644 --- a/native/src/main/java/org/humemai/nativelib/NativeCodec.java +++ b/native/src/main/java/org/humemai/nativelib/NativeCodec.java @@ -51,8 +51,38 @@ private NativeCodec() { * Appends {@code "rows":[..],"meta":[..],"drained":b} (no braces) with up to max rows. meta has one entry per row: 0 * for a projection, or [rid, typeName, kind] for a record row (kind v, e or d). */ + /** + * The names of the properties whose value was a float[] (f) or a double[] (d) in a row of a batch. A JSON reply writes both as a list of + * numbers, so the reader cannot tell a vector from a list of doubles: the reply names the vectors, {@code "fv":[..]} and {@code "dv":[..]} + * after "drained" (absent when there is none), and a columnar read types those columns as vectors, as a column frame does. + */ + static final class VecHints { + final java.util.LinkedHashSet f = new java.util.LinkedHashSet<>(); + final java.util.LinkedHashSet d = new java.util.LinkedHashSet<>(); + + void appendTo(final StringBuilder sb) { + appendNames(sb, ",\"fv\":", f); + appendNames(sb, ",\"dv\":", d); + } + + private static void appendNames(final StringBuilder sb, final String key, final java.util.Set names) { + if (names.isEmpty()) + return; + sb.append(key).append('['); + boolean first = true; + for (final String name : names) { + if (!first) + sb.append(','); + first = false; + quote(sb, name); + } + sb.append(']'); + } + } + static boolean appendBatch(final StringBuilder sb, final ResultSet rs, final int max) { final StringBuilder meta = new StringBuilder("["); + final VecHints hints = new VecHints(); sb.append("\"rows\":["); int n = 0; boolean drained = false; @@ -71,7 +101,7 @@ static boolean appendBatch(final StringBuilder sb, final ResultSet rs, final int sb.append(','); meta.append(','); } - appendRow(sb, meta, row); + appendRow(sb, meta, row, hints); } catch (final Throwable t) { sb.setLength(rowStart); meta.setLength(metaStart); @@ -90,10 +120,11 @@ static boolean appendBatch(final StringBuilder sb, final ResultSet rs, final int sb.append("],\"meta\":").append(meta).append("],\"drained\":").append(drained); if (readError != null) sb.append(",\"rerr\":").append(readError); + hints.appendTo(sb); return drained; } - private static void appendRow(final StringBuilder sb, final StringBuilder meta, final Result row) { + private static void appendRow(final StringBuilder sb, final StringBuilder meta, final Result row, final VecHints hints) { sb.append('{'); boolean first = true; for (final String name : row.getPropertyNames()) { @@ -102,7 +133,12 @@ private static void appendRow(final StringBuilder sb, final StringBuilder meta, first = false; quote(sb, name); sb.append(':'); - append(sb, row.getProperty(name)); + final Object value = row.getProperty(name); + if (value instanceof float[]) + hints.f.add(name); + else if (value instanceof double[]) + hints.d.add(name); + append(sb, value); } sb.append('}'); if (row.isElement()) { diff --git a/native/src/main/java/org/humemai/nativelib/NativeExtras.java b/native/src/main/java/org/humemai/nativelib/NativeExtras.java index a62574a..d230919 100644 --- a/native/src/main/java/org/humemai/nativelib/NativeExtras.java +++ b/native/src/main/java/org/humemai/nativelib/NativeExtras.java @@ -73,6 +73,7 @@ import org.graalvm.word.WordFactory; import java.util.ArrayList; +import java.util.Arrays; import java.util.HashMap; import java.util.HashSet; import java.util.List; @@ -250,6 +251,45 @@ static Object[] varargs(final Object props) { return out; } + /** + * The float32 vector columns of create_vertices rows: vecs is {"names": [..], "dims": [..], "offs": [..]} and the side buffer holds, for + * column c, one vector after the other (rows * dims[c] floats) from float offset offs[c]. Row i gets the property names[c] as a float[] + * of its slice, appended to its property array: the float[] that a {"\u0001":["B",offset,count]} tag per row gave, from one matrix and + * no tag per row. A null or empty vecs leaves the rows as they are. + */ + private static void withVectors(final List flats, final Object vecs, final long buf, final long bufFloats) { + if (!(vecs instanceof Map v)) + return; + final List names = (List) v.get("names"), dims = (List) v.get("dims"), offs = (List) v.get("offs"); + final int columns = names.size(), rows = flats.size(); + if (columns == 0) + return; + if (buf == 0 || dims.size() != columns || offs.size() != columns) + throw new IllegalArgumentException("vector columns without a matching side buffer"); + final int[] dim = new int[columns]; + final long[] off = new long[columns]; + for (int c = 0; c < columns; c++) { + dim[c] = ((Number) dims.get(c)).intValue(); + off[c] = ((Number) offs.get(c)).longValue(); + if (dim[c] < 1 || off[c] < 0 || off[c] + (long) rows * dim[c] > bufFloats) + throw new IllegalArgumentException("vector column outside the side buffer"); + } + final CFloatPointer fp = WordFactory.pointer(buf); + for (int i = 0; i < rows; i++) { + final Object[] row = flats.get(i); + final Object[] grown = Arrays.copyOf(row, row.length + 2 * columns); + for (int c = 0; c < columns; c++) { + final float[] f = new float[dim[c]]; + final int base = (int) (off[c] + (long) i * dim[c]); + for (int k = 0; k < dim[c]; k++) + f[k] = fp.read(base + k); + grown[row.length + 2 * c] = names.get(c); + grown[row.length + 2 * c + 1] = f; + } + flats.set(i, grown); + } + } + static String ridList(final RID[] rids) { final StringBuilder sb = new StringBuilder("["); for (int i = 0; i < rids.length; i++) { @@ -488,8 +528,10 @@ else if (opName.equals("create_vertices")) final RID[] rids; if (!(a.get("rows") instanceof Integer)) rids = batch.createVertices((String) a.get("type"), intOf(a.get("count"), 0)); - else + else { + withVectors(flats, a.get("vecs"), buf, bufFloats); rids = batch.createVertices((String) a.get("type"), flats.toArray(new Object[0][])); + } return ArcadeNative.out("{\"ok\":true,\"rids\":" + ridList(rids) + "}"); } case "new_edge" -> batch.newEdge(new RID((String) a.get("src")), (String) a.get("type"), new RID((String) a.get("dst")), @@ -527,6 +569,51 @@ else if (opName.equals("create_vertices")) private static final Map ASYNC_FAILS = new ConcurrentHashMap<>(); private static final Map> ASYNC_FIRST = new ConcurrentHashMap<>(); + /** + * A failure token for one parallel load: the id the caller reads (and so removes) with the "failures" operation, and the callback that + * counts. The caller creates it before the load ("token" operation) and passes the id in the load's header, so that it holds the id + * whatever happens to the call, an interrupt included, and always has the means to wait for the writers and remove the entries. + */ + static final class FailureToken { + final long fid; + final AtomicLong count; + final AtomicReference first; + final ErrorCallback onError; + + private FailureToken(final long fid, final AtomicLong count, final AtomicReference first) { + this.fid = fid; + this.count = count; + this.first = first; + this.onError = t -> { + count.incrementAndGet(); + first.compareAndSet(null, t); + }; + } + + /** A new token, registered for the "failures" operation. */ + static FailureToken create() { + final FailureToken token = new FailureToken(ArcadeNative.IDS.getAndIncrement(), new AtomicLong(), new AtomicReference<>()); + ASYNC_FAILS.put(token.fid, token.count); + ASYNC_FIRST.put(token.fid, token.first); + return token; + } + + /** The token that "token" created, for the load that its id was passed to. */ + static FailureToken of(final long fid) { + final AtomicLong count = ASYNC_FAILS.get(fid); + final AtomicReference first = ASYNC_FIRST.get(fid); + if (count == null || first == null) + throw new IllegalArgumentException("unknown failure token " + fid + " (already read, or never created)"); + return new FailureToken(fid, count, first); + } + + /** The load failed before it returned: the caller gets the error, not a token to read failures from. */ + void drop() { + ASYNC_FAILS.remove(fid); + ASYNC_FIRST.remove(fid); + } + } + private static String replyRows(final ResultSet rs) { final StringBuilder sb = new StringBuilder("{\"ok\":true,\"rs\":0,"); NativeCodec.appendBatch(sb, rs, Integer.MAX_VALUE); @@ -824,15 +911,9 @@ else if (a.get("slot") instanceof Number slot) ex.appendSamples((String) a.get("type"), stamps, columns); } case "insert_parallel" -> { - final long fid = ArcadeNative.IDS.getAndIncrement(); - final AtomicLong count = new AtomicLong(); - final AtomicReference first = new AtomicReference<>(); - ASYNC_FAILS.put(fid, count); - ASYNC_FIRST.put(fid, first); - final ErrorCallback onError = t -> { - count.incrementAndGet(); - first.compareAndSet(null, t); - }; + final FailureToken token = FailureToken.create(); + final long fid = token.fid; + final ErrorCallback onError = token.onError; final String type = (String) a.get("type"); try { final List rows = (List) a.get("rows"); @@ -846,12 +927,14 @@ else if (a.get("slot") instanceof Number slot) ex.createRecord(doc, null, onError); } } catch (final Throwable t) { - ASYNC_FAILS.remove(fid); // the caller gets the error, not a token to read failures from - ASYNC_FIRST.remove(fid); + token.drop(); // the caller gets the error, not a token to read failures from throw t; } return ArcadeNative.out("{\"ok\":true,\"fid\":" + fid + "}"); } + case "token" -> { + return ArcadeNative.out("{\"ok\":true,\"fid\":" + FailureToken.create().fid + "}"); + } case "failures" -> { final long fid = ((Number) a.get("fid")).longValue(); final AtomicLong count = ASYNC_FAILS.remove(fid); diff --git a/native/wheel/parity_crash.py b/native/wheel/parity_crash.py new file mode 100644 index 0000000..51f0629 --- /dev/null +++ b/native/wheel/parity_crash.py @@ -0,0 +1,192 @@ +"""parity_crash.py: what native/wheel/parity_wheel.py does when a test file's process dies, as pure code (no files, no subprocess). + +A test file runs in its own process. Four things can happen to that process: + + result pytest wrote its junit XML. Pass or fail is read from the XML; the exit code is not consulted (a failing test is a + failure, never a crash, and it is never retried). + TIMEOUT the harness killed it after --timeout. Strict, never retried. + BADXML the XML exists and does not parse. Strict, never retried. + crash no XML at all: the process died before pytest wrote its result (access violation, abort, kill by the runner, an + os._exit in a library). The exit code is the only evidence left, so it is recorded: CRASH(rc=0xC0000005,3221225477). + +Retry (off by default, switched on with --retry-crash, which test-native.yml passes on Windows legs only): a crash is run once more in +a fresh process. If the second run produces a result the file is marked FLAKY-CRASH(rc=) and counted from the second run (a +test failure in it still fails the leg); if it crashes again the status is CRASH(rc=;retry rc=) and the leg fails. A +retry budget per leg (default 3 files) keeps a wheel that crashes everywhere from passing on retries. + +A process that wrote its result and then died in shutdown (DIRTY-EXIT) is recorded, never retried and never failed. + +Everything here takes the facts of an attempt (Attempt) and returns the status and the attempts; the harness runs the processes. +""" + +from __future__ import annotations + +import signal +import sys +from dataclasses import dataclass +from typing import Callable, Optional, Tuple + +CRASH = "CRASH" +FLAKY_CRASH = "FLAKY-CRASH" +DIRTY_EXIT = "DIRTY-EXIT" +TIMEOUT = "TIMEOUT" +BADXML = "BADXML" + +# The exit codes pytest itself returns (0 ok, 1 failures, 2 interrupted or collection errors, 3 internal error, 4 usage, 5 nothing collected). +# Any other code after the XML was written means the process died later, in the interpreter's or a library's shutdown. +PYTEST_EXIT_CODES = range(0, 6) + + +def format_rc(rc: Optional[int], windows: Optional[bool] = None) -> str: + """The exit code the way a crash is read: 0xC0000005,3221225477 (Windows NTSTATUS) or -11,SIGSEGV (a POSIX signal). + + Python reports an unsigned 32-bit code for a Windows process and the negative signal number for a POSIX one; a negative code on + Windows is the same NTSTATUS read as signed, so it is folded back to 32 bits there. + """ + if rc is None: + return "none" + if windows is None: + windows = sys.platform == "win32" + if windows or rc >= 0: + return f"0x{rc & 0xFFFFFFFF:08X},{rc & 0xFFFFFFFF}" + try: + name = signal.Signals(-rc).name + except ValueError: + name = f"signal {-rc}" + return f"{rc},{name}" + + +@dataclass(frozen=True) +class Attempt: + """The facts about one run of one test file.""" + + rc: Optional[ + int + ] # the exit code; None when the harness killed the process (timed_out) + result: bool # the junit XML was written and parses + timed_out: bool = False + bad_xml: bool = False # the XML exists and does not parse + + +def is_crash(attempt: Attempt) -> bool: + """No result of any kind and not killed by the harness: the process died on its own.""" + return not (attempt.result or attempt.timed_out or attempt.bad_xml) + + +def dirty_exit(attempt: Attempt) -> bool: + """The XML was written but the process ended with a code pytest never returns: it died in shutdown, after the tests.""" + return ( + attempt.result + and attempt.rc is not None + and attempt.rc not in PYTEST_EXIT_CODES + ) + + +def dirty_events(name: str, outcome: "Outcome") -> list: + """One DIRTY-EXIT record per attempt that wrote its result and then died in shutdown (the metadata, the summary and the issue comment list it). + + It is a record and nothing else: the status of the file is what its XML says (the outcome is not retried and the leg is not failed). + The log of such an attempt is the file's final log, name.log. + """ + return [ + { + "file": name, + "kind": DIRTY_EXIT, + "first_rc": format_rc(a.rc), + "retry_rc": "", + "retry": "not retried, result kept", + "status": outcome.status, + "attempt": n, + "log": f"{name}.log", + } + for n, a in enumerate(outcome.attempts, 1) + if dirty_exit(a) + ] + + +@dataclass +class RetryPolicy: + """Whether a crashed file may be run again, and how many files of one leg may be.""" + + enabled: bool = False + budget: int = 3 + used: int = 0 + + def take(self) -> bool: + if not self.enabled or self.used >= self.budget: + return False + self.used += 1 + return True + + +@dataclass(frozen=True) +class Outcome: + attempts: Tuple[Attempt, ...] + status: str # "" when the file produced a result; else TIMEOUT, BADXML, CRASH(...) or FLAKY-CRASH(...) + + @property + def final(self) -> Attempt: + return self.attempts[-1] + + @property + def retried(self) -> bool: + return len(self.attempts) > 1 + + @property + def flaky(self) -> bool: + return self.status.startswith(FLAKY_CRASH) + + @property + def hard_crash(self) -> bool: + return self.status.startswith(CRASH) + + @property + def timed_out(self) -> bool: + return self.final.timed_out + + +def run_file(run_attempt: Callable[[int], Attempt], policy: RetryPolicy) -> Outcome: + """Run one test file by `run_attempt(1)` and, only for a crash and only when the policy allows, once more by `run_attempt(2)`.""" + first = run_attempt(1) + if not is_crash(first): + return Outcome( + (first,), TIMEOUT if first.timed_out else BADXML if first.bad_xml else "" + ) + crash = f"rc={format_rc(first.rc)}" + if not policy.take(): + why = ";retry budget spent" if policy.enabled else "" + return Outcome((first,), f"{CRASH}({crash}{why})") + second = run_attempt(2) + if second.result: + return Outcome((first, second), f"{FLAKY_CRASH}({crash})") + if second.timed_out: + again = "TIMEOUT" + elif second.bad_xml: + again = "BADXML" + else: + again = f"rc={format_rc(second.rc)}" + return Outcome((first, second), f"{CRASH}({crash};retry {again})") + + +def event_of(name: str, outcome: Outcome, final_failed: int) -> Optional[dict]: + """The record of a crash for the metadata file (parity-wheel.meta.json), or None for a file that never crashed. + + `final_failed` is the number of failed or errored tests of the run that produced the result (0 when there is none). + """ + if not (outcome.flaky or outcome.hard_crash): + return None + first, last = outcome.attempts[0], outcome.final + if outcome.flaky: + retry = "passed" if final_failed == 0 else f"ran, {final_failed} failed" + elif outcome.retried: + retry = "crashed again" + else: + retry = "not retried" + return { + "file": name, + "kind": FLAKY_CRASH if outcome.flaky else CRASH, + "first_rc": format_rc(first.rc), + "retry_rc": format_rc(last.rc) if outcome.retried else "", + "retry": retry, + "status": outcome.status, + } diff --git a/native/wheel/parity_wheel.py b/native/wheel/parity_wheel.py index 8cc094f..3fe363c 100644 --- a/native/wheel/parity_wheel.py +++ b/native/wheel/parity_wheel.py @@ -6,172 +6,357 @@ A run per test file in its own process (a hang or crash costs one file), junit XML per file, then one table: passed / failed / error / NOTYET (the test needs a Java-object API, reported by tests/conftest.py as a skip whose reason starts with NOTYET, raised as NotOnNativeYet) / NA-JVM (a row of NATIVE_NA_JVM_TESTS: the test reaches into the JVM on purpose) / other skips / TIMEOUT-or-crash. Files in tests/conftest.py's NATIVE_NOT_APPLICABLE are not collected (JVM-only). + +A file whose process dies without writing its result is a CRASH, recorded with its exit code: CRASH(rc=0xC0000005,3221225477). With +--retry-crash (test-native.yml passes it on Windows legs only; the default is strict) the file is run once more in a fresh process: a retry +that produces a result is FLAKY-CRASH(rc=...) in the status column (the gate lets it pass, the job summary and issue 295 record it), a second +crash stays CRASH and fails the leg. A failing test, a TIMEOUT and a BADXML are never retried. The decision is native/wheel/parity_crash.py. +On Windows each process also writes a faulthandler trace to .attemptN.fault.txt next to its XML (tests/conftest.py, +ARCADEDB_TEST_FAULTHANDLER_FILE); an empty one is deleted. Beside the table, .meta.json lists the wheel, the Python and every crash. """ import argparse +import ast import glob +import json import os import shutil import subprocess import sys +import time import xml.etree.ElementTree as ET -p = argparse.ArgumentParser() -p.add_argument("--repo", required=True) -p.add_argument("--python", default=sys.executable) -p.add_argument("--out", required=True) -p.add_argument("--timeout", type=int, default=900) -p.add_argument( - "--cores", - default="0-5,8-11", - help="taskset core list (Linux only; ignored where taskset does not exist)", -) -p.add_argument("--files", default="") -p.add_argument( - "--max-notyet", - type=int, - default=None, - help="fail (exit 1) when more tests than this are NOTYET skips; a final release uses 0", -) -a = p.parse_args() - -pkg = os.path.abspath(a.repo) -files = sorted(glob.glob(os.path.join(pkg, "tests", "test_*.py"))) -import ast +import parity_crash as pc # the directory of this script is sys.path[0] + +COUNT_KEYS = ["passed", "failed", "error", "notyet", "na_jvm", "skipped"] + +# On Windows the test process writes a faulthandler trace to this file (tests/conftest.py, native backend only). +FAULT_FILE_ENV = "ARCADEDB_TEST_FAULTHANDLER_FILE" + + +def parse_args(argv=None): + p = argparse.ArgumentParser() + p.add_argument("--repo", required=True) + p.add_argument("--python", default=sys.executable) + p.add_argument("--out", required=True) + p.add_argument("--timeout", type=int, default=900) + p.add_argument( + "--cores", + default="0-5,8-11", + help="taskset core list (Linux only; ignored where taskset does not exist)", + ) + p.add_argument("--files", default="") + p.add_argument( + "--max-notyet", + type=int, + default=None, + help="fail (exit 1) when more tests than this are NOTYET skips; a final release uses 0", + ) + p.add_argument( + "--retry-crash", + action="store_true", + help="run a test file whose process died without a result ONCE more in a fresh process (FLAKY-CRASH when that " + "produces a result, CRASH when it dies again). Off by default: the workflow sets it on Windows legs only.", + ) + p.add_argument( + "--max-crash-retries", + type=int, + default=3, + help="with --retry-crash, the most files of one run that may be retried; a crash after that stays a CRASH", + ) + return p.parse_args(argv) + + +# A file that was closed a moment ago can still be locked on Windows (Defender, the indexer). Housekeeping on logs and traces waits this long once. +HOUSEKEEPING_PAUSE = 0.5 + + +def housekeeping(what, op, *args): + """op(*args); on an OSError once more after a short pause; a second failure prints a ::warning:: and returns False. Never raises. + + Only for the files the harness keeps for people (logs, traces, a stale XML): losing one of them must never fail the leg or hide a result. + """ + for attempt in (1, 2): + try: + op(*args) + return True + except OSError as err: + if attempt == 1: + time.sleep(HOUSEKEEPING_PAUSE) + else: + print( + f"::warning::{what} was skipped: {type(err).__name__}: {err}", + flush=True, + ) + return False + + +def write_text(path, text): + with open(path, "w") as fh: + fh.write(text) + + +def traces_of(name, xmldir, attempts): + """{'trace1': file name, ...} for the faulthandler trace files that exist (an empty one was deleted).""" + found = {} + for n in attempts: + trace = f"{name}.attempt{n}.fault.txt" + if os.path.exists(os.path.join(xmldir, trace)): + found[f"trace{n}"] = trace + return found -na_files, win_files = {}, {} -for node in ast.parse(open(os.path.join(pkg, "tests", "conftest.py")).read()).body: - if isinstance(node, ast.Assign): - name = getattr(node.targets[0], "id", "") - if name == "NATIVE_NOT_APPLICABLE": - na_files = ast.literal_eval(node.value) - elif name == "WINDOWS_NOT_APPLICABLE": - win_files = ast.literal_eval(node.value) -files = [f for f in files if os.path.basename(f) not in na_files] -if ( - sys.platform == "win32" -): # conftest.py's collect_ignore does not apply to a file named on the command line - files = [f for f in files if os.path.basename(f) not in win_files] -if not os.path.isfile(os.path.join(pkg, "docs", "index.md")): - # tests/conftest.py collect-ignores test_docs_examples.py without the Markdown docs site; a file named on the command line is collected anyway - files = [f for f in files if os.path.basename(f) != "test_docs_examples.py"] -if a.files: - want = set(a.files.split(",")) - files = [f for f in files if os.path.basename(f) in want] -env = dict(os.environ, PYTHONUNBUFFERED="1") -env.pop("PYTHONPATH", None) # the wheel, not a source tree, must be what is imported -env.pop("ARCADEDB_BACKEND", None) # the backend comes from the installed distribution -env.pop("ARCADEDB_NATIVE_LIB", None) # the library comes from the installed wheel -chk = subprocess.run( - [ - a.python, - "-I", - "-c", - "import arcadedb as n; print(n.__file__, n.__version__, n.__backend__, n.ENGINE_VERSION, n.SOURCE_COMMIT); " - 'assert n.__backend__ == "native"; assert "site-packages" in n.__file__', - ], - env=env, - cwd="/", - capture_output=True, - text=True, -) -print("WHEEL CHECK:", chk.stdout.strip(), chk.stderr.strip()[-300:], flush=True) -assert chk.returncode == 0, "tests would not run against the installed wheel" -a.out = os.path.abspath(a.out) -xmldir = os.path.splitext(a.out)[0] + "-xml" -os.makedirs(xmldir, exist_ok=True) -rows, tot = [], dict.fromkeys( - ["passed", "failed", "error", "notyet", "na_jvm", "skipped", "timeout"], 0 -) -for f in files: - name = os.path.basename(f) - xml = os.path.join(xmldir, name + ".xml") - cmd = [ - *( - ["taskset", "-c", a.cores] - if sys.platform == "linux" and shutil.which("taskset") - else [] - ), - *( - ["nice", "-n", "10"] - if sys.platform != "win32" and shutil.which("nice") - else [] - ), - a.python, - "-m", - "pytest", - # No --capture=sys on Windows: pytest's default fd capture (dup2 onto fd 1 and 2) is the regression check for engine log - # text landing in database files there (tests/test_log_redirect.py is the direct test; the whole-file runs of test_vector.py - # were the ones that failed before the engine stopped logging to the console on Windows). - f"tests/{name}", - "-q", - "-p", - "no:cacheprovider", - f"--junitxml={xml}", - "-o", - "junit_family=xunit1", - ] + +def count_xml(xml): + """(counts, names of the failing tests) of a junit XML, or None when it does not parse.""" + c = dict.fromkeys(COUNT_KEYS, 0) + fails = [] try: - r = subprocess.run( - cmd, cwd=pkg, env=env, capture_output=True, text=True, timeout=a.timeout + for tc in ET.parse(xml).getroot().iter("testcase"): + if tc.find("failure") is not None: + c["failed"] += 1 + fails.append(tc.get("name")) + elif tc.find("error") is not None: + c["error"] += 1 + fails.append(tc.get("name")) + elif tc.find("skipped") is not None: + msg = tc.find("skipped").get("message") or "" + c[ + ( + "notyet" + if msg.startswith("NOTYET") + else "na_jvm" if msg.startswith("NA-JVM") else "skipped" + ) + ] += 1 + else: + c["passed"] += 1 + except ET.ParseError: + return None + return c, fails + + +def main(argv=None): + a = parse_args(argv) + pkg = os.path.abspath(a.repo) + files = sorted(glob.glob(os.path.join(pkg, "tests", "test_*.py"))) + + na_files, win_files = {}, {} + for node in ast.parse(open(os.path.join(pkg, "tests", "conftest.py")).read()).body: + if isinstance(node, ast.Assign): + name = getattr(node.targets[0], "id", "") + if name == "NATIVE_NOT_APPLICABLE": + na_files = ast.literal_eval(node.value) + elif name == "WINDOWS_NOT_APPLICABLE": + win_files = ast.literal_eval(node.value) + files = [f for f in files if os.path.basename(f) not in na_files] + if ( + sys.platform == "win32" + ): # conftest.py's collect_ignore does not apply to a file named on the command line + files = [f for f in files if os.path.basename(f) not in win_files] + if not os.path.isfile(os.path.join(pkg, "docs", "index.md")): + # tests/conftest.py collect-ignores test_docs_examples.py without the Markdown docs site; a file named on the command line is collected anyway + files = [f for f in files if os.path.basename(f) != "test_docs_examples.py"] + if a.files: + want = set(a.files.split(",")) + files = [f for f in files if os.path.basename(f) in want] + env = dict(os.environ, PYTHONUNBUFFERED="1") + env.pop( + "PYTHONPATH", None + ) # the wheel, not a source tree, must be what is imported + env.pop( + "ARCADEDB_BACKEND", None + ) # the backend comes from the installed distribution + env.pop("ARCADEDB_NATIVE_LIB", None) # the library comes from the installed wheel + env.pop(FAULT_FILE_ENV, None) # set per attempt below, on Windows only + chk = subprocess.run( + [ + a.python, + "-I", + "-c", + "import json, sys, arcadedb as n; print(n.__file__, n.__version__, n.__backend__, n.ENGINE_VERSION, n.SOURCE_COMMIT); " + 'print("META", json.dumps({"python": sys.version.split()[0], "wheel_version": n.__version__, "engine_version": n.ENGINE_VERSION, "source_commit": n.SOURCE_COMMIT})); ' + 'assert n.__backend__ == "native"; assert "site-packages" in n.__file__', + ], + env=env, + cwd="/", + capture_output=True, + text=True, + ) + print("WHEEL CHECK:", chk.stdout.strip(), chk.stderr.strip()[-300:], flush=True) + assert chk.returncode == 0, "tests would not run against the installed wheel" + ident = {} + for line in chk.stdout.splitlines(): + if line.startswith("META "): + ident = json.loads(line[len("META ") :]) + a.out = os.path.abspath(a.out) + xmldir = os.path.splitext(a.out)[0] + "-xml" + os.makedirs(xmldir, exist_ok=True) + policy = pc.RetryPolicy(enabled=a.retry_crash, budget=a.max_crash_retries) + if policy.enabled: + print( + f"CRASH RETRY: on, a file that dies without a result is run once more (at most {policy.budget} files)", + flush=True, ) - status = "" - open(os.path.join(xmldir, name + ".log"), "w").write( - r.stdout[-20000:] + "\n" + r.stderr[-5000:] + rows, events = [], [] + tot = dict.fromkeys([*COUNT_KEYS, "timeout", "flaky_crash", "crash"], 0) + + for f in files: + name = os.path.basename(f) + xml = os.path.join(xmldir, name + ".xml") + log = os.path.join(xmldir, name + ".log") + parsed = {} # attempt number -> (counts, failing tests) of its XML + + def attempt(n): + if n > 1: + # keep the log of the first attempt: the retry writes name.log + housekeeping( + f"keeping the log of the first attempt of {name}", + os.replace, + log, + os.path.join(xmldir, name + ".attempt1.log"), + ) + stale = False + if os.path.exists(xml): + # a stale XML of an earlier run must not pass for a result + stale = not housekeeping(f"removing a stale {name}.xml", os.remove, xml) + started = time.time() + trace = os.path.join(xmldir, f"{name}.attempt{n}.fault.txt") + env_n = dict(env) + if sys.platform == "win32": + env_n[FAULT_FILE_ENV] = trace + cmd = [ + *( + ["taskset", "-c", a.cores] + if sys.platform == "linux" and shutil.which("taskset") + else [] + ), + *( + ["nice", "-n", "10"] + if sys.platform != "win32" and shutil.which("nice") + else [] + ), + a.python, + "-m", + "pytest", + # No --capture=sys on Windows: pytest's default fd capture (dup2 onto fd 1 and 2) is the regression check for engine log + # text landing in database files there (tests/test_log_redirect.py is the direct test; the whole-file runs of test_vector.py + # were the ones that failed before the engine stopped logging to the console on Windows). + f"tests/{name}", + "-q", + "-p", + "no:cacheprovider", + f"--junitxml={xml}", + "-o", + "junit_family=xunit1", + ] + timed_out, rc = False, None + try: + r = subprocess.run( + cmd, + cwd=pkg, + env=env_n, + capture_output=True, + text=True, + timeout=a.timeout, + ) + rc = r.returncode + housekeeping( + f"writing the log of {name}", + write_text, + log, + r.stdout[-20000:] + "\n" + r.stderr[-5000:], + ) + except subprocess.TimeoutExpired as e: + timed_out = True + housekeeping( + f"writing the log of {name}", + write_text, + log, + ( + ((e.stdout or b"")[-8000:]).decode() + if isinstance(e.stdout, bytes) + else str(e.stdout)[-8000:] + ), + ) + if os.path.exists(trace) and os.path.getsize(trace) == 0: + housekeeping(f"removing the empty trace of {name}", os.remove, trace) + result, bad_xml = False, False + # an XML that could not be removed and was not written again by this attempt is not its result + if os.path.exists(xml) and not ( + stale and os.path.getmtime(xml) < started - 2 + ): + got = count_xml(xml) + if got is None: + bad_xml = True + else: + result, parsed[n] = True, got + return pc.Attempt( + rc=rc, result=result, timed_out=timed_out, bad_xml=bad_xml + ) + + out = pc.run_file(attempt, policy) + c, fails = parsed.get(len(out.attempts), (dict.fromkeys(COUNT_KEYS, 0), [])) + for k in c: + tot[k] += c[k] + tot["timeout"] += out.timed_out + tot["flaky_crash"] += out.flaky + tot["crash"] += out.hard_crash + ev = pc.event_of(name, out, c["failed"] + c["error"]) + if ev: + ev.update(traces_of(name, xmldir, range(1, len(out.attempts) + 1))) + events.append(ev) + # a process that wrote its result and then died in shutdown: recorded, never retried, never failed + for dirty in pc.dirty_events(name, out): + dirty.update(traces_of(name, xmldir, [dirty["attempt"]])) + events.append(dirty) + rows.append((name, c, out.status, fails, len(out.attempts), out.final.rc)) + print( + f"{name:45s} pass={c['passed']:4d} fail={c['failed']:3d} err={c['error']:3d} notyet={c['notyet']:3d} na_jvm={c['na_jvm']:3d} skip={c['skipped']:3d} {out.status}", + flush=True, ) - except subprocess.TimeoutExpired as e: - status = "TIMEOUT" - open(os.path.join(xmldir, name + ".log"), "w").write( - ((e.stdout or b"")[-8000:]).decode() - if isinstance(e.stdout, bytes) - else str(e.stdout)[-8000:] + if out.retried: + print( + f" {name}: the first run died without a result ({pc.format_rc(out.attempts[0].rc)}); " + f"the run once more: {'a result' if out.final.result else 'no result again'} ({pc.format_rc(out.final.rc)})", + flush=True, + ) + for n, at in enumerate(out.attempts, 1): + if pc.dirty_exit(at): + print( + f"::warning::{name} attempt {n} wrote its result and then exited with {pc.format_rc(at.rc)}", + flush=True, + ) + with open(a.out, "w") as fh: + fh.write( + "file\tpassed\tfailed\terror\tnotyet\tna_jvm\tskipped\tstatus\tfailing_tests\tattempts\texit\n" ) - c = dict.fromkeys(["passed", "failed", "error", "notyet", "na_jvm", "skipped"], 0) - fails = [] - if os.path.exists(xml): - try: - for tc in ET.parse(xml).getroot().iter("testcase"): - if tc.find("failure") is not None: - c["failed"] += 1 - fails.append(tc.get("name")) - elif tc.find("error") is not None: - c["error"] += 1 - fails.append(tc.get("name")) - elif tc.find("skipped") is not None: - msg = tc.find("skipped").get("message") or "" - c[ - ( - "notyet" - if msg.startswith("NOTYET") - else "na_jvm" if msg.startswith("NA-JVM") else "skipped" - ) - ] += 1 - else: - c["passed"] += 1 - except ET.ParseError: - status = status or "BADXML" - else: - status = status or "CRASH" - for k in c: - tot[k] += c[k] - if status == "TIMEOUT": - tot["timeout"] += 1 - rows.append((name, c, status, fails)) - print( - f"{name:45s} pass={c['passed']:4d} fail={c['failed']:3d} err={c['error']:3d} notyet={c['notyet']:3d} na_jvm={c['na_jvm']:3d} skip={c['skipped']:3d} {status}", - flush=True, - ) -with open(a.out, "w") as fh: - fh.write( - "file\tpassed\tfailed\terror\tnotyet\tna_jvm\tskipped\tstatus\tfailing_tests\n" - ) - for name, c, status, fails in rows: + for name, c, status, fails, attempts, rc in rows: + fh.write( + f"{name}\t{c['passed']}\t{c['failed']}\t{c['error']}\t{c['notyet']}\t{c['na_jvm']}\t{c['skipped']}\t{status}\t{';'.join(fails[:40])}\t{attempts}\t{pc.format_rc(rc)}\n" + ) fh.write( - f"{name}\t{c['passed']}\t{c['failed']}\t{c['error']}\t{c['notyet']}\t{c['na_jvm']}\t{c['skipped']}\t{status}\t{';'.join(fails[:40])}\n" + f"TOTAL\t{tot['passed']}\t{tot['failed']}\t{tot['error']}\t{tot['notyet']}\t{tot['na_jvm']}\t{tot['skipped']}\ttimeouts={tot['timeout']} flaky_crash={tot['flaky_crash']} crash={tot['crash']}\t\t\t\n" ) - fh.write( - f"TOTAL\t{tot['passed']}\t{tot['failed']}\t{tot['error']}\t{tot['notyet']}\t{tot['na_jvm']}\t{tot['skipped']}\ttimeouts={tot['timeout']}\t\n" - ) -print("TOTAL", tot) -if a.max_notyet is not None and tot["notyet"] > a.max_notyet: - sys.exit(f"{tot['notyet']} tests are NOTYET skips, above the limit {a.max_notyet}") + with open(os.path.splitext(a.out)[0] + ".meta.json", "w") as fh: + json.dump( + { + "key": env.get("KEY", ""), + "platform": sys.platform, + "python": ident.get("python", ""), + "wheel_version": ident.get("wheel_version", ""), + "engine_version": ident.get("engine_version", ""), + "source_commit": ident.get("source_commit", ""), + "retry_crash": policy.enabled, + "crash_events": events, + }, + fh, + indent=1, + ) + print("TOTAL", tot) + if a.max_notyet is not None and tot["notyet"] > a.max_notyet: + sys.exit( + f"{tot['notyet']} tests are NOTYET skips, above the limit {a.max_notyet}" + ) + + +if __name__ == "__main__": + main() diff --git a/native/wheel/report_flaky_crash.py b/native/wheel/report_flaky_crash.py new file mode 100644 index 0000000..2affe9e --- /dev/null +++ b/native/wheel/report_flaky_crash.py @@ -0,0 +1,245 @@ +#!/usr/bin/env python3 +"""report_flaky_crash.py: make a parity crash loud, once per workflow run. + +native/wheel/parity_wheel.py retries a test file whose process died without a result (Windows legs only) and writes +.meta.json beside its table, with one crash event per file: FLAKY-CRASH (the retry produced a result, so the leg may pass) +or CRASH (it died again, or no retry was allowed, so the leg fails), or DIRTY-EXIT (the file wrote its result and the process then +died in shutdown: recorded, never retried, the leg is not failed). This script reads the meta files of every leg (the downloaded +`native-parity-*` artifacts), then + + 1. appends a table to the job summary ($GITHUB_STEP_SUMMARY), and + 2. posts ONE comment on issue 295 listing every event of the run (run URL, run attempt, Python, wheel, exit codes), but only on a + workflow_dispatch or a tag push (a release), never on a pull request, never from another repository than ArcadeData/arcadedb-python. + +The comment is a log, not a gate: when the token cannot write issues, or gh fails for any other reason, the script prints a `::warning::` +line and still exits 0. It never fails the run. The text carries no em dash. + + python native/wheel/report_flaky_crash.py --artifacts [--summary ] [--dry-run] +""" + +from __future__ import annotations + +import argparse +import glob +import json +import os +import subprocess # nosec B404 - runs the gh CLI with a fixed argument list +import sys + +REPO = "ArcadeData/arcadedb-python" +ISSUE = 295 +META = "parity-wheel.meta.json" + + +def load_events(root: str) -> list[dict]: + """Every crash event of every leg under `root` (one artifact directory per leg), each with its leg's identity.""" + events = [] + for path in sorted(glob.glob(os.path.join(root, "**", META), recursive=True)): + try: + with open(path, encoding="utf-8") as fh: + meta = json.load(fh) + except (OSError, ValueError) as err: + print(f"::warning::{path} could not be read: {err}", flush=True) + continue + artifact = os.path.basename(os.path.dirname(path)) + for ev in meta.get("crash_events", []): + events.append( + { + **ev, + "leg": meta.get("key") or artifact, + "python": meta.get("python", ""), + "wheel": meta.get("wheel_version", ""), + "artifact": artifact, + "retry_on": bool(meta.get("retry_crash")), + } + ) + return events + + +def _table(events: list[dict]) -> list[str]: + lines = [ + "| leg | Python | wheel | file | first exit code | retry |", + "|---|---|---|---|---|---|", + ] + for e in events: + retry = e["retry"] if not e["retry_rc"] else f"{e['retry']} ({e['retry_rc']})" + lines.append( + f"| {e['leg']} | {e['python']} | {e['wheel']} | `{e['file']}` | {e['kind']} {e['first_rc']} | {retry} |" + ) + return lines + + +def _where(events: list[dict]) -> list[str]: + lines = [] + for e in events: + files = [e[k] for k in ("trace1", "trace2") if e.get(k)] + # name.attempt1.log exists only when a retry ran (the retry's own log is name.log); a crash that was not retried has name.log alone + log = e.get("log") or ( + f"{e['file']}.attempt1.log" if e.get("retry_rc") else f"{e['file']}.log" + ) + names = ", ".join(f"`{n}`" for n in [*files, log]) + lines.append( + f"- `{e['file']}` ({e['leg']}, Python {e['python']}): artifact `{e['artifact']}`, folder `parity-wheel-xml/`: {names}" + ) + return lines + + +def summary_md(events: list[dict]) -> str: + """The job summary block; empty when no file crashed.""" + if not events: + return "" + flaky = [e for e in events if e["kind"] == "FLAKY-CRASH"] + hard = [e for e in events if e["kind"] == "CRASH"] + dirty = [e for e in events if e["kind"] == "DIRTY-EXIT"] + out = ["### Parity crashes", ""] + if flaky: + out += [ + f"**FLAKY-CRASH: {len(flaky)} file(s) died without a result and passed when run once more in a fresh process.** " + "A native crash was hidden by the retry on these legs. Each one is logged on issue " + f"#{ISSUE} (when the run is a release or a dispatch).", + "", + ] + if hard: + out += [ + f"**CRASH: {len(hard)} file(s) died without a result and the leg failed.**", + "", + ] + if dirty: + out += [ + f"**DIRTY-EXIT: {len(dirty)} file(s) wrote their result and then the process died in shutdown.** " + "The result stands: the file was not retried and the leg was not failed by this.", + "", + ] + out += [*_table(events), "", "Crash traces and logs:", ""] + out += _where(events) + return "\n".join(out) + "\n" + + +def comment_md( + events: list[dict], run_url: str, attempt: str, event: str, ref: str +) -> str: + """The comment for issue 295: one per run, every event in it.""" + kinds = sorted({e["kind"] for e in events}) + out = [ + f"{' and '.join(kinds)} in a native parity run.", + "", + f"Run: {run_url} (attempt {attempt or '1'}, event {event}, ref {ref})", + "", + *_table(events), + "", + "A test file died without a result and the harness ran it once more in a fresh process (`parity_wheel.py --retry-crash`, Windows legs only). " + "A retry that says `passed` made its leg green: without the retry that leg would have failed. " + "`CRASH` means the file died again (or no retry was left) and the leg failed. " + "`DIRTY-EXIT` means the file wrote its result and the process then died in shutdown: it was not retried and did not fail the leg.", + "", + "The faulthandler trace and the log are in the uploaded parity artifact of each leg:", + "", + *_where(events), + ] + return "\n".join(out) + "\n" + + +def may_post(event: str, ref_type: str, repo: str) -> str: + """'' when the comment may be posted, else the reason it may not.""" + if repo != REPO: + return f"repository {repo or '(unknown)'} is not {REPO}" + if event == "workflow_dispatch": + return "" + if event == "push" and ref_type == "tag": + return "" + return f"event {event or '(unknown)'} on a {ref_type or 'ref'} is neither a release nor a dispatch" + + +def post_comment(body: str, run=subprocess.run) -> tuple[bool, str]: + """POST the comment with the gh CLI and GH_TOKEN; (True, '') or (False, why). Never raises.""" + cmd = [ + "gh", + "api", + "--method", + "POST", + f"repos/{REPO}/issues/{ISSUE}/comments", + "--input", + "-", + ] + try: + r = run( + cmd, + input=json.dumps({"body": body}), + capture_output=True, + text=True, + timeout=120, + ) # nosec B603 + except (OSError, subprocess.SubprocessError) as err: + return False, f"{type(err).__name__}: {err}" + if r.returncode != 0: + return ( + False, + (r.stderr or r.stdout or f"gh exited {r.returncode}").strip()[:300], + ) + return True, "" + + +def report(events, env, summary_path="", dry_run=False, run=subprocess.run) -> int: + """Write the summary, post the comment when allowed. Returns 0: a log is never a gate.""" + if not events: + print("no crash event in this run", flush=True) + return 0 + md = summary_md(events) + print(md, flush=True) + if summary_path: + with open(summary_path, "a", encoding="utf-8") as fh: + fh.write(md) + server = env.get("GITHUB_SERVER_URL", "https://github.com") + repo = env.get("GITHUB_REPOSITORY", "") + run_url = f"{server}/{repo}/actions/runs/{env.get('GITHUB_RUN_ID', '')}" + body = comment_md( + events, + run_url, + env.get("GITHUB_RUN_ATTEMPT", ""), + env.get("GITHUB_EVENT_NAME", ""), + env.get("GITHUB_REF_NAME", ""), + ) + why = may_post( + env.get("GITHUB_EVENT_NAME", ""), env.get("GITHUB_REF_TYPE", ""), repo + ) + if why: + print(f"issue #{ISSUE}: no comment posted ({why})", flush=True) + return 0 + if dry_run: + print(f"issue #{ISSUE}: dry run, the comment would be:\n{body}", flush=True) + return 0 + if not (env.get("GH_TOKEN") or env.get("GITHUB_TOKEN")): + print( + f"::warning::the FLAKY-CRASH comment on #{ISSUE} was skipped: no GH_TOKEN", + flush=True, + ) + return 0 + ok, err = post_comment(body, run=run) + if ok: + print(f"issue #{ISSUE}: comment posted", flush=True) + else: + print( + f"::warning::the FLAKY-CRASH comment on #{ISSUE} was skipped, the token cannot write issues or gh failed: {err}", + flush=True, + ) + return 0 + + +def main(argv=None) -> int: + ap = argparse.ArgumentParser() + ap.add_argument("--artifacts", required=True) + ap.add_argument("--summary", default=os.environ.get("GITHUB_STEP_SUMMARY", "")) + ap.add_argument("--dry-run", action="store_true") + a = ap.parse_args(argv) + try: + return report(load_events(a.artifacts), os.environ, a.summary, a.dry_run) + except Exception as err: # noqa: BLE001 - a log is never a gate + print( + f"::warning::the crash report failed: {type(err).__name__}: {err}", + flush=True, + ) + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/scripts/Dockerfile.build b/scripts/Dockerfile.build index de26178..d62576c 100644 --- a/scripts/Dockerfile.build +++ b/scripts/Dockerfile.build @@ -181,6 +181,10 @@ RUN echo "🔍 Analyzing JARs with jdeps..." && \ exit 1; \ fi +# The default class-data-sharing archive of the JRE, dumped last with the flags the wheel starts the JVM with (#313) +COPY scripts/dump_cds_archive.sh /build/dump_cds_archive.sh +RUN bash /build/dump_cds_archive.sh /build/jre + # Stage 3: Build Python wheel FROM python:${PYTHON_VERSION}-slim AS python-builder diff --git a/scripts/build-native.sh b/scripts/build-native.sh index ee6f72d..3505eaa 100755 --- a/scripts/build-native.sh +++ b/scripts/build-native.sh @@ -275,6 +275,11 @@ if [[ "$PLATFORM" == windows/* ]]; then echo -e "${CYAN}📊 JRE size after Windows cleanup: ${YELLOW}${JRE_SIZE}${NC}" fi +# The default class-data-sharing archive of the JRE, dumped last with the flags the wheel starts the JVM with (#313) +bash "$SCRIPT_DIR/dump_cds_archive.sh" "$PY_BINDINGS_DIR/temp_jre" +JRE_SIZE=$(du -sh "$PY_BINDINGS_DIR/temp_jre" | cut -f1) +echo -e "${CYAN}📊 JRE size with the CDS archive: ${YELLOW}${JRE_SIZE}${NC}" + # Step 3: Copy JRE to package (JARs already filtered in place) echo -e "${CYAN}📦 Preparing package...${NC}" diff --git a/scripts/dump_cds_archive.sh b/scripts/dump_cds_archive.sh new file mode 100755 index 0000000..35d17fd --- /dev/null +++ b/scripts/dump_cds_archive.sh @@ -0,0 +1,46 @@ +#!/usr/bin/env bash +# SPDX-FileCopyrightText: 2021-present Arcade Data Ltd (info@arcadedata.com) and HumemAI (info@humem.ai) +# SPDX-License-Identifier: Apache-2.0 +# +# Dump the default class-data-sharing (CDS) archive of the bundled JRE, so the JVM maps the JDK's classes at start instead of +# loading and verifying them (#313). A JRE made by jlink has no archive; the JVM then starts without one. +# +# Usage: dump_cds_archive.sh JRE_DIR +# +# The archive is the JVM's default one (lib/server, or bin/server on Windows): the JVM looks for it by itself at every start and +# ignores it, without an error, when it was dumped for other settings. So it must be dumped with the settings the wheel starts +# with: -XX:+UseCompactObjectHeaders, which src/arcadedb/jvm.py sets (an archive dumped without it is ignored by a JVM that has it, +# and the JVM names the default archive after it). -Xmx1g keeps compressed oops on whatever the RAM of this build host is: the +# archive is the one a heap below 32 GB uses. A start with a larger heap (jvm.py gives the JVM 50 percent of the RAM) takes no +# archive, as before. The archive holds the JDK's classes only: the engine's jars are named by absolute path and checked by +# mtime in an archive, which an installed wheel cannot match. +# +# Run it last, on the finished JRE: the archive records the size of lib/modules. It checks its own result with -Xshare:on, which +# fails the JVM start when the archive cannot be used. +set -euo pipefail + +JRE_DIR="${1:?usage: dump_cds_archive.sh JRE_DIR}" +JAVA="${JRE_DIR}/bin/java" +CDS_FLAGS=(-XX:+UseCompactObjectHeaders -Xmx1g) + +if [[ ! -x "$JAVA" && ! -x "${JAVA}.exe" ]]; then + echo "dump_cds_archive.sh: no java in ${JRE_DIR}/bin" >&2 + exit 1 +fi + +echo "Dumping the default CDS archive of ${JRE_DIR} with: ${CDS_FLAGS[*]}" +"$JAVA" "${CDS_FLAGS[@]}" -Xshare:dump + +ARCHIVES="$(find "$JRE_DIR" -name '*.jsa' -type f)" +if [[ -z "$ARCHIVES" ]]; then + echo "dump_cds_archive.sh: the dump wrote no .jsa file under ${JRE_DIR}" >&2 + exit 1 +fi +echo "CDS archive:" +while IFS= read -r archive; do + echo " ${archive} ($(du -k "$archive" | cut -f1) KiB)" +done <<< "$ARCHIVES" + +# -Xshare:on: no archive, or one for other settings, is an error here instead of a quiet start without it +"$JAVA" "${CDS_FLAGS[@]}" -Xshare:on -version +echo "The JVM starts with the archive (-Xshare:on)" diff --git a/src/arcadedb/_columnar.py b/src/arcadedb/_columnar.py new file mode 100644 index 0000000..fe1ed34 --- /dev/null +++ b/src/arcadedb/_columnar.py @@ -0,0 +1,520 @@ +"""The column frame of both wheels, and the one decoder that reads it. + +A frame is what ``ColumnBatcher`` (the JVM wheel's bridge jar) writes and what the native library's ``arcadedb_n_next_col`` writes: a +little-endian ``int32`` header length, a JSON header (``count``, ``cols``: name, type, null-bitmap bytes, data bytes and, for a +vector, ``dim``), then for each column its null bitmap (bit i of byte i/8 set means row i is null) and its buffer. A column type is +``i8`` (int64), ``f8`` (float64), ``b1`` (one byte per bool), ``dt`` and ``dtu`` (int64 epoch milliseconds and microseconds), +``f4v`` and ``f8v`` (``dim`` floats per row), ``str`` (int32 offsets, count + 1 of them, then the UTF-8 bytes: Arrow's own string +layout), ``dec`` (a BigDecimal as text in the string layout) and ``json`` (a JSON array of the column, for values no fixed layout +carries: nested lists and maps, RIDs, a DATE next to datetimes, a mixed column). + +The interface and the format are shared, the plumbing is per wheel: what differs is how the bytes get here (JPype or ctypes) and the +text of a ``json`` column (the JVM wheel's tagged JSON with a side array of Java objects, the native library's tagged JSON), so the +decoders take ``json_values(data, side, count)`` from the wheel. Nothing here imports JPype. +""" + +import json +from typing import Any, Callable, Dict, List, Tuple + + +def _cast_all_to_string(arrs: list, pa) -> list: + """ + Cast every array to a string type. + + Tolerates a type pyarrow cannot cast directly (e.g. a vector/list column) + by going through Python objects instead. + """ + unified = [] + for a in arrs: + try: + unified.append(a.cast(pa.string())) + except pa.ArrowException: + unified.append( + pa.array( + [None if v is None else str(v) for v in a.to_pylist()], + type=pa.string(), + ) + ) + return unified + + +def _decode_strings(data, count: int, mask) -> list: + """The strings of one column buffer: int32 offsets, then the UTF-8 blob. + + ``mask`` is the null mask (True = null) or None when the column has no nulls. + """ + import numpy as np + + offs = np.frombuffer(data[: (count + 1) * 4], dtype=" list: + """The DECIMAL values of one column buffer, as ``Decimal`` objects (None for + null). The bridge writes each ``BigDecimal`` in full as text, so no digit is + lost to a double.""" + from decimal import Decimal + + return [ + None if s is None else Decimal(s) for s in _decode_strings(data, count, mask) + ] + + +def _missing_column_part(count: int, template): + """A batch of ``count`` rows that lack a column some other batch has: nulls in + the shape ``to_columns`` uses for that column (NaN for numbers, NaT for + datetimes, None for everything else).""" + import numpy as np + + if isinstance(template, np.ndarray): + if template.ndim == 2: + return np.full((count, template.shape[1]), np.nan, dtype=template.dtype) + if template.dtype.kind in "iuf": + return np.full(count, np.nan) + if template.dtype.kind == "M": + return np.full(count, np.datetime64("NaT"), dtype=template.dtype) + if template.dtype.kind == "O": + return np.full(count, None, dtype=object) + return [None] * count + + +def _is_untyped_arrow_chunk(a, pa) -> bool: + """True for a chunk that says nothing about its column's type: all values + null (a batch whose rows lack the property, or carry it as null), or a list + column holding only empty lists, which Arrow types ``list``.""" + if len(a) == 0 or a.null_count == len(a) or pa.types.is_null(a.type): + return True + return pa.types.is_list(a.type) and pa.types.is_null(a.type.value_type) + + +def _common_decimal_type(types: list, pa): + """One decimal type every given decimal type fits in (decimal128 up to 38 + digits, decimal256 up to 76), or None when it would need more.""" + scale = max(t.scale for t in types) + precision = max(t.precision - t.scale for t in types) + scale + if precision <= 38: + return pa.decimal128(precision, scale) + if precision <= 76: + return pa.decimal256(precision, scale) + return None + + +def _unify_arrow_chunk_types(arrs: list, pa) -> list: + """ + Unify a column's per-batch Arrow arrays onto one Arrow type. + + ArcadeDB is schemaless per document, so a property's Java type can + legally vary row to row; ColumnBatcher infers each batch's column type + independently, so two batches of the same result set can produce + incompatible Arrow types for the same column (#7108) - e.g. int64 in one + batch, float64 in the next. ``pa.chunked_array`` requires every chunk to + share one type, so leaving them as-is raises at concatenation time on a + result set that is otherwise entirely legal. + + Numeric types widen (int64 -> float64, matching to_columns' pandas-style + promotion) when every value survives the cast exactly; pyarrow's cast is + safe by default and raises rather than silently lose precision, which an + int64 outside float64's +-2**53 exact range would (code review on #7108), + so that case - and anything else, including a json or vector column that + varied - degrades to string instead, so the column always ends up with + exactly one type rather than an exception. + + int64/float64 is the only numeric pair worth special-casing: ColumnBatcher + (src/java/.../ColumnBatcher.java) already widens every + Java integer type (Long/Integer/Short/Byte) to the same "i8" wire encoding + and every Java floating type (Double/Float) to "f8" *within* one batch, so + across batches the only numeric type mismatch this ever has to unify is + exactly this pair - there is no int32/int64 or bool/int64 case to widen. + """ + types = {a.type for a in arrs} + if len(types) <= 1: + return arrs + + # A chunk that carries no type information (every value null, or only empty + # lists) takes the type the other chunks agree on, so the column's type does + # not depend on where the batch boundaries fall (#114). + untyped = [_is_untyped_arrow_chunk(a, pa) for a in arrs] + if any(untyped) and not all(untyped): + typed = _unify_arrow_chunk_types( + [a for a, u in zip(arrs, untyped) if not u], pa + ) + target = typed[0].type + filled = iter(typed) + out = [] + for a, u in zip(arrs, untyped): + if not u: + out.append(next(filled)) + elif a.null_count == len(a): + out.append(pa.nulls(len(a), type=target)) + else: + try: + out.append(a.cast(target)) + except pa.ArrowException: + return _cast_all_to_string(arrs, pa) + return out + + if types <= {pa.int64(), pa.float64()}: + try: + return [a.cast(pa.float64()) for a in arrs] + except pa.ArrowException: + pass + + if all(pa.types.is_list(t) or pa.types.is_fixed_size_list(t) for t in types): + # vectors and lists of numbers: a dimension that changes between frames makes a list, float32 next to float64 is float64 + # (the widening is exact), never text + values = {t.value_type for t in types} + if values <= {pa.float32(), pa.float64()}: + # always float64: the widening is exact, and a frame whose rows differ in length is a json column (list) on its + # own, so float32 here would make the type depend on whether the change falls inside a frame or between two + sizes = { + t.list_size if pa.types.is_fixed_size_list(t) else None for t in types + } + target = ( + pa.list_(pa.float64(), next(iter(sizes))) + if len(sizes) == 1 and None not in sizes + else pa.list_(pa.float64()) + ) + try: + return [pa.array(a.to_pylist(), type=target) for a in arrs] + except pa.ArrowException: + pass + + if all(pa.types.is_decimal(t) for t in types): + common = _common_decimal_type(list(types), pa) + if common is not None: + return [a.cast(common) for a in arrs] + + return _cast_all_to_string(arrs, pa) + + +MAX_FRAME_ROWS = 2**31 - 1 # a frame's row limit crosses to the library as a C int + + +def frame_rows(batch_size) -> int: + """The row limit of a frame for ``batch_size``: at least 1 (a ValueError otherwise), at most what a C int holds, so a huge + ``batch_size`` means "as many as one frame takes" instead of wrapping around to zero or a negative number. + """ + rows = int(batch_size) + if rows < 1: + raise ValueError(f"batch_size must be at least 1, got {batch_size}") + return min(rows, MAX_FRAME_ROWS) + + +def empty_columns(columns) -> Dict[str, Any]: + """The ``to_columns`` dict of a result with no rows: no column, or the ones the caller asked for, empty.""" + return {} if columns is None else {name: [] for name in columns} + + +def empty_arrow(columns): + """The ``to_arrow`` table of a result with no rows: no column, or the ones the caller asked for, empty and of the null type.""" + import pyarrow as pa + + return pa.table({} if columns is None else {name: pa.nulls(0) for name in columns}) + + +def _null_mask(buf, pos: int, nulls_len: int, count: int): + """``(null_bits, mask)`` of a column: ``mask`` is a bool array (True = null) or None when no row is null.""" + import numpy as np + + null_bits = np.frombuffer(buf[pos : pos + nulls_len], dtype=np.uint8) + if not null_bits.any(): + return null_bits, None + return null_bits, np.unpackbits(null_bits, bitorder="little")[:count].astype(bool) + + +def read_header(buf) -> Tuple[Dict[str, Any], int]: + """``(header, position of the first column)`` of a frame.""" + hlen = int.from_bytes(buf[:4], "little") + return json.loads(bytes(buf[4 : 4 + hlen])), 4 + hlen + + +def decode_columns_frame( + buf, side, json_values: Callable, micros: bool = False +) -> Tuple[int, Dict[str, Any], Dict[str, Any]]: + """``(count, parts, header)`` of one frame as ``to_columns`` columns: ``parts`` maps a column name to a numpy array + (int64, float64, bool, datetime64) or a Python list (text, JSON-typed values, a bool column with nulls); a count of 0 has + no parts. A vector column is a (count, dim) matrix, a null row of it NaN. + + ``micros`` is the ``to_dataframe`` form: a naive datetime column is ``datetime64[us]`` (``dtu``), and a vector column is a + list with one numpy array per cell and None for a null (a NaN row would read as a vector of NaN, not as a missing value). + """ + import numpy as np + + header, pos = read_header(buf) + count = header["count"] + if count == 0: + return 0, {}, header + parts: Dict[str, Any] = {} + for col in header["cols"]: + name, ctype = col["name"], col["type"] + nulls_len = col["nulls"] + _, mask = _null_mask(buf, pos, nulls_len, count) + has_nulls = mask is not None + pos += nulls_len + data = buf[pos : pos + col["bytes"]] + pos += col["bytes"] + + if ctype == "i8": + # a copy: np.frombuffer over bytes is read-only, and every array of to_columns is writable + arr = np.frombuffer(data, dtype=" 2-D array (count, dim); an empty vector has no "dim" in the header. The null rows are + # made NaN here, whatever the writer put in the slot: a row of zeros would read as a real zero vector + dim = col.get("dim", 0) + dt = " bool: + return ( + isinstance(part, list) and len(part) == count and all(v is None for v in part) + ) + + +# float64 holds every integer in +-2**53 exactly, and not every one beyond it +_TWO53 = 1 << 53 + + +def _int64_beyond_2_53_next_to_float64(parts) -> bool: + """True when the parts of one column hold an int64 array with a value beyond +-2**53 and a float64 array: ``np.concatenate`` + would make the integers doubles and round them. One frame of such a column is a list of exact Python values (the writer's + ``json`` column); the merge of frames does the same, so the column does not depend on where the frames end. + """ + import numpy as np + + arrays = [p for p in parts if isinstance(p, np.ndarray) and p.ndim == 1] + if not any(a.dtype.kind == "f" for a in arrays): + return False + return any( + a.dtype.kind == "i" and a.size and (a.max() > _TWO53 or a.min() < -_TWO53) + for a in arrays + ) + + +def merge_columns( + names: List[str], batches: List[Tuple[int, Dict[str, Any]]], columns=None +) -> Dict[str, Any]: + """The ``to_columns`` dict of the parts of every frame. The type of a column belongs to the whole result, not to where the frames + end: a frame that lacks the column, or holds only nulls for it, takes the type of the frames that have values (NaN, NaT or None + in its rows). An int64 frame with a value beyond +-2**53 next to a float64 frame makes a list of exact Python values, not a + float64 array. With no rows at all the result is :func:`empty_columns`.""" + import numpy as np + + if not batches or not names: + return empty_columns(columns) + out: Dict[str, Any] = {} + for name in names: + typed = [ + p[name] for c, p in batches if name in p and not _is_all_none(p[name], c) + ] + template = typed[0] if typed else next(p[name] for _, p in batches if name in p) + parts = [ + ( + p[name] + if name in p and not (typed and _is_all_none(p[name], c)) + else _missing_column_part(c, template) + ) + for c, p in batches + ] + np_parts = [p for p in parts if not isinstance(p, list)] + if ( + parts + and len(np_parts) == len(parts) + and not _int64_beyond_2_53_next_to_float64(typed) + ): + try: + out[name] = np.concatenate(parts) if len(parts) > 1 else parts[0] + continue + except Exception: # nosec B110 - fall through to generic merge + pass + column: list = [] + for p in parts: + column.extend(p if isinstance(p, list) else p.tolist()) + out[name] = column + return out + + +def _pandas_would_round(values) -> bool: + """True for a list that holds an integer beyond +-2**53 next to a float: pandas infers float64 for a list of ints and floats, + and an integer that is not an exact double comes out rounded.""" + kinds = set(map(type, values)) # one C-speed pass: a text column ends here + if int not in kinds or float not in kinds: + return False + return any(type(v) is int and not -_TWO53 <= v <= _TWO53 for v in values) + + +def dataframe_from_columns(columns): + """The ``pandas.DataFrame`` of the ``to_columns`` dict ``columns`` (what both wheels' ``to_dataframe`` builds). A column that is a + list holding an integer beyond +-2**53 next to a float is an ``object`` column of the exact Python values, as ``to_columns()`` + returns them: pandas would make it float64 and round the integer (#340). Every other column is handed to pandas as it was. + """ + import pandas as pd + + exact = { + name: pd.Series(values, dtype=object) + for name, values in columns.items() + if isinstance(values, list) and _pandas_would_round(values) + } + return pd.DataFrame({**columns, **exact} if exact else columns) + + +def decode_arrow_frame( + buf, side, json_values: Callable +) -> Tuple[int, Dict[str, Any], Dict[str, Any]]: + """``(count, chunks, header)`` of one frame as ``to_arrow`` columns: ``chunks`` maps a column name to a ``pyarrow.Array``. + Integers keep their type and their nulls (``to_columns`` would make them float64 with NaN), strings wrap the frame's own + offsets and bytes, vectors are fixed-size lists of float32 or float64.""" + import numpy as np + import pyarrow as pa + + header, pos = read_header(buf) + count = header["count"] + if count == 0: + return 0, {}, header + chunks: Dict[str, Any] = {} + for col in header["cols"]: + name, ctype = col["name"], col["type"] + nulls_len = col["nulls"] + null_bits, mask = _null_mask(buf, pos, nulls_len, count) + has_nulls = mask is not None + pos += nulls_len + data = buf[pos : pos + col["bytes"]] + pos += col["bytes"] + + if ctype == "i8": + # kept as int64 WITH validity, where to_columns promotes the whole column to float64/NaN + arr = pa.array(np.frombuffer(data, dtype=" column) to the async writers in one crossing, wait for them and return the row count, or None when a + column cannot be sent this way (nothing was handed over then). A record that a writer rejects raises + (``Database._finish_parallel``), as for ``insert_many(parallel=True)``, and so does an interrupt, after the writers have finished: + the failure token is made before the crossing, so the call always has the id to wait for and to read. + """ + if not _usable(): + return None + packed = columnar.pack(columns, columnar.Declared(db, type_name, declared)) + if packed is None: + return None + rows, cols = packed + from . import extras + + fid = extras.failure_token(db) + try: + _send(db, type_name, cols, rows, 0, "Failed to bulk-insert", fid=fid) + except ArcadeDBError as e: + raise db._parallel_load_failed(type_name, e, [fid]) from e + except BaseException: + db._abandon_parallel([fid]) # a ^C: the writers still run, so wait for them + raise + return db._finish_parallel(type_name, [fid], rows) def insert_rows_columnar(db, type_name, chunk): diff --git a/src/arcadedb/_native/codec.py b/src/arcadedb/_native/codec.py index cf6d3f1..51d5d74 100644 --- a/src/arcadedb/_native/codec.py +++ b/src/arcadedb/_native/codec.py @@ -1,5 +1,7 @@ """Python <-> native JSON: query parameters out, typed rows in.""" +import ctypes +import itertools import json import os from collections.abc import Mapping @@ -276,6 +278,64 @@ def encode_gb_args(args): return text, buf, state[1] +_STR_TYPE = frozenset((str,)) + + +def _plain_rows(rows): + """True when every row is a dict whose keys are str and whose values are plain scalars: :func:`_prep` and the default hook would + change nothing, so ``json.dumps`` writes the rows as they are. The census runs in C (a set of types per level), not in Python per row. + """ + if not set(map(type, set(itertools.chain.from_iterable(rows)))) <= _STR_TYPE: + return False + return ( + set(map(type, itertools.chain.from_iterable(map(dict.values, rows)))) <= _PLAIN + ) + + +def _side_buffer(flat): + """The float32 array `flat` as the side-buffer argument of ``arcadedb_n_gb_bin``: a ctypes view of its memory (no copy), or its bytes + when the array is read-only or not contiguous.""" + try: + return (ctypes.c_char * flat.nbytes).from_buffer(flat) + except (TypeError, ValueError): + return flat.tobytes() + + +def encode_vertex_args(type_name, rows, names, matrices): + """The ``args`` of ``arcadedb_n_gb_bin`` for ``create_vertices`` whose vectors are float32 matrices: ``(text, side buffer, float count)``. + + `rows` hold the properties other than `names`; matrices[c] is the ``(len(rows), dim)`` float32 array of the property names[c], one + row of it per vertex. The matrices are the front of the side buffer, back to back, and the text carries their names, dimensions and + float offsets (``"vecs"``): no tag and no Python call per row. A value of `rows` that needs the hook (a datetime, a float32 array that + did not qualify, a list of floats) is written as in :func:`encode_gb_args`, after the matrices in the buffer. + """ + offs, total = [], 0 + for matrix in matrices: + offs.append(total) + total += matrix.size + args = { + "type": type_name, + "rows": rows, + "vecs": { + "names": names, + "dims": [int(m.shape[1]) for m in matrices], + "offs": offs, + }, + } + if _plain_rows(rows): + text, arrays = _dumps(args), list(matrices) + else: + state = [list(matrices), total] + text = json.dumps(_prep(args), default=lambda v: _gb_default(v, state)) + arrays = state[0] + flat = ( + arrays[0].reshape(-1) + if len(arrays) == 1 + else _np.concatenate([a.reshape(-1) for a in arrays]) + ) + return text, _side_buffer(flat), flat.size + + _INT64_MAX = 2**63 - 1 _NS_DATETIME = None if _np is None else _np.dtype("datetime64[ns]") diff --git a/src/arcadedb/_native/colframes.py b/src/arcadedb/_native/colframes.py new file mode 100644 index 0000000..31e47cc --- /dev/null +++ b/src/arcadedb/_native/colframes.py @@ -0,0 +1,225 @@ +"""Column frames on the native build: the glue between ``arcadedb_n_next_col`` and the decoder both wheels share (``arcadedb._columnar``). + +A result of the native build reaches Python in two forms. The rows after the first reply are column frames, which the library writes and +the shared decoder reads. The first reply of a query (up to ``_FIRST_BATCH`` rows) is JSON text that the row APIs read, and a columnar read +finds those rows (and any the caller read by row) as dicts in the result's buffer. They cannot be read again from the cursor, so they are +written here as one more frame, in the layout the library writes, by the rules the library's encoder follows, and read by the same +decoder: a column has one type however a result splits into the first reply and frames. The JSON reply names the properties that were +``float[]`` or ``double[]`` (``fv`` and ``dv``): a list of numbers in a row cannot say that by itself, and a vector column is a column of +declared arrays, not a column of lists that happen to fit float32. This frame goes away with the JSON reply. +""" + +import json +import struct +from datetime import date, datetime, timedelta, timezone +from decimal import Decimal + +from .. import _columnar +from .codec import loads_tagged + +_EPOCH = datetime(1970, 1, 1) +_MS = timedelta(milliseconds=1) +_US = timedelta(microseconds=1) +_INT64 = (-(1 << 63), (1 << 63) - 1) +_TWO53 = 1 << 53 + + +def json_values(data, side, count): + """The values of a ``json`` column of a frame from the library: its tagged JSON, restored as a row's values are.""" + return loads_tagged(bytes(data).decode("utf-8"))[0] + + +def _head_json_values(data, side, count): + """The values of a ``json`` column of the head frame: the buffered values themselves, by the index the column carries.""" + return side[int.from_bytes(bytes(data), "little")] + + +def _naive_utc(v): + if isinstance(v, datetime): + if v.tzinfo is not None: + v = v.astimezone(timezone.utc).replace(tzinfo=None) + return v + return datetime(v.year, v.month, v.day) + + +def _kind(values, micros): + """The column type of a frame that the values would get from the library's encoder: ``i8``, ``f8``, ``b1``, ``dt``, ``dtu``, + ``str``, ``dec`` or ``json``; ``null`` for a column of nothing but nulls.""" + kinds = set() + big_int = False + for v in values: + if v is None: + continue + t = type(v) + if t is bool: + k = "b1" + elif t is int: + if not _INT64[0] <= v <= _INT64[1]: + k = "json" # a BigInteger + else: + k = "i8" + if not -_TWO53 <= v <= _TWO53: + big_int = True + elif t is float: + k = "f8" + elif t is datetime or t is date: + k = "dt" + elif t is str: + k = "str" + elif t is Decimal: + k = "dec" + else: + k = "json" + kinds.add(k) + if k == "json": + break + if not kinds: + return "null" + if kinds == {"i8", "f8"}: + # an integer beyond 2**53 next to a floating value has no exact double: the values stay what they are + return "json" if big_int else "f8" + if len(kinds) > 1: + return "json" + kind = kinds.pop() + if kind == "dt": + if not micros: + return "dt" + if all(v is None or (type(v) is datetime and v.tzinfo is None) for v in values): + return "dtu" + return "json" + return kind + + +def _vector_dim(values): + """The one length of the lists of a column of float lists, or 0 when it is not one (a non-list, an empty or a ragged list).""" + dim = 0 + for v in values: + if v is None: + continue + if type(v) is not list or not v or any(type(x) is not float for x in v): + return 0 + if dim == 0: + dim = len(v) + elif len(v) != dim: + return 0 + return dim + + +def _bitmap(values): + out = bytearray((len(values) + 7) // 8) + for i, v in enumerate(values): + if v is None: + out[i >> 3] |= 1 << (i & 7) + return bytes(out) + + +def _column(values, vector, micros, side): + """``(type, data, dim)`` of one column: the buffer the library's encoder writes for it. ``vector`` is ``"f"`` or ``"d"`` for a + property that was a ``float[]`` or a ``double[]``, else None.""" + import numpy as np + + kind = _kind(values, micros) + nan = float("nan") + if kind == "i8": + return ( + kind, + np.array([0 if v is None else v for v in values], dtype=" _INT32_MAX: + return None + lengths = _np.fromiter(map(len, items), dtype=_np.int64, count=n) + offsets = _np.zeros(n + 1, dtype=_np.int32) + _np.cumsum(lengths, out=offsets[1:], dtype=_np.int32) + blob_array = ( + _np.frombuffer(data, dtype=_np.uint8) if data else _np.zeros(0, dtype=_np.uint8) + ) + aux = (blob_array.ctypes.data, blob_array.nbytes) if data else (0, 0) + return Packed(name, kind, 0, offsets, aux=aux, nulls=nulls, keep=(blob_array, data)) + + +def _epoch_micros(value): + """Microseconds since 1970-01-01T00:00:00 of the UTC wall clock of a ``datetime`` (a naive one is taken as it is, as ``_tag_datetime``).""" + if value.utcoffset() is not None: + value = value.astimezone(_UTC) + seconds = ( + (value.toordinal() - _EPOCH_ORDINAL) * 86400 + + value.hour * 3600 + + value.minute * 60 + + value.second + ) + return seconds * 1_000_000 + value.microsecond + + def _list_column(name, values, declared): """A list or tuple of Python values as a column: one type (None allowed) or a list of equal-length float arrays; else None.""" n = len(values) @@ -203,6 +252,33 @@ def _list_column(name, values, declared): return None # an integer beyond 64 bits kind = "q" if only is int else "d" if only is float else "b" return Packed(name, kind, 0, arr, nulls=nulls) + if issubclass(only, datetime): + if only.__name__ == "NaTType": + return None # the JSON path stores it as a null + try: + ticks = [0 if v is None else _epoch_micros(v) for v in values] + except (OverflowError, ValueError): + return None # the JSON path raises what it raised before + return Packed( + name, "T", 1_000_000, _np.array(ticks, dtype=_np.int64), nulls=nulls + ) + if only is date: + days = [0 if v is None else v.toordinal() - _EPOCH_ORDINAL for v in values] + return Packed(name, "D", 0, _np.array(days, dtype=_np.int32), nulls=nulls) + if only is Decimal: + texts = [] + for v in values: + if v is None: + texts.append(b"") + elif v.is_finite(): + texts.append(str(v).encode("ascii")) + else: + return None # the JSON path sends a non-finite Decimal down the per-row path + return _blob_column(name, "n", texts, nulls) + if only is bytes or only is bytearray: + return _blob_column( + name, "x", values if present is None else [v or b"" for v in values], nulls + ) if only is list and declared.get(name) == "ARRAY_OF_FLOATS" and present is None: try: arr = _np.array(values, dtype=_np.float32) @@ -248,10 +324,11 @@ def _arrow_column(name, arr, declared): if arr.null_count else None ) - if types.is_string(t) or types.is_large_string(t): + is_text = types.is_string(t) or types.is_large_string(t) + if is_text or types.is_binary(t) or types.is_large_binary(t): bufs = arr.buffers() off_buf, data_buf = bufs[1], bufs[2] - if types.is_string(t): + if types.is_string(t) or types.is_binary(t): offsets = _np.frombuffer(off_buf, dtype=_np.int32)[ arr.offset : arr.offset + n + 1 ] @@ -266,8 +343,42 @@ def _arrow_column(name, arr, declared): size = 0 if data_buf is None else min(data_buf.size, _INT32_MAX) aux = (data_buf.address, size) if size else (0, 0) return Packed( - name, "s", 0, offsets, aux=aux, nulls=nulls, keep=(arr, off_buf, data_buf) + name, + "s" if is_text else "x", + 0, + offsets, + aux=aux, + nulls=nulls, + keep=(arr, off_buf, data_buf), + ) + if types.is_timestamp(t): + values = arr.cast(pa.int64()).fill_null(0).to_numpy(zero_copy_only=False) + if t.unit == "s" and values.size: + if values.min() < _LDT_MIN_SECONDS or values.max() > _LDT_MAX_SECONDS: + return None # the row path raises its usual error for it, before anything is stored + return Packed( + name, + "T", + _TICKS[t.unit], + _np.ascontiguousarray(values, dtype=_np.int64), + nulls=nulls, + ) + if types.is_date(t): + if types.is_date64(t): # milliseconds since the epoch, floor-divided to days + raw = arr.cast(pa.int64()).fill_null(0).to_numpy(zero_copy_only=False) + days = raw // 86_400_000 + if days.size and ( + days.min() < -(_INT32_MAX + 1) or days.max() > _INT32_MAX + ): + return None # int32 days would wrap: the row path raises its usual error for it + else: + days = arr.cast(pa.int32()).fill_null(0).to_numpy(zero_copy_only=False) + return Packed( + name, "D", 0, _np.ascontiguousarray(days, dtype=_np.int32), nulls=nulls ) + if types.is_decimal(t): + texts = [b"" if v is None else str(v).encode("ascii") for v in arr.to_pylist()] + return _blob_column(name, "n", texts, nulls) if types.is_integer(t) or types.is_floating(t) or types.is_boolean(t): if arr.null_count: arr = arr.fill_null(False if types.is_boolean(t) else 0) @@ -343,17 +454,22 @@ def spec_of(packed): return (ctypes.c_int64 * len(flat))(*flat) -def header_of(rows, commit_every, packed): - """The JSON header text: row count, commit interval and ``[name, kind, dim]`` per column.""" +def header_of(rows, commit_every, packed, fid=None): + """The JSON header text: row count, commit interval and ``[name, kind, dim]`` per column; with a failure token ``fid`` it also says + ``"parallel": true``, which hands the records to the async executor (the image then ignores the commit interval and counts the + failed records on that token). + """ import json - return json.dumps( - { - "n": rows, - "commit": min(int(commit_every), 2**31 - 1), - "cols": [[c.name, c.kind, c.dim] for c in packed], - } - ) + header = { + "n": rows, + "commit": min(int(commit_every), 2**31 - 1), + "cols": [[c.name, c.kind, c.dim] for c in packed], + } + if fid is not None: + header["parallel"] = True + header["fid"] = fid + return json.dumps(header) def rows_to_columns(rows): diff --git a/src/arcadedb/_native/columns.py b/src/arcadedb/_native/columns.py deleted file mode 100644 index 1d78992..0000000 --- a/src/arcadedb/_native/columns.py +++ /dev/null @@ -1,199 +0,0 @@ -"""Rows to columns for the native backend (the columnar path is Python over the rows; the JPype path decodes a Java buffer).""" - -from datetime import date, datetime, timezone -from decimal import Decimal - - -def _names(rows, columns): - """The column names: ``columns``, or the keys of the rows in the order they are first seen.""" - if columns is not None: - return list(columns) - seen = {} - known = seen.keys() - for row in rows: - # a row whose keys are all known (nearly every row) costs one C-level subset test, not one Python step per key - if not row.keys() <= known: - for k in row: - seen.setdefault(k, None) - return list(seen) - - -def _kind(values): - """'b', 'i', 'f', 'dt', 'dec', 'obj' for the non-null values of a column.""" - kinds = set() - for v in values: - if v is None: - continue - t = type(v) - if t is bool: - kinds.add("b") - elif t is int: - kinds.add("i") - elif t is float: - kinds.add("f") - elif t is datetime or t is date: - kinds.add("dt") - elif t is Decimal: - kinds.add("dec") - else: - kinds.add("obj") - if not kinds: - return "obj" - if len(kinds) == 1: - return kinds.pop() - if kinds == {"i", "f"}: - return "f" - return "obj" - - -def _ms(v): - if isinstance(v, datetime): - if v.tzinfo is not None: - v = v.astimezone(timezone.utc).replace(tzinfo=None) - return v - return datetime(v.year, v.month, v.day) - - -def _float_vectors(values): - """A column of equal-length float lists (None allowed) that float32 holds exactly, as a (count, dim) float32 array with - NaN rows for the nulls; None for any other column. Rows cannot say float[] from List, so only a column whose every - value survives float32 unchanged is read as a vector column (the JPype path decides on the Java element type). - """ - import numpy as np - - dim = 0 - for v in values: - if v is None: - continue - if type(v) is not list or not v or not all(type(x) is float for x in v): - return None - if dim == 0: - dim = len(v) - elif len(v) != dim: - return None - if dim == 0: - return None - arr = np.full((len(values), dim), np.nan, dtype=np.float32) - for i, v in enumerate(values): - if v is not None: - arr[i] = v - if not np.array_equal(arr[i].astype(np.float64), np.array(v)): - return None - return arr - - -def to_columns(rows, columns=None): - import numpy as np - - out = {} - for name in _names(rows, columns): - values = [row.get(name) for row in rows] - kind = _kind(values) - if kind == "obj": - vectors = _float_vectors(values) - if vectors is not None: - out[name] = vectors - continue - has_null = any(v is None for v in values) - if kind == "i": - out[name] = ( - np.array([np.nan if v is None else v for v in values], dtype=np.float64) - if has_null - else np.array(values, dtype=np.int64) - ) - elif kind == "f": - out[name] = np.array( - [np.nan if v is None else v for v in values], dtype=np.float64 - ) - elif kind == "b": - out[name] = values if has_null else np.array(values, dtype=bool) - elif kind == "dt": - out[name] = np.array( - [ - ( - np.datetime64("NaT", "ms") - if v is None - else np.datetime64(_ms(v), "ms") - ) - for v in values - ], - dtype="datetime64[ms]", - ) - elif kind == "dec": - arr = np.empty(len(values), dtype=object) - arr[:] = values - out[name] = arr - else: - out[name] = values - return out - - -def _inferred(values): - """``pa.array(values)`` when pyarrow's own inference gives the array the rules of :func:`to_arrow` would give, else None. - - Inference replaces the per-value Python pass of :func:`_kind` for a column of ints, floats, bools, strings, lists and - other values it types on its own. Two inferred types differ from the rules: a temporal type (the rules write - ``timestamp[ms]`` for a datetime and a date, inference gives ``timestamp[us]`` or ``date32``) and an unsigned integer - (an int above int64 is an error under the rules). A column that inference cannot type is also left to the rules, - whatever the error (``ArrowInvalid``, ``ArrowTypeError``, and ``OverflowError`` for an int above int64, which the - explicit-type call turns into ``ArrowInvalid``), so a failed inference never decides the outcome: the rules have the - fallbacks (exact text for an int beyond 2**53 next to floats, for a Decimal wider than decimal256, for a value of mixed - types). - """ - import pyarrow as pa - - try: - array = pa.array(values) - except Exception: # the rules decide the outcome of a column inference cannot type - return None - kind = array.type - if pa.types.is_temporal(kind) or pa.types.is_unsigned_integer(kind): - return None - return array - - -def to_arrow(rows, columns=None): - import pyarrow as pa - - arrays = {} - for name in _names(rows, columns): - values = [row.get(name) for row in rows] - inferred = _inferred(values) - if inferred is not None: - arrays[name] = inferred - continue - kind = _kind(values) - if kind == "i": - arrays[name] = pa.array(values, type=pa.int64()) - elif kind == "f": - try: - arrays[name] = pa.array(values, type=pa.float64()) - except ( - pa.ArrowInvalid - ): # an int beyond +-2**53 next to floats: the column degrades to exact text - arrays[name] = pa.array( - [None if v is None else str(v) for v in values], type=pa.string() - ) - elif kind == "b": - arrays[name] = pa.array(values, type=pa.bool_()) - elif kind == "dt": - arrays[name] = pa.array( - [None if v is None else _ms(v) for v in values], type=pa.timestamp("ms") - ) - elif kind == "dec": - try: - arrays[name] = pa.array(values) - except ( - pa.ArrowInvalid - ): # wider than decimal256 (76 digits): an exact string - arrays[name] = pa.array( - [None if v is None else str(v) for v in values], type=pa.string() - ) - else: - try: - arrays[name] = pa.array([None if v is None else v for v in values]) - except (pa.ArrowInvalid, pa.ArrowTypeError): - arrays[name] = pa.array( - [None if v is None else str(v) for v in values], type=pa.string() - ) - return pa.table(arrays) diff --git a/src/arcadedb/_native/database.py b/src/arcadedb/_native/database.py index bc19f3f..e0c5c20 100644 --- a/src/arcadedb/_native/database.py +++ b/src/arcadedb/_native/database.py @@ -9,6 +9,7 @@ from datetime import date, datetime from typing import Any, Dict, Iterator, List, Optional +from .. import _columnar from .._params import refuse_surplus_positionals from ..exceptions import ArcadeDBError from ..transactions import TransactionContext @@ -40,6 +41,7 @@ _FIRST_BATCH = 256 _EPOCH_DAY_ORDINAL = date(1970, 1, 1).toordinal() _NEXT_BATCH = 2048 +_EAGER = 8 # a flag of _exec that never reaches the library: the statement's first rows come back in its own reply _wrap_record = None _KIND = {"v": "v", "e": "e", "d": "d"} @@ -210,6 +212,10 @@ def __init__(self, database, reply: dict, tagged: bool = True): self._rs = reply["rs"] self._buf = deque(zip(reply["rows"], reply["meta"])) self._more = not reply["drained"] + # the properties that were float[] ("f") or double[] ("d") in the rows read so far: a JSON row cannot tell them from a list of + # doubles, and a columnar read types them as vector columns (colframes.py) + self._vectors: Dict[str, str] = {} + self._note_vectors(reply) self._closed = False self._exhausted = False # the engine computes lazily: an error met while the first rows were read is raised when the rows before it @@ -228,6 +234,7 @@ def _lazy(cls, database, rs_id: int) -> "NativeResultSet": self._rs = rs_id self._buf = deque() self._tagged = False + self._vectors = {} self._more = rs_id > 0 self._closed = False self._exhausted = False @@ -303,6 +310,7 @@ def _fill(self, into: Optional[list] = None) -> bool: self._facets = reply["facets"] if "ferr" in reply: self._facets_error = reply["ferr"] + self._note_vectors(reply) return has_rows or self._pending_error is not None @property @@ -379,6 +387,12 @@ def facets(self) -> Optional[Dict[str, Any]]: self._finish() return facets + def _note_vectors(self, reply: dict) -> None: + for name in reply.get("fv", ()): + self._vectors[name] = "f" + for name in reply.get("dv", ()): + self._vectors[name] = "d" + def _finish(self) -> None: self._exhausted = True self.close() @@ -505,33 +519,105 @@ def iter_json_batches(self, batch_size: int = 10_000, **kwargs): # returns as it is. The flag is read after the batch is built, so it covers every reply the batch's rows came from yield [json_native(row) for row in batch] if self._tagged else batch - def to_dataframe(self, convert_types: bool = True): - import pandas as pd + def _read_frames(self, batch_size, columns, decode, flags=0): + """The rows still to be read, in the two forms a result has: ``(head_rows, batches)``. + + ``head_rows`` are the rows this object holds as dicts (the first reply, and any reply the caller read by row): they are + not in the cursor any more. ``batches`` is the list of ``(count, parts)`` that ``decode(frame_bytes)`` makes of each column + frame the cursor yields (about 1 MiB of values each, ``batch_size`` rows at most; ``flags`` 1 asks for datetimes as + microseconds). The result is finished when this returns. + """ + size = _columnar.frame_rows(batch_size) + head_rows: list = [] + batches: list = [] + if not self._readable(): + return head_rows, batches + spec = json.dumps(list(columns)).encode("utf-8") if columns is not None else b"" + head_rows = [props for props, _ in self._buf] + self._buf.clear() + if self._pending_error is not None: + self._fill() # the rows before the error are consumed: raise it + while self._more and self._rs: + try: + raw = lib.call_frame(self._rs, size, spec, flags) + except lib.ColumnFrameError as exc: + # the library closed the cursor + self._more = False + self._rs = 0 + self._closed = True + self._exhausted = True + raise _read_error(exc.reply) from None + count, parts, header = decode(raw) + if header.get("done"): + self._more = False # the library closed the cursor with the frame that ended the rows + self._rs = 0 + elif count == 0: + # an open cursor must give a row per frame: never loop on a library that does not + self._more = False + self.close() + raise ArcadeDBError( + "The native library returned an empty column frame for a cursor with rows left (frame row limit " + f"{size}). This is a bug in the library or in the wheel." + ) + if count: + batches.append((count, parts)) + self._finish() + return head_rows, batches - return pd.DataFrame(self._drain_dicts()) + def to_dataframe(self, convert_types: bool = True): + """The rows as a ``pandas.DataFrame``, read as column frames (no dict per row). The values are the ones ``to_list()`` returns, + with two exceptions that are the same on both builds. A DATETIME keeps its microseconds (a ``datetime64[us]`` column, where + ``to_columns()`` and ``to_arrow()`` carry milliseconds), a DATE is a ``datetime.date`` in an object column, and a ``Decimal`` + stays one. A column of ``float[]`` vectors holds one numpy ``float32`` array per cell (a row of the matrix ``to_columns()`` + returns, not a Python list per row) and None for a null; ``to_list()`` gives lists of floats. A column that holds an integer + beyond +-2**53 next to a float is an ``object`` column of the exact Python values (``float64`` would round the integer), as + ``to_columns()`` returns them; ``to_arrow()`` gives the exact text. + """ + # availability check: a missing pandas raises before the cursor is read + __import__("pandas") + return _columnar.dataframe_from_columns(self._columns(25_000, None, True)) + + def _columns(self, batch_size, columns, micros): + from . import colframes + + head_rows, batches = self._read_frames( + batch_size, + columns, + lambda raw: colframes.decode_columns(raw, micros), + 1 if micros else 0, + ) + return colframes.columns_result( + head_rows, columns, micros, batches, self._vectors + ) def to_columns(self, batch_size: int = 25_000, columns=None): - """Columns as the JPype path returns them (numpy int64/float64/bool/datetime64[ms] arrays, lists for text and - JSON-typed values, an object array of Decimal), built in Python from the rows: a columnar JSON path, no - binary buffer. None when numpy is missing.""" + """Columns as the JVM build returns them (numpy int64/float64/bool/datetime64[ms] arrays, lists for text and JSON-typed + values, an object array of Decimal), decoded from column frames by the decoder both builds share. The arrays are writable, + and a column's type belongs to the whole result, not to where a batch ends. A column is a float32 matrix when the property is + a ``float[]`` (ARRAY_OF_FLOATS), and a list of lists when it is a LIST of numbers whatever the values; a null vector is a row of + NaN in the matrix (``to_dataframe()`` and ``to_arrow()`` keep it a null). An empty result is + ``{}``, or the ``columns`` asked for as empty lists. None when numpy is missing. + """ try: - import numpy # noqa: F401 + __import__("numpy") # availability check except ImportError: return None - from .columns import to_columns - - return to_columns(self._drain_dicts(), columns) + return self._columns(batch_size, columns, False) def to_arrow(self, batch_size: int = 25_000, columns=None): - """A ``pyarrow.Table`` with the types of :meth:`to_columns` but nulls kept (None when pyarrow is missing).""" + """A ``pyarrow.Table`` with the types of :meth:`to_columns` but nulls kept (None when pyarrow is missing), decoded from + column frames by the decoder both builds share.""" try: - import numpy # noqa: F401 - import pyarrow # noqa: F401 + __import__("numpy") # availability check + __import__("pyarrow") except ImportError: return None - from .columns import to_arrow + from . import colframes - return to_arrow(self._drain_dicts(), columns) + head_rows, batches = self._read_frames( + batch_size, columns, colframes.decode_arrow + ) + return colframes.arrow_result(head_rows, columns, batches, self._vectors) def to_polars(self, batch_size: int = 25_000, columns=None): try: @@ -706,12 +792,14 @@ def _exec( self, flags: int, prefix: str, language: str, command: str, args ) -> NativeResultSet: self._check_not_closed() + eager = flags & _EAGER + flags &= ~_EAGER if args: refuse_surplus_positionals( "command" if flags & 1 else "query", language, command, args ) interrupts.clear() - if flags & 1 and _LAZY_COMMAND_ROWS and lib.CMD_PATH: + if flags & 1 and _LAZY_COMMAND_ROWS and lib.CMD_PATH and not eager: # phase 8: the cursor id comes back as a long (no reply text to free and decode); the rows are read late or never text, buf, count = encode_params(args, binary=True) rid = lib.cmd( @@ -728,7 +816,7 @@ def _exec( raise _error(prefix, json.loads(lib.last_error())) return NativeResultSet._lazy(self, rid) # phase 7 (arcadedb_n_exec flags): 2 = a command's rows are pulled but encoded only when read, 4 = float arrays come back as a float32 tail - if flags & 1 and _LAZY_COMMAND_ROWS: + if flags & 1 and _LAZY_COMMAND_ROWS and not eager: flags |= 2 if _BINARY_REPLY: flags |= 4 @@ -778,6 +866,15 @@ def command(self, language: str, command: str, *args) -> NativeResultSet: """Run a statement that may write.""" return self._exec(1, "Command failed", language, command, args) + def _command_first_row(self, language: str, command: str, *args) -> NativeResultSet: + """:meth:`command` for a caller that reads the first row right away (a create or an update that returns its row): the statement runs + and its first rows come back in the same crossing (``arcadedb_n_exec`` without the lazy bit) instead of a cursor id that a second + crossing (``arcadedb_n_next_bin``) reads. The rows, the values and the errors are those of :meth:`command`, at the same moment: + an error the engine raises while it runs the statement (an INSERT or an UPDATE) is raised by this call, as by :meth:`command`; + one met while rows are computed lazily comes in the reply (``rerr``) and is raised when the row is read, as on the lazy path. + """ + return self._exec(1 | _EAGER, "Command failed", language, command, args) + # -- transactions def _tx_text(self, op: int, prefix: str) -> str: @@ -975,7 +1072,8 @@ def insert_many( a newDocument/set/save loop inside the image, as ``DocumentBatcher.insertManyJson`` on the JPype path). A ``datetime`` (also ``pandas.Timestamp`` and ``numpy.datetime64``), ``date``, ``Decimal``, ``bytes``/``bytearray``, numpy scalar, ``pandas.NA`` or ``pandas.NaT`` sitting directly in a row crosses in that text as a tagged value (``{"\\u0001": [kind, payload]}``, decoded by - ``Json.java``), so it does not move the chunk off this path; such a chunk loses the columnar entry. A row with a value the JSON + ``Json.java``), so it does not move the chunk off this path; but when a column holds only date-times, dates, Decimals or + bytes (None is a null) the chunk goes as columns, the same stored objects. A row with a value the JSON text cannot carry unchanged (NaN or Infinity, an integer beyond 64 bits, a date-time or decimal inside a list or map, a non-str key, a lone surrogate, a ``datetime.time``) goes through ``INSERT INTO t CONTENT :row`` on its own, in one crossing for a run of such rows; the rows around it keep the bulk path and every row is inserted in the order given. Inside the caller's @@ -1093,20 +1191,50 @@ def _command_many(self, language: str, command: str, rows) -> int: raise _error("Command failed", reply) return reply["n"] - def _parallel_load_failed(self, type_name, error) -> ArcadeDBError: - """The error for a parallel load that failed while handing rows to the writers: waits for the queue to drain first, so - the caller sees a final state, and says that the rows handed over before the failure may have been stored. + def _drain_parallel(self, tokens): + """Waits for the async writers, then reads, and so removes, the failure token of every parallel load handed in. Returns + ``(problem, failed, first)``: the error of the wait or of a read (or None), the failed records the writers reported on all the + tokens and the first failure's text. The tokens are read whatever the wait did: an unread token keeps its entries for good. """ + from . import extras + + problem = None try: self._async().wait_completion() - except Exception: # nosec B110 - the original error is the one to report - pass + except Exception as e: + problem = e + failed, first = 0, None + for token in tokens: + try: + result = extras.failures(self, token) + except Exception as e: + problem = problem or e + continue + if result["count"] and first is None: + first = result["first"] + failed += result["count"] + return problem, failed, first + + def _abandon_parallel(self, tokens): + """A parallel load that is being interrupted: waits for the writers (they run on) and removes the tokens; the error that is on its + way out is the one to report. + """ + self._drain_parallel(tokens) + + def _parallel_load_failed(self, type_name, error, tokens=()) -> ArcadeDBError: + """The error for a parallel load that failed while handing rows to the writers: waits for the queue to drain first, so + the caller sees a final state, and says that the rows handed over before the failure may have been stored. + """ + _, failed, _ = self._drain_parallel(tokens) text = str(error) prefix = "Failed to bulk-insert: " if text.startswith(prefix): text = text[len(prefix) :] + reported = ( + f"; the writers had reported {failed} failed record(s)" if failed else "" + ) return ArcadeDBError( - f"Failed to bulk-insert into '{type_name}': {text}; the rows handed to the " + f"Failed to bulk-insert into '{type_name}': {text}{reported}; the rows handed to the " f"parallel writers before the failure may have been stored (the call waited " f"for them before raising)" ) @@ -1128,26 +1256,28 @@ def _insert_many_parallel(self, type_name, rows) -> int: tokens.append(extras.insert_parallel(self, type_name, chunk)["fid"]) except ArcadeDBError as e: # as the JPype path: the rows handed to the writers before the failure are written whatever this call does - raise self._parallel_load_failed(type_name, e) from e + raise self._parallel_load_failed(type_name, e, tokens) from e n += len(chunk) except ArcadeDBError: raise except BaseException: - try: - self._async().wait_completion() - except Exception: # nosec B110 - the original error is the one to report - pass + self._abandon_parallel(tokens) raise + return self._finish_parallel(type_name, tokens, n) + + def _finish_parallel(self, type_name, tokens, n) -> int: + """The end of a parallel load: wait for the async writers, then read the failure count of every token and raise when a record failed.""" try: - self._async().wait_completion() - except Exception as e: - raise ArcadeDBError(f"Failed to bulk-insert into '{type_name}': {e}") from e - n_failed, first = 0, None - for token in tokens: - result = extras.failures(self, token) - if result["count"] and first is None: - first = result["first"] - n_failed += result["count"] + problem, n_failed, first = self._drain_parallel(tokens) + except BaseException: + self._abandon_parallel( + tokens + ) # a ^C in the wait: finish it, and remove the tokens + raise + if problem is not None: + raise ArcadeDBError( + f"Failed to bulk-insert into '{type_name}': {problem}" + ) from problem if n_failed: raise ArcadeDBError( f"Failed to bulk-insert into '{type_name}': the parallel writers reported {n_failed} failed record(s) " @@ -1412,7 +1542,11 @@ def export_to_csv( def insert_columns( self, type_name, columns, commit_every: int = 10_000, parallel: bool = False ) -> int: - """Bulk insert from whole columns (dict of name -> numpy array or list), as rows through :meth:`insert_many`.""" + """Bulk insert from whole columns (dict of name -> numpy array or list): columns the buffers can carry cross in one call + (``arcadedb_n_insert_colp``), any other column set goes as rows through :meth:`insert_many`. With ``parallel=True`` the + same one call hands the records to the async writers (no transaction, ``commit_every`` does not apply, the call waits for the + writers and raises when a record failed), as ``insert_many(parallel=True)`` does for rows. + """ self._check_not_closed() items = list(columns.items()) if hasattr(columns, "items") else None if not items: @@ -1428,7 +1562,13 @@ def insert_columns( known = {n: len(v) for n, v in items if hasattr(v, "__len__")} if len(set(known.values())) > 1: raise ValueError(f"columns differ in length: {known}") - if not parallel: + if parallel: + from .bulk import insert_columns_parallel + + done = insert_columns_parallel(self, type_name, columns) + if done is not None: + return done + else: from .bulk import insert_columns_fast done = insert_columns_fast(self, type_name, columns, commit_every) @@ -1448,8 +1588,9 @@ def insert_dataframe( self, type_name, df, *, batch_size: int = 10_000, parallel: bool = False ) -> int: """A pandas or Polars DataFrame into a type, in batches: the strict Arrow loading of ``Database.insert_dataframe`` - (dtype refusals, declared types, NaN rules, the committed-rows message), batches through :meth:`insert_many` for a - document type and ``GraphBatch.create_vertices`` for a vertex type (``arcadedb/dataframe.py``). + (dtype refusals, declared types, NaN rules, the committed-rows message), batches through the columnar insert + (:meth:`insert_columns`; :meth:`insert_many` for a batch whose columns cannot cross as memory) for a document type and a + vertex type alike (a vertex load refuses to run inside a transaction the caller opened; ``arcadedb/dataframe.py``). """ from ..dataframe import insert_table, to_arrow_table diff --git a/src/arcadedb/_native/extras.py b/src/arcadedb/_native/extras.py index 5f8a470..a0aa517 100644 --- a/src/arcadedb/_native/extras.py +++ b/src/arcadedb/_native/extras.py @@ -19,8 +19,18 @@ from ..callbacks import check_event, check_function from ..exceptions import ArcadeDBError from ..graph_batch import GraphBatch as _BaseGraphBatch +from ..type_conversion import take_vector_columns from . import interrupts, lib, not_yet -from .codec import _PLAIN, _default, _np, _prep, encode_gb_args, loads, sample_array +from .codec import ( + _PLAIN, + _default, + _np, + _prep, + encode_gb_args, + encode_vertex_args, + loads, + sample_array, +) # -------------------------------------------------------------------------------------------------------- upcall @@ -28,6 +38,7 @@ _UPCALL_TYPE = ctypes.CFUNCTYPE( None, ctypes.c_longlong, ctypes.c_char_p, ctypes.c_void_p ) +_DICT_OR_NONE = frozenset((dict, type(None))) _callbacks: dict = {} _callback_ids = itertools.count(1) _callbacks_lock = threading.Lock() @@ -180,6 +191,30 @@ def _op(self, prefix: str, op: str, **args) -> dict: return _call(prefix, "arcadedb_n_gb", self._gb, lib.b(op), lib.b(text)) return _call(prefix, "arcadedb_n_gb", self._gb, lib.b(op), lib.b(_enc(args))) + def _create_chunk(self, prefix: str, type_name: str, rows: list) -> list: + """One crossing and one transaction for `rows` (dicts): the RIDs of the new vertices. + + A property whose value in every row is a float32 numpy array of one length crosses as one float32 matrix in the side buffer + (``take_vector_columns``, ``encode_vertex_args``), with no tag, hook call or Python code per row; every other value is + written as :func:`encode_gb_args` writes it. + """ + taken = take_vector_columns(rows, (), undeclared=True) if lib.GB_BIN else None + if taken is None: + return self._op(prefix, "create_vertices", type=type_name, rows=rows)[ + "rids" + ] + names, matrices, rest = taken + text, buf, count = encode_vertex_args(type_name, rest, names, matrices) + return _call( + prefix, + "arcadedb_n_gb_bin", + self._gb, + lib.b("create_vertices"), + lib.b(text), + buf, + count, + )["rids"] + def new_vertex(self, type_name: str): from .records import Vertex @@ -229,27 +264,14 @@ def _to_java_varargs(self, properties: dict) -> dict: raise TypeError(f"property names must be str, got {key!r}") return properties - def _create_vertices_json_bulk(self, type_name: str, rows): - """The bulk path (JSON rows in, RIDs out): the RIDs, or None for rows JSON would change (the caller falls back).""" - from ..type_conversion import json_bulk_scalar_ok - - for row in rows: - for key, value in (row or {}).items(): - if type(key) is not str or not json_bulk_scalar_ok(value): - return None - return self._op( - f"Failed to create batch vertices for type '{type_name}'", - "create_vertices", - type=type_name, - rows=[r or {} for r in rows], - )["rids"] - def create_vertices(self, type_name: str, count_or_properties) -> list: """The RIDs of the new vertices. Rows go in chunks of the batch's ``commit_every`` (1000 unless graph_batch() got another value; 0: 10,000), one crossing and one transaction per chunk: chunks that already went in stay stored if a later chunk fails. - A non-empty one-dimensional ``float32`` numpy array in a row crosses in a binary side buffer; a list or a ``float64`` array - crosses as number text (``new_edge`` and ``new_edges`` props follow the same rule). + A property whose value in every row is a non-empty one-dimensional ``float32`` numpy array of one length crosses as one + float32 matrix in a binary side buffer, one buffer per property and chunk; any other ``float32`` array of a row crosses in the side + buffer one array at a time, and a list or a ``float64`` array as number text (``new_edge`` and ``new_edges`` props follow the + one-array rule). """ self._check_not_closed() db = self._db @@ -274,23 +296,24 @@ def create_vertices(self, type_name: str, count_or_properties) -> list: ] return rids rows = list(count_or_properties) - for row in rows: - if row is not None and not isinstance(row, dict): - raise TypeError( - "create_vertices() property rows must be dicts or None" - ) + kinds = set(map(type, rows)) + if ( + not kinds <= _DICT_OR_NONE + ): # exact types in C; only an odd row pays the isinstance loop + for row in rows: + if row is not None and not isinstance(row, dict): + raise TypeError( + "create_vertices() property rows must be dicts or None" + ) # chunked, as the JPype wrapper does: one crossing, one JSON text and one reply per chunk, so a 1M-vertex load # never holds the text, the parsed rows and the reply of all of it at once (the Serial GC copies what lives) rids = [] + has_none = type(None) in kinds for start in range(0, len(rows), chunk): - rids.extend( - self._op( - prefix, - "create_vertices", - type=type_name, - rows=[r or {} for r in rows[start : start + chunk]], - )["rids"] - ) + part = rows[start : start + chunk] + if has_none: + part = [r or {} for r in part] + rids.extend(self._create_chunk(prefix, type_name, part)) if not rows: rids = self._op(prefix, "create_vertices", type=type_name, rows=[])[ "rids" @@ -915,6 +938,19 @@ def insert_parallel(database, type_name: str, rows) -> dict: ) +def failure_token(database) -> int: + """A new failure token for one parallel load: the caller makes it before the load, hands its id to the load and reads it with + :func:`failures` after the final wait, so it holds the id whatever happens to the load (an interrupt included). + """ + return _call( + "Failed to make a failure token", + "arcadedb_n_async", + database._h, + b"token", + lib.b(_enc({})), + )["fid"] + + def failures(database, token: int) -> dict: return _call( "Failed to read the parallel writers", diff --git a/src/arcadedb/_native/lib.py b/src/arcadedb/_native/lib.py index 3e49d46..600f381 100644 --- a/src/arcadedb/_native/lib.py +++ b/src/arcadedb/_native/lib.py @@ -232,6 +232,7 @@ class _Params(ctypes.Structure): "arcadedb_n_insert_colp": [P, L, S, S, P, I], "arcadedb_n_next": [P, L, I], "arcadedb_n_next_bin": [P, L, I], + "arcadedb_n_next_col": [P, L, I, S, I], "arcadedb_n_rs_close": [P, L], "arcadedb_n_facets": [P, L], "arcadedb_n_cmd": [P, L, I, S, S, S, S, L, I], @@ -328,16 +329,60 @@ def _thread_handle(): return t.handle +# One reply is read straight from its address: the text becomes a ``str`` in one copy (no ``bytes`` in between), and the library's +# memory is freed in the ``finally`` of the same call, once, whatever the decode does (a reply never outlives the call that took it, +# so no view of it can). ``PyUnicode_FromStringAndSize`` decodes UTF-8 as ``bytes.decode("utf-8")`` does, and raises the same +# UnicodeDecodeError; the function pointers are the ones of this module, ``ctypes.pythonapi`` itself is not changed. +_str_from_utf8 = ctypes.PYFUNCTYPE(ctypes.py_object, ctypes.c_void_p, ctypes.c_ssize_t)( + ("PyUnicode_FromStringAndSize", ctypes.pythonapi) +) +_strlen = ctypes.CFUNCTYPE(ctypes.c_size_t, ctypes.c_void_p)( + ( + "strlen", + ctypes.cdll.msvcrt if sys.platform == "win32" else ctypes.CDLL(None), + ) +) +_TAIL_KEY = b',"bin":' +_TAIL_WINDOW = 40 + + +def _take(lib, th, ptr, tail: bool): + """Read the reply at ``ptr`` as ``(text, floats)`` and free it: ``floats`` is None unless ``tail`` and the reply ends with + ``,"bin":N}`` (then N float32 follow the text's NUL, 4-byte aligned, and the text is returned without the ``,"bin":N`` part). + """ + if not ptr: + raise ArcadeDBError("The native library returned no reply") + try: + end = _strlen(ptr) + floats = None + if tail: + # only the last bytes can hold the marker: peek at them (at most 40 bytes), never at the whole reply + start = end - _TAIL_WINDOW if end > _TAIL_WINDOW else 0 + peek = ctypes.string_at(ptr + start, end - start) + if peek.endswith(b"}"): + i = peek.rfind(_TAIL_KEY) + if i >= 0: + n = int(peek[i + 7 : -1]) + import numpy as np + + off = (end + 1 + 3) & ~3 + floats = np.frombuffer( + ctypes.string_at(ptr + off, 4 * n), dtype=np.float32 + ) + # the text ends at the marker, closed by the "}" that the JSON object needs: written over the marker's comma + # in the reply's own memory (it is freed below), so the decode reads the JSON as it stands + ctypes.c_char.from_address(ptr + start + i).value = b"}" + end = start + i + 1 + return _str_from_utf8(ptr, end), floats + finally: + lib.arcadedb_free(th, ptr) + + def call(name: str, *args) -> dict: """Run one entry point; returns the decoded reply, raises ArcadeDBError-free (callers map errors).""" lib = _lib or _load() th = _thread_handle() - ptr = getattr(lib, name)(th, *args) - try: - text = ctypes.string_at(ptr).decode("utf-8") - finally: - lib.arcadedb_free(th, ptr) - return text + return _take(lib, th, getattr(lib, name)(th, *args), False)[0] def call_bin(name: str, *args): @@ -346,24 +391,43 @@ def call_bin(name: str, *args): """ lib = _lib or _load() th = _thread_handle() - ptr = getattr(lib, name)(th, *args) + return _take(lib, th, getattr(lib, name)(th, *args), True) + + +def call_frame(rs_id: int, max_rows: int, spec: bytes, flags: int) -> bytes: + """One column frame of an open cursor (``arcadedb_n_next_col``): its bytes, in the layout ``arcadedb._columnar`` reads. + + The reply is an int64 byte count and the frame. A negative count is the negated length of an error JSON: it is raised as + :class:`ColumnFrameError` with the decoded reply (the cursor is already closed on the Java side then). ``max_rows`` is a C int: + callers clamp it (``arcadedb._columnar.frame_rows``). + """ + lib = _lib or _load() + if getattr(lib, "arcadedb_n_next_col", None) is None: + raise ArcadeDBError( + "The native library is older than this wheel: it has no arcadedb_n_next_col entry point, which to_arrow, to_columns, " + "to_dataframe and to_polars need. Reinstall the arcadedb wheel (a library and its Python files ship together)." + ) + th = _thread_handle() + ptr = lib.arcadedb_n_next_col(th, rs_id, max_rows, spec, flags) + if not ptr: + raise ArcadeDBError( + "The native library returned no reply to a column frame request" + ) try: - raw = ctypes.string_at(ptr) - floats = None - if raw.endswith(b"}"): - i = raw.rfind(b',"bin":', max(0, len(raw) - 40)) - if i >= 0: - n = int(raw[i + 7 : -1]) - import numpy as np - - off = (len(raw) + 1 + 3) & ~3 - floats = np.frombuffer( - ctypes.string_at(ptr + off, 4 * n), dtype=np.float32 - ) - raw = raw[:i] + b"}" + n = int.from_bytes(ctypes.string_at(ptr, 8), "little", signed=True) + if n < 0: + raise ColumnFrameError(json.loads(ctypes.string_at(ptr + 8, -n))) + return ctypes.string_at(ptr + 8, n) finally: lib.arcadedb_free(th, ptr) - return raw.decode("utf-8"), floats + + +class ColumnFrameError(Exception): + """The error reply (the dict ``errorJson`` writes) of a failed ``arcadedb_n_next_col``.""" + + def __init__(self, reply: dict): + super().__init__(reply.get("error")) + self.reply = reply def alloc(size: int): @@ -384,22 +448,12 @@ def exec_query(handle, flags, language, command, params, count): lib = _lib or _load() t = _local.__dict__.get("thread") th = t.handle if t is not None else _thread_handle() - ptr = lib.arcadedb_n_exec(th, handle, flags, language, command, params, count) - try: - raw = ctypes.string_at(ptr) - floats = None - # a negative start counts from the end and stops at 0 for a short reply - i = raw.rfind(b',"bin":', -40) - if i >= 0 and raw.endswith(b"}"): - n = int(raw[i + 7 : -1]) - import numpy as np - - off = (len(raw) + 1 + 3) & ~3 - floats = np.frombuffer(ctypes.string_at(ptr + off, 4 * n), dtype=np.float32) - raw = raw[:i] + b"}" - finally: - lib.arcadedb_free(th, ptr) - return raw.decode("utf-8"), floats + return _take( + lib, + th, + lib.arcadedb_n_exec(th, handle, flags, language, command, params, count), + True, + ) def call_void(name: str, *args) -> None: diff --git a/src/arcadedb/_native/records.py b/src/arcadedb/_native/records.py index 4711c19..c42d973 100644 --- a/src/arcadedb/_native/records.py +++ b/src/arcadedb/_native/records.py @@ -338,7 +338,7 @@ def save(self): from .extras import veto_count vetoes = veto_count() - row = db.command( + row = db._command_first_row( "sql", self._CREATE.format(t=_q(self._type)), {"row": self._props} ).first() if row is None and veto_count() != vetoes: @@ -388,11 +388,11 @@ def _save_update(self, db) -> None: vetoes = veto_count() params = {"rid": RID(self._rid), "row": self._props} if exact: - row = db.command("sql", self._UPDATE_SENT, params).first() + row = db._command_first_row("sql", self._UPDATE_SENT, params).first() if row is not None and row.get("count") and veto_count() == vetoes: self._base = sent else: - after = db.command("sql", self._UPDATE, params).first() + after = db._command_first_row("sql", self._UPDATE, params).first() if ( after is not None ): # a veto or a record that is gone gives no row: nothing was written, the copy stays @@ -450,7 +450,7 @@ def new_edge(self, label: str, target: "Vertex", **kwargs) -> "Edge": if kwargs: sql += " CONTENT :row" params["row"] = kwargs - row = self._db.command("sql", sql, params).first() + row = self._db._command_first_row("sql", sql, params).first() edge = Edge(self._db, label, mutable=False) edge._adopt(row) return edge diff --git a/src/arcadedb/core.py b/src/arcadedb/core.py index e685888..792d324 100644 --- a/src/arcadedb/core.py +++ b/src/arcadedb/core.py @@ -6,6 +6,7 @@ import gc import itertools +import operator from collections.abc import Mapping from datetime import date from decimal import Decimal @@ -186,13 +187,61 @@ def _java_class(name): return cls +_NONE_TYPE = type(None) +_STR_OR_NONE = frozenset((str, _NONE_TYPE)) + + +def _string_column(values): + """A column of ``str`` and ``None`` as one ``StringColumn`` (the UTF-8 text of all rows in one ``byte[]``, one ``int[]`` of row + 1 + offsets, and one ``boolean[]`` of the null rows when there is a null), or None when the column is not that: empty, or holding any + other type (a mixed column converts per element), or too large for an int offset, or a ``str`` with no UTF-8 form (a lone + surrogate), which the per-element path keeps handling as before. The blob crosses through ``bytes`` into ``JArray(JByte)``, a + copy; nothing relies on a JPype view of Python memory. The type census, the lengths and the join run in C. + """ + n = len(values) + if _np is None or not n: + return None + kinds = set(map(type, values)) + if str not in kinds or not kinds <= _STR_OR_NONE: + return None + mask = None + if _NONE_TYPE in kinds: + mask = _np.fromiter( + map(operator.is_, values, itertools.repeat(None)), dtype=_np.bool_, count=n + ) + values = list(values) # the caller's sequence is not touched + for i in _np.flatnonzero(mask).tolist(): + values[i] = "" + if all(map(str.isascii, values)): + blob = "".join(values).encode("ascii") + lengths = _np.fromiter(map(len, values), dtype=_np.int64, count=n) + else: + try: + encoded = list(map(str.encode, values)) + except UnicodeEncodeError: + return None + blob = b"".join(encoded) + lengths = _np.fromiter(map(len, encoded), dtype=_np.int64, count=n) + del encoded + if len(blob) > _JAVA_INT_MAX: + return None + offsets = _np.zeros(n + 1, dtype=_np.int32) + _np.cumsum(lengths, dtype=_np.int32, out=offsets[1:]) + return _java_class("com.arcadedb.python.StringColumn")( + jpype.JArray(jpype.JByte)(blob), + jpype.JArray(jpype.JInt)(offsets), + None if mask is None else jpype.JArray(jpype.JBoolean)(mask), + ) + + def _column_to_java(name, values): """One column of ``Database.insert_columns`` as ``(java array, length)``. A numpy array of an integer, float, or bool kind crosses as ONE buffer copy into a ``long[]``, ``double[]``, or ``boolean[]``; a string or object array - and any other sequence convert per element into an ``Object[]`` (a ``str`` - reuses one Java String per distinct value, ``None`` is a null). A pandas + and any other sequence of mixed values convert per element into an ``Object[]`` (a ``str`` + reuses one Java String per distinct value, ``None`` is a null); a column of nothing but ``str`` and ``None`` crosses instead as one + ``StringColumn`` (``_string_column``). A pandas ``Series`` is read through ``to_numpy``, with a nullable dtype's ```` as ``None``. A 2-D ``float32`` array is a column of vectors: it crosses as one ``float[]`` and each document gets one row of it. Kinds that do not cross natively raise ``TypeError`` rather @@ -265,6 +314,9 @@ def _column_to_java(name, values): values = list(values.astype(object, copy=False)) elif not isinstance(values, (list, tuple)): values = list(values) + column = _string_column(values) + if column is not None: + return column, len(values) converted = [] seen = {} for value in values: @@ -1082,7 +1134,10 @@ def insert_columns( bool as ``boolean[]``, one buffer copy each) or any sequence of Python values (``str``, ``int``, ``float``, ``bool``, ``None``, and the types ``insert_many``'s per-row fallback accepts), which - converts per element. ``None`` is a null; a float ``NaN`` in a + converts per element, except a sequence of nothing but ``str`` + and ``None``, which crosses as one UTF-8 buffer with its offsets + (a string with no UTF-8 form, a lone surrogate, keeps the per-element + path). ``None`` is a null; a float ``NaN`` in a numpy float column is stored as NaN, not as null (use a sequence with ``None`` for nulls). A pandas nullable column (``Int64``, ``string``) converts with its ```` as null. @@ -1211,7 +1266,8 @@ def insert_dataframe( takes that table directly and has the type mapping, the declared-type rules, and the failure contract). Each column becomes a typed array that crosses the bridge once per batch: integers, floats, and booleans - without nulls as one buffer copy, anything else per element. A document + without nulls as one buffer copy, a string column as one UTF-8 buffer + with its offsets, anything else per element. A document type gets documents; a vertex type gets vertices. An edge type is refused: load the vertices, then buffer the edges with ``graph_batch().new_edges()``. @@ -1287,9 +1343,10 @@ def insert_arrow( The Arrow counterpart of :meth:`insert_columns`: each Arrow column is converted to the typed array ``insert_columns`` sends across the bridge - (``DocumentBatcher.insertColumns``), so a document type loads at that - path's speed; a vertex type loads through ``GraphBatch``. Documents and - vertices only: an edge type is refused. + (``DocumentBatcher.insertColumns``), so a document type and a vertex + type load at that path's speed (a vertex type is not loaded through + ``GraphBatch``; a vertex load refuses to run inside a transaction the + caller opened). Documents and vertices only: an edge type is refused. A property the type does not declare takes the stored type below; a declared property converts to its declared type. diff --git a/src/arcadedb/dataframe.py b/src/arcadedb/dataframe.py index 9d3ee0b..4c7beb2 100644 --- a/src/arcadedb/dataframe.py +++ b/src/arcadedb/dataframe.py @@ -5,17 +5,18 @@ (``Database.insert_dataframe`` and ``Database.insert_arrow``). Arrow is the one internal form: a pandas frame becomes an Arrow table, a Polars frame is already one, and each Arrow column becomes the typed array ``Database.insert_columns`` -sends across the bridge (a document type), or the property rows -``GraphBatch.create_vertices`` takes (a vertex type). No Java was added: both -paths use the existing glue (``DocumentBatcher.insertColumns`` and the -``GraphBatch`` vertex batcher). +sends across the bridge, for a document type and for a vertex type alike (the +vertex type gets vertices from the same ``DocumentBatcher.insertColumns``; it +refuses to run inside a transaction the caller opened, as ``GraphBatch`` does). The native backend (the ``arcadedb`` wheel) shares everything but the last step: the checks, the type mapping, the batching and the error wording are -the same code. A document batch goes over as one JSON text through -``Database.insert_many`` (one crossing per 10,000 rows), a vertex batch through -``GraphBatch.create_vertices``, and a value JSON cannot carry (a date-time, a -decimal, bytes) travels as a tagged value the image decodes. +the same code. A batch of documents or vertices goes over by address through +the columnar entry (``arcadedb_n_insert_colp``) when every column can cross as +memory (numbers, bools, strings, float vectors, date-times, dates, decimals and bytes), +else as one JSON text through ``Database.insert_many`` (one crossing per 10,000 rows), +where a value JSON cannot carry (a date-time, a decimal, bytes) travels as a tagged +value the image decodes. """ import re @@ -321,12 +322,75 @@ def _scalars_to_tagged(kind: str, chunk): ] -def _column_values(name: str, chunk, pa, native: bool = False, arrow: bool = False): +def _is_number_list(pa, arrow_type) -> bool: + """A list, large list or fixed-size list of integers or floats: the column the columnar entry lays out as a float32 matrix.""" + types = pa.types + if not ( + types.is_list(arrow_type) + or types.is_large_list(arrow_type) + or types.is_fixed_size_list(arrow_type) + ): + return False + return types.is_floating(arrow_type.value_type) or types.is_integer( + arrow_type.value_type + ) + + +_JAVA_INT_MAX = 2**31 - 1 + + +def _float32_matrix(pa, chunk): + """A list, large list or fixed-size list column of float32 with no null row, no null element and one length, as one 2-D float32 + numpy matrix (a view of Arrow's value buffer: ``flatten`` follows a slice's offset), else None. None means "not this shape" and + leaves the column to ``to_pylist``, which keeps what that route stored or refused: a null row stays null, a null element is + refused, vectors of different length are stored as given or refused by a vector index. Only float32 is taken: a float64 or integer + value is narrowed by the engine (and refused past the float range), which a cast here would not do. + """ + types = pa.types + arrow_type = chunk.type + if not ( + types.is_list(arrow_type) + or types.is_large_list(arrow_type) + or types.is_fixed_size_list(arrow_type) + ): + return None + n = len(chunk) + if not types.is_float32(arrow_type.value_type) or not n or chunk.null_count: + return None + flat = chunk.flatten() + if flat.null_count or len(flat) % n or len(flat) > _JAVA_INT_MAX: + return None + dim = len(flat) // n + if dim == 0: + return None + if not types.is_fixed_size_list(arrow_type): + import pyarrow.compute as pc + + lengths = pc.list_value_length(chunk) + if pc.min(lengths).as_py() != dim or pc.max(lengths).as_py() != dim: + return None + return flat.to_numpy(zero_copy_only=False).reshape(n, dim) + + +def _column_values( + name: str, + chunk, + pa, + native: bool = False, + arrow: bool = False, + float_vectors: bool = False, +): """One Arrow array as what ``Database.insert_columns`` takes: a numpy array when the column crosses as one buffer (integers, floats and booleans without nulls), else a list of values (``None`` is a null). - ``arrow`` (the native backend's document load): a string column, and a numeric one with nulls, stay Arrow arrays, which - the columnar entry reads in place (the offsets, the bytes and the validity bitmap) instead of as Python values. + ``float_vectors`` (the JVM backend, a property declared ``ARRAY_OF_FLOATS``): a float32 vector column of one length and no null is + one 2-D float32 matrix, which ``insert_columns`` sends as one ``float[]``; ``to_pylist`` made one Python float per element and + ``insert_columns`` one ``ArrayList`` of boxed doubles per row (see ``_float32_matrix`` for what is left to it). + + ``arrow`` (the native backend's document and vertex load): a string, binary, timestamp, date or decimal column, a list of + numbers (a vector) and a numeric column with nulls, stay Arrow arrays, which the columnar entry reads in place (the offsets, + the bytes, the integer ticks, the vector values and the validity bitmap) instead of as Python values; ``_insert_documents_native`` + turns them into Python or tagged values for the row path when a batch cannot go columnar. """ types = pa.types if types.is_dictionary(chunk.type): # a pandas categorical or a Polars Categorical @@ -361,6 +425,14 @@ def _column_values(name: str, chunk, pa, native: bool = False, arrow: bool = Fal values = values.astype("float64") # float16/32 widen exactly return values return chunk.to_pylist() + if arrow and kind in ("timestamp", "date", "decimal", "binary"): + return chunk + if arrow and kind == "list" and _is_number_list(pa, chunk.type): + return chunk # a vector column: the columnar entry reads the values in place (to_pylist makes one Python float per element) + if float_vectors and kind == "list": + matrix = _float32_matrix(pa, chunk) + if matrix is not None: + return matrix if kind in ("timestamp", "date"): return ( _temporal_to_tagged(chunk, pa) if native else _temporal_to_java(chunk, pa) @@ -421,68 +493,80 @@ def insert_table( native = _is_native(db) written = 0 # a document load inside the caller's own transaction joins it (insert_columns commits - # nothing then); a vertex load refuses to run inside one - in_caller_transaction = (not is_vertex) and bool(db.is_transaction_active()) - graph_batch = db.graph_batch() if is_vertex else None - try: - for batch in table.to_batches(max_chunksize=batch_size): - if batch.num_rows == 0: - continue - columns = { - name: _column_values( - name, - batch.column(i), - pa, - native, - arrow=native and not is_vertex and not parallel, + # nothing then); a vertex load refuses to run inside one, as GraphBatch.createVertices does. + # parallel=True never joins it: the async writers commit the records themselves, so a rollback + # of the caller's transaction does not remove them + in_transaction = bool(db.is_transaction_active()) + in_caller_transaction = in_transaction and not parallel + if is_vertex and in_transaction: + raise ArcadeDBError( + f"Failed to create batch vertices for type '{type_name}': a vertex load commits its own batches " + f"and cannot run inside a transaction the caller opened: commit or roll back that transaction first" + ) + for batch in table.to_batches(max_chunksize=batch_size): + if batch.num_rows == 0: + continue + columns = { + name: _column_values( + name, + batch.column(i), + pa, + native, + arrow=native, + float_vectors=not native and declared.get(name) == "ARRAY_OF_FLOATS", + ) + for i, name in enumerate(names) + } + try: + if native: + _insert_documents_native( + db, type_name, names, columns, batch_size, parallel, declared + ) + else: + db.insert_columns( + type_name, columns, commit_every=batch_size, parallel=parallel ) - for i, name in enumerate(names) - } - try: - if is_vertex: - _insert_vertices( - graph_batch, type_name, names, columns, batch.num_rows - ) - elif native: - _insert_documents_native( - db, type_name, names, columns, batch_size, parallel, declared - ) - else: - db.insert_columns( - type_name, columns, commit_every=batch_size, parallel=parallel - ) - except ArcadeDBError as exc: - if written and in_caller_transaction: - # the batches joined the caller's transaction: nothing is committed - # until the caller commits, and a rollback removes them - raise ArcadeDBError( - f"{exc}; the {written} row(s) of the batches before this one " - f"were written in your transaction, not committed" - ) from exc - if written: - raise ArcadeDBError( - f"{exc}; the {written} row(s) of the batches before this one " - f"were committed" - ) from exc - raise - written += batch.num_rows - finally: - if graph_batch is not None: - graph_batch.close() + except ArcadeDBError as exc: + if written and in_caller_transaction: + # the batches joined the caller's transaction: nothing is committed + # until the caller commits, and a rollback removes them + raise ArcadeDBError( + f"{exc}; the {written} row(s) of the batches before this one " + f"were written in your transaction, not committed" + ) from exc + if written and parallel: + # the async writers commit on their own, inside the caller's transaction or not + raise ArcadeDBError( + f"{exc}; the {written} row(s) of the batches before this one " + f"were handed to the async writers, which commit on their own: " + f"a rollback of your transaction does not remove them" + ) from exc + if written: + raise ArcadeDBError( + f"{exc}; the {written} row(s) of the batches before this one " + f"were committed" + ) from exc + raise + written += batch.num_rows + del ( + columns, + batch, + ) # sent: drop this batch's columns before the next batch is built, so two batches are never alive together return written -def _insert_vertices( - graph_batch, type_name: str, names: List[str], columns: Dict[str, Any], n: int -): - """One batch of vertices: the columns as one property dict per row, through ``GraphBatch.create_vertices``.""" - lists = [] - for name in names: - values = columns[name] - lists.append(values.tolist() if hasattr(values, "tolist") else values) - graph_batch.create_vertices( - type_name, [dict(zip(names, row)) for row in zip(*lists)] - ) +def _arrow_row_values(col) -> list: + """A column kept as an Arrow array, as the values the row path takes: Python values, or tagged values for a timestamp, date, + decimal or binary column (the form it had before the columnar entry took them, so the stored value is unchanged). + """ + import pyarrow as pa + + kind = _kind(pa, col.type) + if kind in ("timestamp", "date"): + return _temporal_to_tagged(col, pa) + if kind in ("decimal", "binary"): + return _scalars_to_tagged(kind, col) + return col.to_pylist() def _insert_documents_native( @@ -494,12 +578,23 @@ def _insert_documents_native( parallel: bool, declared: Optional[Dict[str, str]] = None, ): - """One batch of documents on the native backend. When every column can cross as memory (numbers, bools, strings, nulls, - float vectors for an ``ARRAY_OF_FLOATS`` property) the batch goes through the columnar entry, no Python values and no JSON - text; otherwise the columns become one dict per row and go to ``insert_many`` (one crossing per 10,000 rows). Either way - the batch is one transaction, or joins the caller's. + """One batch of documents or vertices on the native backend. When every column can cross as memory (numbers, bools, strings, + nulls, date-times, dates, decimals, bytes, float vectors for an ``ARRAY_OF_FLOATS`` property) the batch goes through the + columnar entry, no Python values and no JSON text; otherwise the columns become one dict per row and go to ``insert_many`` (one crossing per 10,000 rows). Either way + the batch is one transaction, or joins the caller's. With ``parallel`` the same columnar entry hands the records to the async + writers (no transaction), and the call waits for them and raises when a record failed, as ``insert_many(parallel=True)`` does. """ - if not parallel: + if parallel: + from ._native.bulk import insert_columns_parallel + + if ( + insert_columns_parallel( + db, type_name, columns, {} if declared is None else declared + ) + is not None + ): + return + else: from ._native.bulk import insert_columns_fast if ( @@ -520,7 +615,7 @@ def _insert_documents_native( if hasattr( col, "to_pylist" ): # a column kept as an Arrow array for the columnar entry - lists.append(col.to_pylist()) + lists.append(_arrow_row_values(col)) else: # jvm-payload: not-a-crossing -- no JVM here: insert_many serialises the rows to JSON text lists.append(col.tolist() if hasattr(col, "tolist") else col) diff --git a/src/arcadedb/graph_batch.py b/src/arcadedb/graph_batch.py index 0349584..dbd9ba7 100644 --- a/src/arcadedb/graph_batch.py +++ b/src/arcadedb/graph_batch.py @@ -14,11 +14,14 @@ from .type_conversion import ( convert_python_to_java, declared_float_vectors, - is_vector_candidate, + declared_scalars_ok, + json_bulk_dumps, json_bulk_dumps_tagged, json_bulk_scalar_ok, json_bulk_shape_ok, + screen_vertex_rows, split_float_vectors, + vector_width, ) _LOGGER = get_logger(__name__) @@ -190,9 +193,12 @@ def create_vertices( count_or_properties: Either an integer vertex count or an iterable of per-vertex property dictionaries. Use `None` or `{}` for vertices without properties when passing an iterable. A property declared - ``ARRAY_OF_FLOATS`` whose value in every row is a list or a numpy - array of numbers of one length crosses as one binary float32 - matrix instead of JSON text. + ``ARRAY_OF_FLOATS`` whose value in every row is a list, a numpy + array or a Java ``float[]`` or ``double[]`` of numbers of one length + crosses as one binary float32 matrix instead of JSON text, and so + does any other property whose value in every row is a float32 + numpy array or a Java ``float[]`` (``to_java_float_array``) of one + length: one buffer per property and chunk, stored as a ``float[]``. A list (not one of only numbers), tuple, dict, date-time, date, Decimal or bytes value also stays on the bulk path: it crosses inside the JSON as a tagged value and is stored as the @@ -246,68 +252,80 @@ def create_vertices( ) from e _BULK_CHUNK = 100_000 # rows per boundary crossing in bulk paths + # Rows with vector columns cross in blocks of about this many floats (1 MiB of float32) over all their vector columns, at least + # _VECTOR_BLOCK_MIN_ROWS rows: 10,000 rows of 96 floats cross in 4 blocks, not in one. The engine builds the vertices of a crossing + # from the JSON text, one float[] buffer and one copy of every row's vector, and a smaller crossing keeps less of that alive when + # a collection runs (the peak RSS of a load of Java float[] vectors was 1.1 to 1.3 times the 26.11.1 wheel's, which never copies them). + _VECTOR_BLOCK_FLOATS = 262_144 + _VECTOR_BLOCK_MIN_ROWS = 512 def _create_vertices_json_bulk(self, type_name, rows): """Bulk path: rows cross as one JSON string, RIDs return as one joined string (two bulk copies instead of per-value marshaling, measured - ~2.4x). The float vectors of ARRAY_OF_FLOATS properties cross as one - binary matrix beside the JSON instead of as text (split_float_vectors). + ~2.4x). The float vectors of ARRAY_OF_FLOATS properties, and the float32 + arrays and Java float[] of any property, cross as one binary matrix + beside the JSON instead of as text (split_float_vectors). A list (not one of only numbers), tuple, dict, date-time, date, Decimal or bytes value crosses inside the JSON as the tagged value json_bulk_dumps_tagged writes and VertexBatcher decodes (the stored value is what the property-matrix path stores). An input whose values - are all scalars takes the untagged json.dumps path as before. A chunk - holding a value the text cannot carry (a NaN, an integer beyond 64 - bits or a date-time inside a list, a subnormal) goes through the - property-matrix path on its own. Returns None when - unavailable/unsuitable so the caller falls back to the property-matrix - path for the whole input.""" + are all scalars is written by json_bulk_dumps. The values are checked + by their types and columns, in C (type_conversion.screen_vertex_rows), + not in a loop over the rows: a scalar the text would change (a NaN, an + integer beyond 64 bits, a lone surrogate, a subnormal) anywhere in the + input sends the whole input through the property-matrix path, and + rows of exact scalars only are then written with no further check. + Other rows are checked in the one json.dumps pass of each chunk (see + type_conversion._dumps_once); a chunk holding a date-time inside a + list, or a vector column the matrix cannot carry, goes through the + property-matrix path on its own. Returns None when unavailable/unsuitable + (a property name that is not a str, a type the text does not carry) so + the caller falls back to the property-matrix path for the whole input.""" try: vertex_batcher = jpype.JClass("com.arcadedb.python.VertexBatcher") declared = set(declared_float_vectors(self._java_db, type_name)) except Exception: return None - tagged = False # a value that needs the tagged text; False keeps the untagged code of before - for row in rows: - if row: - for key, value in row.items(): - if type(key) is not str: - # a property name that is not a str, which json.dumps would - # write as text ({1: "a"} as "1") where the matrix path - # refuses it - return None - if key in declared and is_vector_candidate(value): - continue # the vector split below takes it, or its chunk goes the matrix way - if not json_bulk_scalar_ok(value): - if not json_bulk_shape_ok(value): - # a type the tagged JSON does not carry (a set, a numpy - # or pandas value, any subclass, a list of only numbers: - # the matrix path converts those), or a scalar JSON - # would change: an integer beyond 64 bits, NaN or - # Infinity, a lone surrogate - return None - tagged = True - - import json - - # chunked so huge ingests never materialize one giant JSON string + verdict = screen_vertex_rows(rows, declared) + if verdict is None: + # a property name that is not a str (json.dumps would write 1 as "1" where the matrix path refuses it), or a type the + # tagged JSON does not carry (a set, a numpy or pandas value, any subclass, a list of only numbers: the matrix path + # converts those) + return None + tagged, vetted = verdict + + if vetted: + # every value is an exact scalar the text carries unchanged (screened in C above): the chunks are written without a check + def dumps(chunk_rows): + return json_bulk_dumps(chunk_rows, vetted=True) + + else: + dumps = json_bulk_dumps_tagged if tagged else json_bulk_dumps + # chunked so huge ingests never materialize one giant JSON string; rows that hold vectors cross in blocks of about + # _VECTOR_BLOCK_FLOATS floats per crossing, so the float[] buffer of a matrix and what the engine builds from it stay small + step = self._BULK_CHUNK + width = vector_width(rows, declared) + if width: + step = min( + step, + max(self._VECTOR_BLOCK_MIN_ROWS, self._VECTOR_BLOCK_FLOATS // width), + ) rids = [] - for start in range(0, len(rows), self._BULK_CHUNK): - chunk = rows[start : start + self._BULK_CHUNK] - split = split_float_vectors(self._java_db, type_name, chunk, declared) + for start in range(0, len(rows), step): + # the previous chunk's rest rows, text and matrices go before this chunk's are built: two float[] buffers were alive at once + split = rest = text = None + chunk = rows[start : start + step] + split = split_float_vectors( + self._java_db, type_name, chunk, declared, undeclared=True + ) rest = split[0] if split else chunk - if not tagged: - text = None - if not declared or self._json_bulk_ok(rest): - text = json.dumps([row or {} for row in rest]) - else: - text = None - if self._json_bulk_declared_ok(rest, declared): - try: - text = json_bulk_dumps_tagged([row or {} for row in rest]) - except (TypeError, ValueError): - text = None # a value the text cannot carry, see the docstring + text = None + if declared_scalars_ok(rest, declared): + try: + text = dumps([row or {} for row in rest]) + except (TypeError, ValueError): + text = None # a value the text cannot carry, see the docstring if text is None: # a list the matrix cannot carry (ragged, a null, not numbers) # is left in this chunk, or a value the text would change: the @@ -330,25 +348,6 @@ def _create_vertices_json_bulk(self, type_name, rows): rids.extend(joined.split(";")) return rids - @staticmethod - def _json_bulk_ok(rows): - for row in rows: - if row: - for value in row.values(): - if not json_bulk_scalar_ok(value): - return False - return True - - @staticmethod - def _json_bulk_declared_ok(rows, declared): - """False when a declared ARRAY_OF_FLOATS property still holds a list (or any non-scalar) after the vector split.""" - for row in rows: - if row: - for key in declared: - if key in row and not json_bulk_scalar_ok(row[key]): - return False - return True - def new_edge( self, source_vertex_or_rid, diff --git a/src/arcadedb/jvm.py b/src/arcadedb/jvm.py index 2289ccf..f38067a 100644 --- a/src/arcadedb/jvm.py +++ b/src/arcadedb/jvm.py @@ -889,7 +889,9 @@ def _build_jvm_args( ): merged_args.append("-Darcadedb.polyglotEngineEnabled=false") - # Enable compact object headers (JDK 25+) to reduce heap footprint + # Enable compact object headers (JDK 25+) to reduce heap footprint. + # The bundled JRE's class-data-sharing archive is dumped with this flag (scripts/dump_cds_archive.sh, #313): the JVM ignores an + # archive dumped with another setting of it, so a change here is a change there, and tests/test_jvm_cds.py fails when they differ. if not any("UseCompactObjectHeaders" in arg for arg in merged_args): merged_args.append("-XX:+UseCompactObjectHeaders") diff --git a/src/arcadedb/results.py b/src/arcadedb/results.py index 3e226bb..e9f92a1 100644 --- a/src/arcadedb/results.py +++ b/src/arcadedb/results.py @@ -12,6 +12,16 @@ import jpype as _jpype from jpype import JException +from ._columnar import ( + dataframe_from_columns, + decode_arrow_frame, + decode_columns_frame, + empty_arrow, + empty_columns, + frame_rows, + merge_arrow, + merge_columns, +) from ._logging import get_logger from .callbacks import raise_pending_interrupt from .exceptions import ArcadeDBError @@ -92,6 +102,11 @@ def hook(obj): return hook +def _typed_json_values(data, side, count): + """The values of a ``json`` column of a ColumnBatcher frame: the tagged JSON (TypedRows) with its side array of Java objects.""" + return json.loads(bytes(data), object_hook=_typed_hook(side)) + + def _loads_typed_row(java_row): """The dict of one row or record that ``ReaderCalls.rowJson`` / ``documentJson`` wrote. @@ -240,162 +255,6 @@ def _java_class_name(value: Any) -> str: return str(value.getClass().getName()) -def _cast_all_to_string(arrs: list, pa) -> list: - """ - Cast every array to a string type. - - Tolerates a type pyarrow cannot cast directly (e.g. a vector/list column) - by going through Python objects instead. - """ - unified = [] - for a in arrs: - try: - unified.append(a.cast(pa.string())) - except pa.ArrowException: - unified.append( - pa.array( - [None if v is None else str(v) for v in a.to_pylist()], - type=pa.string(), - ) - ) - return unified - - -def _decode_strings(data, count: int, mask) -> list: - """The strings of one column buffer: int32 offsets, then the UTF-8 blob. - - ``mask`` is the null mask (True = null) or None when the column has no nulls. - """ - import numpy as np - - offs = np.frombuffer(data[: (count + 1) * 4], dtype=" list: - """The DECIMAL values of one column buffer, as ``Decimal`` objects (None for - null). The bridge writes each ``BigDecimal`` in full as text, so no digit is - lost to a double.""" - from decimal import Decimal - - return [ - None if s is None else Decimal(s) for s in _decode_strings(data, count, mask) - ] - - -def _missing_column_part(count: int, template): - """A batch of ``count`` rows that lack a column some other batch has: nulls in - the shape ``to_columns`` uses for that column (NaN for numbers, NaT for - datetimes, None for everything else).""" - import numpy as np - - if isinstance(template, np.ndarray): - if template.ndim == 2: - return np.full((count, template.shape[1]), np.nan, dtype=template.dtype) - if template.dtype.kind in "iuf": - return np.full(count, np.nan) - if template.dtype.kind == "M": - return np.full(count, np.datetime64("NaT"), dtype=template.dtype) - if template.dtype.kind == "O": - return np.full(count, None, dtype=object) - return [None] * count - - -def _is_untyped_arrow_chunk(a, pa) -> bool: - """True for a chunk that says nothing about its column's type: all values - null (a batch whose rows lack the property, or carry it as null), or a list - column holding only empty lists, which Arrow types ``list``.""" - if len(a) == 0 or a.null_count == len(a) or pa.types.is_null(a.type): - return True - return pa.types.is_list(a.type) and pa.types.is_null(a.type.value_type) - - -def _common_decimal_type(types: list, pa): - """One decimal type every given decimal type fits in (decimal128 up to 38 - digits, decimal256 up to 76), or None when it would need more.""" - scale = max(t.scale for t in types) - precision = max(t.precision - t.scale for t in types) + scale - if precision <= 38: - return pa.decimal128(precision, scale) - if precision <= 76: - return pa.decimal256(precision, scale) - return None - - -def _unify_arrow_chunk_types(arrs: list, pa) -> list: - """ - Unify a column's per-batch Arrow arrays onto one Arrow type. - - ArcadeDB is schemaless per document, so a property's Java type can - legally vary row to row; ColumnBatcher infers each batch's column type - independently, so two batches of the same result set can produce - incompatible Arrow types for the same column (#7108) - e.g. int64 in one - batch, float64 in the next. ``pa.chunked_array`` requires every chunk to - share one type, so leaving them as-is raises at concatenation time on a - result set that is otherwise entirely legal. - - Numeric types widen (int64 -> float64, matching to_columns' pandas-style - promotion) when every value survives the cast exactly; pyarrow's cast is - safe by default and raises rather than silently lose precision, which an - int64 outside float64's +-2**53 exact range would (code review on #7108), - so that case - and anything else, including a json or vector column that - varied - degrades to string instead, so the column always ends up with - exactly one type rather than an exception. - - int64/float64 is the only numeric pair worth special-casing: ColumnBatcher - (src/java/.../ColumnBatcher.java) already widens every - Java integer type (Long/Integer/Short/Byte) to the same "i8" wire encoding - and every Java floating type (Double/Float) to "f8" *within* one batch, so - across batches the only numeric type mismatch this ever has to unify is - exactly this pair - there is no int32/int64 or bool/int64 case to widen. - """ - types = {a.type for a in arrs} - if len(types) <= 1: - return arrs - - # A chunk that carries no type information (every value null, or only empty - # lists) takes the type the other chunks agree on, so the column's type does - # not depend on where the batch boundaries fall (#114). - untyped = [_is_untyped_arrow_chunk(a, pa) for a in arrs] - if any(untyped) and not all(untyped): - typed = _unify_arrow_chunk_types( - [a for a, u in zip(arrs, untyped) if not u], pa - ) - target = typed[0].type - filled = iter(typed) - out = [] - for a, u in zip(arrs, untyped): - if not u: - out.append(next(filled)) - elif a.null_count == len(a): - out.append(pa.nulls(len(a), type=target)) - else: - try: - out.append(a.cast(target)) - except pa.ArrowException: - return _cast_all_to_string(arrs, pa) - return out - - if types <= {pa.int64(), pa.float64()}: - try: - return [a.cast(pa.float64()) for a in arrs] - except pa.ArrowException: - pass - - if all(pa.types.is_decimal(t) for t in types): - common = _common_decimal_type(list(types), pa) - if common is not None: - return [a.cast(common) for a in arrs] - - return _cast_all_to_string(arrs, pa) - - class ResultSet: """Iterator wrapper for ArcadeDB query results. @@ -843,10 +702,17 @@ def to_dataframe(self, convert_types: bool = True): Requires pandas to be installed. - Values are the ones ``to_list()`` returns: a DATETIME keeps its - microseconds (a ``datetime64[us]`` column, where ``to_columns()`` and - ``to_arrow()`` carry milliseconds), a DATE is a ``datetime.date`` in an - object column, and a ``Decimal`` stays one. + Values are the ones ``to_list()`` returns, with two exceptions that are + the same on both builds. A DATETIME keeps its microseconds (a + ``datetime64[us]`` column, where ``to_columns()`` and ``to_arrow()`` + carry milliseconds), a DATE is a ``datetime.date`` in an object column, + and a ``Decimal`` stays one. A column of ``float[]`` vectors holds one + numpy ``float32`` array per cell (a row of the matrix ``to_columns()`` + returns, no copy into a Python list per row) and None for a null; + ``to_list()`` gives lists of floats. A column that holds an integer + beyond +-2**53 next to a float is an ``object`` column of the exact + Python values (``float64`` would round the integer), as ``to_columns()`` + returns them; ``to_arrow()`` gives the exact text. Args: convert_types: Convert Java types to Python (default: True) @@ -878,13 +744,8 @@ def to_dataframe(self, convert_types: bool = True): # to_arrow() keep the documented milliseconds. columns = self._read_columns(25_000, None, micros=True) if columns is not None: - # pandas rejects 2-D ndarrays as column values; vector - # columns (f4v/f8v) become object columns of row slices - columns = { - k: (list(v) if getattr(v, "ndim", 1) > 1 else v) - for k, v in columns.items() - } - return pd.DataFrame(columns) + # a vector column is already one numpy array per cell, None for a null + return dataframe_from_columns(columns) return pd.DataFrame(self.to_list(convert_types=convert_types)) @@ -896,7 +757,9 @@ def to_polars( Reads :meth:`to_arrow` and hands the table to Polars, so nulls keep their type (a nullable integer stays an integer column) and a DECIMAL - column is a Polars ``Decimal``. Requires polars and pyarrow + column is a Polars ``Decimal``. A column of ``float[]`` vectors is a + Polars ``Array(Float32, n)`` (``to_arrow()`` types it as a fixed-size + list of float32). Requires polars and pyarrow (``uv add 'arcadedb-jvm[polars]'``). Args: @@ -942,7 +805,13 @@ def to_columns( Null handling follows pandas conventions: int/datetime columns with nulls are promoted to float64 with NaN / datetime64 NaT; string and JSON columns use None. A DECIMAL column is an object array of exact - ``Decimal`` values (#115). + ``Decimal`` values (#115). The type of a column belongs to the whole + result, not to where a batch ends: a batch that lacks the column or + holds only nulls for it takes the type of the batches that have + values. The arrays are writable. A vector column is a ``(count, dim)`` matrix + and a null vector is a row of NaN in it (``to_dataframe()`` and ``to_arrow()`` keep + it a null). An empty result is ``{}``, or, when + ``columns`` is given, those columns as empty lists. The columns are the union of every row's property names, in order of first appearance, because a document is schemaless: a property the @@ -968,142 +837,48 @@ def _read_columns(self, batch_size, columns, micros): column of ``date`` objects, which is what ``to_dataframe()`` needs to return the values ``to_list()`` does.""" try: - import numpy as np + __import__("numpy") # availability check except ImportError: return None - import json - column_batcher = _bridge_class("ColumnBatcher") if column_batcher is None: return None # Every batch reports the union of its own rows' property names. The - # result's columns are the union over the batches, in order of first - # appearance (#113); a batch lacking one gets nulls for it when merged. + # result's columns are the union over the batches, in order of + # first appearance (#113); a batch lacking one gets nulls for it when merged. names: List[str] = [] batches: List[Tuple[int, Dict[str, Any]]] = [] - def decode_batch(buf, side): - hlen = int.from_bytes(buf[:4], "little") - header = json.loads(bytes(buf[4 : 4 + hlen])) - count = header["count"] - if count == 0: - return 0 - batch_parts: Dict[str, Any] = {} - pos = 4 + hlen - for col in header["cols"]: - name, ctype = col["name"], col["type"] - nulls_len = col["nulls"] - null_bits = np.frombuffer(buf[pos : pos + nulls_len], dtype=np.uint8) - has_nulls = bool(null_bits.any()) - if has_nulls: - mask = np.unpackbits(null_bits, bitorder="little")[:count].astype( - bool - ) - pos += nulls_len - data = buf[pos : pos + col["bytes"]] - pos += col["bytes"] - - if ctype == "i8": - arr = np.frombuffer(data, dtype=" 2-D array (count, dim); - # Java already writes NaN rows into null slots - # an empty vector has no "dim" in the header - dim = col.get("dim", 0) - dt = " 1 else parts[0] - continue - except Exception: # nosec B110 - fall through to generic merge - pass - column = [] - for p in parts: - column.extend(p if isinstance(p, list) else p.tolist()) - out[name] = column - return out + for name in parts: + if name not in names: + names.append(name) + batches.append((count, parts)) + + return merge_columns(names, batches, columns) def to_arrow( self, batch_size: int = 25_000, columns: Optional[Sequence[str]] = None @@ -1135,13 +910,11 @@ def to_arrow( falls back to the row-based paths. """ try: - import numpy as np - import pyarrow as pa + __import__("numpy") # availability check + __import__("pyarrow") # availability check except ImportError: return None - import json - column_batcher = _bridge_class("ColumnBatcher") if column_batcher is None: return None @@ -1152,141 +925,30 @@ def to_arrow( names: List[str] = [] batches: List[Tuple[int, Dict[str, Any]]] = [] - def validity(null_bits, count, has_nulls): - """Arrow validity is 1=valid; the bridge writes 1=null, so invert. - Bits past `count` are padding and Arrow ignores them.""" - if not has_nulls: - return None - return pa.py_buffer(bytes(np.invert(null_bits).tobytes())) - - def decode_batch(buf, side): - hlen = int.from_bytes(buf[:4], "little") - header = json.loads(bytes(buf[4 : 4 + hlen])) - count = header["count"] - if count == 0: - return 0 - batch_chunks: Dict[str, Any] = {} - pos = 4 + hlen - for col in header["cols"]: - name, ctype = col["name"], col["type"] - nulls_len = col["nulls"] - null_bits = np.frombuffer(buf[pos : pos + nulls_len], dtype=np.uint8) - has_nulls = bool(null_bits.any()) - mask = None - if has_nulls: - mask = np.unpackbits(null_bits, bitorder="little")[:count].astype( - bool - ) - pos += nulls_len - data = buf[pos : pos + col["bytes"]] - pos += col["bytes"] - - if ctype == "i8": - # kept as int64 WITH validity, where to_columns would have - # promoted the whole column to float64/NaN - arr = pa.array(np.frombuffer(data, dtype=" str: return text -def json_bulk_dumps(rows: Any) -> str: - """json.dumps for the bulk paths: raises ValueError (or TypeError) when - any value in `rows` would change on the way to the engine. +_DICT_ONLY = frozenset((dict,)) +# the kinds of a column whose text json.dumps itself decides: a NaN or Infinity it refuses (allow_nan=False), a subnormal `_dumps_checked` refuses +_TEXT_SAFE = frozenset((type(None), bool, float)) +_STR_ONLY = frozenset((str,)) + - `rows` is a list of dicts (one per record). The common value types are - checked in this loop without a call per value, because every value of a - large load passes through it; floats need no check here, since - `allow_nan=False` makes json.dumps itself refuse NaN and the infinities. +def _check_rows(rows: Any, tagged: bool) -> None: + """The per-row walk: raise ValueError (or TypeError) for the first row, in row order, that holds a value the JSON text would change. + + This is the exact reference. The bulk dumps do not run it on every call: `_dumps_once` runs it only when the text or the types of + the rows show that a value may be bad, and when json.dumps itself refused the rows, so the error is always the walk's. + `tagged` is insert_many's format (see json_bulk_dumps_tagged), where the kinds bulk_tag_default writes are allowed in a row. """ int_min, int_max = _INT64_MIN, _INT64_MAX + kinds = _TAGGED_KINDS for row in rows: for key in row: if type(key) is not str: @@ -832,11 +839,107 @@ def json_bulk_dumps(rows: Any) -> str: ) elif kind is float or kind is bool or value is None: continue + elif tagged and kind in kinds: + continue # bulk_tag_default checks it as json.dumps writes it + elif tagged and kind is dict and _TAG in value: + raise ValueError( + "a map keyed by the tag marker cannot go through the JSON bulk path" + ) + elif tagged and ( + isinstance(value, (datetime, date, Decimal, bytes, bytearray)) + or (_np is not None and isinstance(value, _np.generic)) + or _is_pandas_null(kind) + ): + continue # a subclass (pandas.Timestamp), a numpy scalar, pandas NA or NaT else: _json_bulk_check( value ) # nested lists and dicts, and subclasses of the types above - return _dumps_checked(rows) + + +def _kind_is_tagged(kind: type) -> bool: + """Whether bulk_tag_default (not the walk) checks a value of this type: the kinds `_check_rows` lets through in tagged mode.""" + return ( + kind in _TAGGED_KINDS + or issubclass(kind, (datetime, date, Decimal, bytes, bytearray)) + or (_np is not None and issubclass(kind, _np.generic)) + or _is_pandas_null(kind) + ) + + +def _rows_need_walk(rows: Any, tagged: bool) -> bool: + """Whether the rows may hold a value the JSON text would change, decided from their types and their columns, in C. + + Only exact dict rows with str keys can be decided here. Each column is one list of its values and one census of their types; + a column of the exact scalar types is screened (`_scalar_column_suspect`: the 64-bit range of its ints, the UTF-8 form of its + strs), a column of the kinds bulk_tag_default writes (tagged) holds nothing the walk has to see, and any other type, or a + column of mixed kinds, sends the rows to the walk. A false alarm costs the walk and changes nothing else. This replaces two + scans of the JSON text with regular expressions (a lone surrogate, an integer token of 2**63 and above): the one for the + integer token cost 2.5 microseconds a row of a three-column table (12 ms of a 5,000-row call), a column screen about 0.2. + """ + if not set(map(type, rows)) <= _DICT_ONLY: + return True + keys = set(itertools.chain.from_iterable(rows)) + if not set(map(type, keys)) <= _STR_ONLY: + return True + for key in keys: + column = list(map(dict.get, rows, itertools.repeat(key))) + kinds = set(map(type, column)) + extra = kinds - _BULK_SCALAR_TYPES + if extra: + if not (tagged and all(map(_kind_is_tagged, extra))): + return True + if kinds - extra - {_NONE_KIND}: + return True # tagged kinds beside scalars in one column: the walk judges each value + continue # bulk_tag_default checks these as json.dumps writes them + if not kinds <= _TEXT_SAFE and _scalar_column_suspect(column, kinds): + return True + return False + + +def _dumps_once(rows: Any, tagged: bool, vetted: bool = False) -> str: + """One json.dumps pass for the bulk paths, with the checks of `_check_rows` decided from the types and columns of the rows. + + A str or int the text would change cannot be seen by a json.dumps hook (the encoder writes both itself), so the rows decide: + `_rows_need_walk` screens their columns in C and sends the rows to the walk when a value may be bad or its type is one it cannot + vouch for. When json.dumps itself refuses the rows (NaN, an unknown type, a subnormal, a hook error) the walk runs first, as the + validation loop used to run before json.dumps, so the error of the first bad row in row order is still the walk's; if the walk + finds nothing, json.dumps's own error stands. A false alarm costs the walk and changes nothing else. + + `vetted` is for a caller that has already screened these rows with the same census (`screen_vertex_rows`: every value an exact + scalar, no NaN, Infinity, subnormal, integer beyond 64 bits or lone surrogate): the text is written without a second look. + """ + try: + if vetted: + text = json.dumps(rows, allow_nan=False) + elif tagged: + text = _dumps_checked(rows, default=bulk_tag_default) + else: + text = _dumps_checked(rows) + except (TypeError, ValueError) as error: + refused = error + else: + refused = None + if refused is not None: + # outside the except block, so the walk's error does not carry json.dumps's as __context__ (the old loop ran first) + try: + _check_rows(rows, tagged) + raise refused + finally: + del refused + if not vetted and _rows_need_walk(rows, tagged): + _check_rows(rows, tagged) + return text + + +def json_bulk_dumps(rows: Any, vetted: bool = False) -> str: + """json.dumps for the bulk paths: raises ValueError (or TypeError) when + any value in `rows` would change on the way to the engine. + + `rows` is a list of dicts (one per record). The rows are written once; see `_dumps_once` for how the values are checked and + for `vetted`. + """ + return _dumps_once(rows, False, vetted) # --------------------------------------------------------------------------- @@ -933,43 +1036,7 @@ def json_bulk_dumps_tagged(rows: Any) -> str: through, written by bulk_tag_default. Raises TypeError or ValueError exactly where json_bulk_dumps does for every other value (a datetime inside a list or map included), so the caller can send that row down the per-row path. """ - int_min, int_max = _INT64_MIN, _INT64_MAX - tagged = _TAGGED_KINDS - for row in rows: - for key in row: - if type(key) is not str: - raise ValueError("a non-str key cannot go through the JSON bulk path") - for value in row.values(): - kind = type(value) - if kind is str: - if not value.isascii() and not _str_survives_json(value): - raise ValueError( - "a string with a lone surrogate cannot go through the JSON bulk path" - ) - elif kind is int: - if not int_min <= value <= int_max: - raise ValueError( - f"an integer beyond 64 bits cannot go through the JSON bulk path: {value}" - ) - elif kind is float or kind is bool or value is None: - continue - elif kind in tagged: - continue # bulk_tag_default checks it as json.dumps writes it - elif kind is dict and _TAG in value: - raise ValueError( - "a map keyed by the tag marker cannot go through the JSON bulk path" - ) - elif ( - isinstance(value, (datetime, date, Decimal, bytes, bytearray)) - or (_np is not None and isinstance(value, _np.generic)) - or _is_pandas_null(kind) - ): - continue # a subclass (pandas.Timestamp), a numpy scalar, pandas NA or NaT - else: - _json_bulk_check( - value - ) # nested lists and dicts, and subclasses of the types above - return _dumps_checked(rows, default=bulk_tag_default) + return _dumps_once(rows, True) def bulk_row_ok(row: Any, skip=()) -> bool: @@ -983,6 +1050,178 @@ def bulk_row_ok(row: Any, skip=()) -> bool: return True +_ROWS_OR_NONE = frozenset((dict, type(None))) + + +def _column(rows: Any, key: str): + """The value of `key` in every non-empty row (None where the row lacks it), as one C-level iterator.""" + return map(dict.get, filter(None, rows), itertools.repeat(key)) + + +_NONE_KIND = type(None) +_SCREEN_BLOCK = 100_000 # rows per block of the scalar screen, so its transient lists and joined texts stay bounded + + +def _scalar_column_suspect(column: list, kinds: Any = None) -> bool: + """Whether one block of a column of exact scalar values may hold a value the JSON text would change. + + Conservative (a false alarm only sends the column to the exact per-value judgement) and run in C: the type census, the joined + text of a str column (a lone surrogate has no UTF-8 form), min and max of an int column (the 64-bit range), and a float column as + one float64 array (NaN, Infinity, a subnormal). Columns of mixed kinds are suspect. `kinds` is the set of the types of `column` + when the caller has taken that census already (it is not changed). + """ + kinds = set(map(type, column)) if kinds is None else set(kinds) + if _NONE_KIND in kinds: + column = [value for value in column if value is not None] + kinds.discard(_NONE_KIND) + if kinds <= {bool}: + return False + if len(kinds) != 1: + return True + (kind,) = kinds + if kind is str: + text = "".join(column) + if text.isascii(): + return False + try: + text.encode("utf-8") + except UnicodeEncodeError: + return True + return False + if kind is int: + return not (_INT64_MIN <= min(column) and max(column) <= _INT64_MAX) + if kind is float and _np is not None: + values = _np.fromiter(column, dtype=_np.float64, count=len(column)) + if not _np.isfinite(values).all(): + return True + values = values[values != 0.0] + return bool((_np.abs(values) < _FLOAT_NORMAL_MIN).any()) + return True + + +def _screen_key(rows: Any, key: str, screen: bool): + """(odd, suspect) for the column `key` of all `rows`: the types in it beyond the exact scalars, and, with `screen`, whether a column + of exact scalars only may hold a value the JSON text would change. One list and one census per block of `_SCREEN_BLOCK` rows. + """ + odd = set() + suspect = False + for start in range(0, len(rows), _SCREEN_BLOCK): + block = ( + rows if len(rows) <= _SCREEN_BLOCK else rows[start : start + _SCREEN_BLOCK] + ) + column = list(_column(block, key)) + kinds = set(map(type, column)) + extra = kinds - _BULK_SCALAR_TYPES + if extra: + odd |= extra + elif screen and not suspect: + suspect = _scalar_column_suspect(column, kinds) + return odd, suspect + + +def _classify_rows(rows: Any, declared: Any, screen_scalars: bool): + """The verdict of classify_vertex_rows and screen_vertex_rows: None, or ``(tagged, vetted)``. + + `vetted` is True when every column of the rows holds exact scalars only and `screen_scalars` found nothing the text would change + in it (a column the screen flagged counts when every value passed the judgement by value): then no chunk of these rows needs a + check of its own.""" + if not set(map(type, rows)) <= _ROWS_OR_NONE: + tagged = _classify_walk(rows, declared) + return ( + None if tagged is None else (tagged, False) + ) # a row that is no dict: the walk raises what it always raised + keys = set(itertools.chain.from_iterable(filter(None, rows))) + if not set(map(type, keys)) <= _STR_ONLY: + return None + tagged = False + vetted = screen_scalars + for key in keys: + odd, suspect = _screen_key(rows, key, screen_scalars) + if odd: + vetted = False + if key in declared and all(_is_vector_kind(kind) for kind in odd): + continue # the vectors of a declared column + if all(_is_float_array_kind(kind) for kind in odd): + continue # a column of arrays: take_vector_columns takes the float32 ones; a chunk it leaves goes the matrix way + elif not suspect: + continue # exact scalars; with screen_scalars a column that passed the screen + for value in _column(rows, key): + if (type(value) in _BULK_SCALAR_TYPES and not screen_scalars) or ( + key in declared and is_vector_candidate(value) + ): + continue + if ( + screen_scalars + and type(value) is float + and 0.0 < abs(value) < _FLOAT_NORMAL_MIN + ): + return None # a subnormal: the text is read back as a DECIMAL + if not json_bulk_scalar_ok(value): + if not json_bulk_shape_ok(value): + return None + tagged = True + return tagged, vetted and not tagged + + +def classify_vertex_rows(rows: Any, declared: Any, screen_scalars: bool = False): + """What GraphBatch.create_vertices must know about its rows before it writes any chunk, decided by types. + + `screen_scalars` is for an input of several chunks, which are sent one after the other: a value the text would change in a late + chunk would leave the earlier ones stored from the JSON text (an int as an Integer) and the late one from the matrix path (a Long), + so the stored type of a property would depend on which chunk held it. With `screen_scalars` the columns of exact scalars are + screened in C over the whole input first (`_screen_key`), and a column that may hold a bad value is judged value by value, so a + bad scalar anywhere gives None for the whole input. + + Returns None when the rows must all go the property-matrix way (a property name that is not a str, or a value of a type the + tagged JSON does not carry: a set, a numpy or pandas value, a list or tuple of only numbers, any other object); otherwise True when + some value needs the tagged text (json_bulk_dumps_tagged) and False when json_bulk_dumps carries every row. A value of an exact + scalar type is not judged here unless `screen_scalars`: whether the text carries it (a NaN, an integer beyond 64 bits, a lone + surrogate) is then decided in the dumps of its chunk. The property of `declared` (ARRAY_OF_FLOATS) that holds a list or a numpy + array is left to split_float_vectors. The census over the keys and the values runs in C; a Python-level loop runs only over a + column that holds a type other than the exact scalars, or that the screen flagged. + """ + verdict = _classify_rows(rows, declared, screen_scalars) + return None if verdict is None else verdict[0] + + +def screen_vertex_rows(rows: Any, declared: Any): + """classify_vertex_rows with the whole-input screen, for any number of chunks: None, or ``(tagged, vetted)``. + + `vetted` says that every value of every row is an exact scalar the JSON text carries unchanged (a column holding any other type, + a vector or a tagged kind, is not vetted). The chunks of such rows are then written by ``json_bulk_dumps(chunk, vetted=True)`` + with no check of their own. One chunk is screened here too, so its values are looked at once, as columns in C, and not again in + the text: a value the text would change sends all of it the property-matrix way, which is what its own dumps did before. + """ + return _classify_rows(rows, declared, True) + + +def _classify_walk(rows: Any, declared: Any): + tagged = False + for row in rows: + if row: + for key, value in row.items(): + if type(key) is not str: + return None + if key in declared and is_vector_candidate(value): + continue + if not json_bulk_scalar_ok(value): + if not json_bulk_shape_ok(value): + return None + tagged = True + return tagged + + +def declared_scalars_ok(rows: Any, declared: Any) -> bool: + """False when a declared ARRAY_OF_FLOATS property still holds a list (or any non-scalar) after the vector split.""" + for key in declared: + if set(map(type, _column(rows, key))) <= _BULK_SCALAR_TYPES: + continue # scalars: the dumps of the chunk decides whether the text carries them + for row in rows: + if row and key in row and not json_bulk_scalar_ok(row[key]): + return False + return True + + _FLOAT_VECTORS = None # the bridge's FloatVectors class once resolved; False when the bridge jar lacks it @@ -1004,41 +1243,103 @@ def declared_float_vectors(java_db, type_name): return [str(name) for name in vectors.declared(java_db, type_name)] +_JAVA_VECTOR_ARRAYS = None # (float[] class, double[] class) of JPype once the JVM runs; () where there is no JVM (the native wheel) + + +def _java_vector_arrays(): + """The JPype classes of a Java ``float[]`` and a ``double[]`` (what ``to_java_float_array`` returns on the JVM wheel), () without a JVM. + + Resolved once the JVM has started: JPype builds an array class from a Java class, so it cannot be done at import. + """ + global _JAVA_VECTOR_ARRAYS + if _JAVA_VECTOR_ARRAYS is None: + try: + if not jpype.isJVMStarted(): + return () + _JAVA_VECTOR_ARRAYS = ( + jpype.JArray(jpype.JFloat), + jpype.JArray(jpype.JDouble), + ) + except Exception: # the native wheel's stand-in for jpype has no such classes + _JAVA_VECTOR_ARRAYS = () + return _JAVA_VECTOR_ARRAYS + + +def _is_vector_kind(kind): + """The type of a value that a declared ARRAY_OF_FLOATS column may hold as a vector: a list, a numpy array, a Java float[] or double[].""" + return ( + kind is list + or (_np is not None and issubclass(kind, _np.ndarray)) + or issubclass(kind, _java_vector_arrays()) + ) + + +def _is_float_array_kind(kind): + """The type of a value that may be a float32 vector on any property: a numpy array or a Java float[].""" + arrays = _java_vector_arrays() + return (_np is not None and issubclass(kind, _np.ndarray)) or ( + bool(arrays) and issubclass(kind, arrays[0]) + ) + + def is_vector_candidate(value): - """A value split_float_vectors may take out of the JSON: a list or a numpy array.""" - return type(value) is list or (_np is not None and isinstance(value, _np.ndarray)) + """A value split_float_vectors may take out of the JSON: a list, a numpy array, or a Java float[] or double[].""" + return _is_vector_kind(type(value)) -def split_float_vectors(java_db, type_name, rows, declared=None): - """Take the float vectors of `rows` out of the JSON text of a bulk load. +def _is_float32_vector(value): + """A numpy float32 array or a Java float[]: the value the property-matrix path stores as a float[] on any property.""" + if _np is not None and isinstance(value, _np.ndarray): + return value.dtype == _np.float32 + arrays = _java_vector_arrays() + return bool(arrays) and isinstance(value, arrays[0]) - A list of floats costs Python about 450 ns a number to write as text and the - engine about as much again to parse it, so a vector load spent most of its - time on the text. For each property of `type_name` that is declared - ARRAY_OF_FLOATS, whose value in EVERY row is a list or a numpy array of - numbers of one length, the vectors cross as one float32 matrix instead - (``FloatVectors.Matrix``, one buffer copy). The engine converts the numbers of a list to float for - such a property, so the float32 matrix stores the same values. Every other - property, and a column that is missing in a row, ragged, not numeric, or - null in a row, stays in the JSON. - Returns ``(rest_rows, names, matrices)``: `rest_rows` are the rows without - those properties, `names` a ``String[]`` and `matrices` an ``Object[]`` of - ``FloatVectors.Matrix`` for the bridge; or None when no property qualifies - (the caller sends `rows` as they are). +def _without_keys(rows, gone): + """`rows` (each one holds every name of `gone`) as new dicts without those keys; the values are the same objects. + + Rows that all have the same keys in the same order are rebuilt from columns by C iterators (itemgetter, zip, dict), with no Python + code per row; any other rows take the dict comprehension. + """ + if len(set(map(tuple, rows))) == 1: + keep = [key for key in rows[0] if key not in gone] + if not keep: + return list(map(dict, itertools.repeat((), len(rows)))) + columns = [list(map(operator.itemgetter(key), rows)) for key in keep] + return list(map(dict, map(zip, itertools.repeat(keep), zip(*columns)))) + gone = set(gone) + return [{k: v for k, v in row.items() if k not in gone} for row in rows] + + +def take_vector_columns(rows, declared, undeclared=False): + """The float vector columns of `rows` as float32 matrices, and the rows without them (the one split of both wheels). + + A declared (ARRAY_OF_FLOATS) property qualifies when its value in EVERY row is a list, a numpy array or a Java float[] or double[] of + numbers of one length: the engine converts the numbers to float for such a property, so a float32 matrix stores the same values. + With `undeclared`, any other property qualifies when its value in every row is a float32 numpy array or a Java float[] of one length + (the first row decides which names are tried, the stacked column must come out as float32): the property-matrix path stores such a + value as a float[], and so does the matrix. A float64 array, a list, a Java double[] on an undeclared property, a missing value, a + null, a ragged or non-numeric column does not qualify and stays in the rows. + + Returns None when no column qualifies, else ``(names, matrices, rest)``: the matrices are C-contiguous float32 arrays of shape + ``(len(rows), dim)``, `rest` the rows without those properties. """ if _np is None or not rows: return None - vectors = _float_vectors_class() - if vectors is None: - return None - if declared is None: - declared = declared_float_vectors(java_db, type_name) + declared = set(declared) + names = list(declared) + first = rows[0] + if undeclared and isinstance(first, dict): + names.extend( + key + for key, value in first.items() + if key not in declared and _is_float32_vector(value) + ) taken = [] matrices = [] - for name in declared: + for name in names: try: - column = [row[name] for row in rows] + column = list(map(operator.itemgetter(name), rows)) except (KeyError, TypeError): continue if not is_vector_candidate(column[0]): @@ -1049,24 +1350,72 @@ def split_float_vectors(java_db, type_name, rows, declared=None): continue if array.ndim != 2 or array.shape[1] == 0 or array.dtype.kind not in "fiu": continue + if name not in declared and array.dtype != _np.float32: + continue with _np.errstate( all="ignore" ): # a double beyond float32 becomes Infinity, as in the engine - flat = _np.ascontiguousarray(array, dtype=_np.float32).reshape(-1) + matrix = _np.ascontiguousarray(array, dtype=_np.float32) taken.append(name) - matrices.append( - vectors.Matrix(jpype.JArray(jpype.JFloat)(flat), int(array.shape[1])) - ) + matrices.append(matrix) if not taken: return None - if len(taken) == 1: - only = taken[0] - rest = [{k: v for k, v in row.items() if k != only} for row in rows] - else: - gone = set(taken) - rest = [{k: v for k, v in row.items() if k not in gone} for row in rows] + return taken, matrices, _without_keys(rows, taken) + + +def vector_width(rows, declared): + """How many floats the vector columns of the first row hold (0 when it holds none): a list, array, Java ``float[]`` or ``double[]`` + in a declared property, or a float32 array or Java ``float[]`` in any property. GraphBatch.create_vertices sizes its crossings by it. + """ + first = rows[0] if rows else None + if not isinstance(first, dict): + return 0 + width = 0 + for key, value in first.items(): + if (key in declared and is_vector_candidate(value)) or _is_float32_vector( + value + ): + try: + width += len(value) + except TypeError: # a 0-dimensional array + pass + return width + + +def split_float_vectors(java_db, type_name, rows, declared=None, undeclared=False): + """Take the float vectors of `rows` out of the JSON text of a bulk load (the JVM wheel). + + A list of floats costs Python about 450 ns a number to write as text and the + engine about as much again to parse it, so a vector load spent most of its + time on the text. The vectors of the columns that take_vector_columns picks + cross as one float32 matrix per property instead (``FloatVectors.Matrix``, + one buffer copy). Every other property, and a column that is missing in a + row, ragged, not numeric, or null in a row, stays in the JSON. + + Returns ``(rest_rows, names, matrices)``: `rest_rows` are the rows without + those properties, `names` a ``String[]`` and `matrices` an ``Object[]`` of + ``FloatVectors.Matrix`` for the bridge; or None when no property qualifies + (the caller sends `rows` as they are). + """ + if _np is None or not rows: + return None + vectors = _float_vectors_class() + if vectors is None: + return None + if declared is None: + declared = declared_float_vectors(java_db, type_name) + taken = take_vector_columns(rows, declared, undeclared) + if taken is None: + return None + names, matrices, rest = taken + float_array = jpype.JArray(jpype.JFloat) return ( rest, - jpype.JArray(jpype.JString)(taken), - jpype.JArray(jpype.JObject)(matrices), + jpype.JArray(jpype.JString)(names), + jpype.JArray(jpype.JObject)( + [ + vectors.Matrix(float_array(matrix.reshape(-1)), int(matrix.shape[1])) + for matrix in matrices + ] + ), ) diff --git a/src/java/com/arcadedb/python/ColumnBatcher.java b/src/java/com/arcadedb/python/ColumnBatcher.java index cf971a2..168b285 100644 --- a/src/java/com/arcadedb/python/ColumnBatcher.java +++ b/src/java/com/arcadedb/python/ColumnBatcher.java @@ -151,16 +151,22 @@ public static Object[] nextColumnBatch(final ResultSet rs, final int max, final final byte[] nulls = new byte[(count + 7) / 8]; // detect the column type across ALL values; mixed/unsupported -> json String type = null; + boolean sawInt = false, sawFloating = false, bigInt = false; for (int r = 0; r < count; r++) { final Object v = rows.get(r)[c]; if (v == null) continue; final String t; - if (v instanceof Long || v instanceof Integer || v instanceof Short || v instanceof Byte) + if (v instanceof Long || v instanceof Integer || v instanceof Short || v instanceof Byte) { t = "i8"; - else if (v instanceof Double || v instanceof Float) + sawInt = true; + final long l = ((Number) v).longValue(); + if (l > (1L << 53) || l < -(1L << 53)) + bigInt = true; + } else if (v instanceof Double || v instanceof Float) { t = "f8"; - else if (v instanceof Boolean) + sawFloating = true; + } else if (v instanceof Boolean) t = "b1"; else if (v instanceof Date || v instanceof Calendar || v instanceof LocalDateTime || v instanceof LocalDate || v instanceof Instant @@ -198,6 +204,9 @@ else if (!type.equals(t)) { } if (type == null) type = "str"; // all-null column + // an integer beyond 2**53 next to a floating value has no exact double: the values stay what they are, in a json column + if (type.equals("f8") && sawInt && sawFloating && bigInt) + type = "json"; int vdim = -1; if (type.equals("f4v") || type.equals("f8v")) { diff --git a/src/java/com/arcadedb/python/DocumentBatcher.java b/src/java/com/arcadedb/python/DocumentBatcher.java index bbcae4f..0edeb45 100644 --- a/src/java/com/arcadedb/python/DocumentBatcher.java +++ b/src/java/com/arcadedb/python/DocumentBatcher.java @@ -31,6 +31,7 @@ import com.arcadedb.database.Database; import com.arcadedb.database.MutableDocument; import com.arcadedb.database.async.ErrorCallback; +import com.arcadedb.exception.NeedRetryException; import com.arcadedb.schema.DocumentType; import com.arcadedb.schema.EdgeType; import com.arcadedb.schema.VertexType; @@ -38,6 +39,7 @@ import com.arcadedb.serializer.json.JSONObject; import java.util.Map; +import java.util.function.IntConsumer; import java.util.concurrent.atomic.AtomicLong; import java.util.concurrent.atomic.AtomicReference; @@ -69,76 +71,106 @@ public static long insertManyJson(final Database db, final String typeName, fina // commits and the failure rollback apply only to the transactions opened // here, matching the Python per-row fallback (#7882). final boolean vertex = n > 0 && isVertexType(db, typeName); - final boolean wasActive = db.isTransactionActive(); - if (!wasActive) - db.begin(); - try { - for (int i = 0; i < n; i++) { - final MutableDocument doc = newRecord(db, typeName, vertex); - fill(doc, rows.getJSONObject(i), i, vectorNames, vectors); - doc.save(); - if (!wasActive && commitEvery > 0 && (i + 1) % commitEvery == 0) { - db.commit(); - db.begin(); - } - } - if (!wasActive) - db.commit(); - } catch (final Throwable e) { - if (!wasActive && db.isTransactionActive()) { - try { - db.rollback(); - } catch (final Throwable rollbackError) { - e.addSuppressed(rollbackError); - } - } - throw e; - } + writeChunks(db, n, commitEvery, i -> { + final MutableDocument doc = newRecord(db, typeName, vertex); + fill(doc, rows.getJSONObject(i), i, vectorNames, vectors); + doc.save(); + }); return n; } /** * Columnar insert: the documents are built Java-side from whole columns, so each column crosses the FFI once (a long[] or - * double[] copied from a numpy buffer, a boolean[], or an Object[] of Strings, boxed numbers, and nulls) instead of one + * double[] copied from a numpy buffer, a boolean[], a StringColumn (one UTF-8 byte[] plus the int[] offsets), or an Object[] of + * boxed numbers, Strings of a mixed column, and nulls) instead of one * JSON text per batch (bindings issue #150: 2.24x over insertManyJson on the first * 2,000,000 TPC-H SF1 line items, nine typed properties, commit every 10,000, same sums and count). * * Same failure contract as insertManyJson (#7882): batch commits and the failure rollback apply only to a transaction * opened here; a caller's own transaction is the caller's to commit or roll back. * - * @param columns one array per name, each of length n: long[], double[], boolean[] or Object[]; a null element sets - * the property to null, as a JSON null does on the insertManyJson path + * @param columns one array per name, each of length n: long[], double[], boolean[], Object[], a FloatVectors.Matrix or a + * StringColumn; a null element (or null row of a StringColumn) sets the property to null, as a JSON null does + * on the insertManyJson path */ public static long insertColumns(final Database db, final String typeName, final String[] names, final Object[] columns, final int n, final int commitEvery) { checkColumns(names, columns, n); final boolean vertex = n > 0 && isVertexType(db, typeName); - final boolean wasActive = db.isTransactionActive(); - if (!wasActive) - db.begin(); - try { - for (int i = 0; i < n; i++) { - final MutableDocument doc = newRecord(db, typeName, vertex); - fillRow(doc, names, columns, i); - doc.save(); - if (!wasActive && commitEvery > 0 && (i + 1) % commitEvery == 0) { - db.commit(); - db.begin(); - } - } - if (!wasActive) - db.commit(); - } catch (final Throwable e) { - if (!wasActive && db.isTransactionActive()) { + writeChunks(db, n, commitEvery, i -> { + final MutableDocument doc = newRecord(db, typeName, vertex); + fillRow(doc, names, columns, i); + doc.save(); + }); + return n; + } + + /** + * The retries of a chunk and the back-off before them are the engine's GraphBatch defaults (commitRetries 10, commitRetryDelayMs 1000, + * doubling up to 10 s), so a load that went through createVertices before keeps the same patience. + */ + private static final int COMMIT_RETRIES = 10; + private static final long COMMIT_RETRY_DELAY_MS = 1_000L; + private static final long MAX_COMMIT_RETRY_DELAY_MS = 10_000L; + + /** + * Writes rows [0, n) with writeRow (build and save row i), in transactions opened here: one per commitEvery rows (0: one for all). A + * chunk whose save or commit raises a NeedRetryException (an MVCC conflict with another writer, a lock timeout) is rolled back and + * written again from its first row after a back-off, up to COMMIT_RETRIES times; after the last attempt the error is thrown unchanged. + * Any other failure rolls the open chunk back and is thrown; the chunks committed before it stay. A transaction the caller opened is the + * caller's: the rows join it, nothing is committed, rolled back or retried here. + */ + private static void writeChunks(final Database db, final int n, final int commitEvery, final IntConsumer writeRow) { + if (db.isTransactionActive()) { + for (int i = 0; i < n; i++) + writeRow.accept(i); + return; + } + for (int from = 0; from < n;) { + final int to = commitEvery > 0 ? (int) Math.min((long) n, (long) from + commitEvery) : n; + for (int attempt = 0;;) { try { - db.rollback(); - } catch (final Throwable rollbackError) { - e.addSuppressed(rollbackError); + db.begin(); + for (int i = from; i < to; i++) + writeRow.accept(i); + db.commit(); + break; + } catch (final NeedRetryException e) { + if (!rollbackQuietly(db, e) || ++attempt > COMMIT_RETRIES) + throw e; + backoffBeforeRetry(attempt); + } catch (final Throwable e) { + rollbackQuietly(db, e); + throw e; } } - throw e; + from = to; + } + } + + /** Rolls back the open transaction, if any; false when the rollback itself failed (the failure is added to e as suppressed). */ + private static boolean rollbackQuietly(final Database db, final Throwable e) { + if (!db.isTransactionActive()) + return true; + try { + db.rollback(); + return true; + } catch (final Throwable rollbackError) { + e.addSuppressed(rollbackError); + return false; + } + } + + private static void backoffBeforeRetry(final int attempt) { + long delay = COMMIT_RETRY_DELAY_MS; + for (int i = 1; i < attempt && delay < MAX_COMMIT_RETRY_DELAY_MS; i++) + delay = Math.min(delay * 2, MAX_COMMIT_RETRY_DELAY_MS); + try { + Thread.sleep(delay); + } catch (final InterruptedException ie) { + Thread.currentThread().interrupt(); + throw new NeedRetryException("Interrupted while waiting to retry the commit of a bulk-insert chunk"); } - return n; } /** @@ -195,9 +227,11 @@ else if (col instanceof Object[] a) len = a.length; else if (col instanceof FloatVectors.Matrix m) len = m.rows(); + else if (col instanceof StringColumn s) + len = s.rows(); else throw new IllegalArgumentException("column '" + names[c] + "' is a " + (col == null ? "null" : col.getClass().getName()) - + ", not a long[], double[], boolean[], Object[] or FloatVectors.Matrix"); + + ", not a long[], double[], boolean[], Object[], FloatVectors.Matrix or StringColumn"); if (len != n) throw new IllegalArgumentException("column '" + names[c] + "' has " + len + " values for " + n + " rows"); } @@ -215,6 +249,8 @@ else if (col instanceof boolean[] a) v = a[i]; else if (col instanceof FloatVectors.Matrix m) v = m.row(i); + else if (col instanceof StringColumn s) + v = s.get(i); else v = ((Object[]) col)[i]; doc.set(names[c], v); diff --git a/src/java/com/arcadedb/python/StringColumn.java b/src/java/com/arcadedb/python/StringColumn.java new file mode 100644 index 0000000..0946f12 --- /dev/null +++ b/src/java/com/arcadedb/python/StringColumn.java @@ -0,0 +1,61 @@ +/* + * Python-bindings bridge: a string column of Database.insert_columns that crosses as one UTF-8 blob plus an int32 offsets array. + * + * The JPype guidance names one conversion per element as the slow way across the boundary, and an Object[] of Strings costs one + * conversion per row (and one Java String per distinct value, all alive at once). Here the whole column is ONE byte[] (the UTF-8 + * text of every row, back to back), ONE int[] of row + 1 offsets into it, and optionally ONE boolean[] marking the null rows. + * Row i is blob[offsets[i], offsets[i + 1]), decoded when the document of that row is built, so the Strings are short-lived. + * + * The decoded String is the one Python's own str holds: UTF-8 round-trips every code point, so a character beyond the BMP becomes + * its surrogate pair, U+0000 stays a NUL and an empty row is the empty String. Python sends no column it cannot encode (a lone + * surrogate has no UTF-8 form): that column keeps the per-element path. + * + * Compiled into arcadedb-python-bridge.jar during the wheel build and consumed by DocumentBatcher.insertColumns() and + * insertColumnsParallel(). + */ +package com.arcadedb.python; + +import java.nio.charset.StandardCharsets; + +public final class StringColumn { + private final byte[] blob; + private final int[] offsets; + private final boolean[] nulls; + private final int rows; + + /** + * @param blob the UTF-8 text of all rows, back to back + * @param offsets rows + 1 non-decreasing offsets into {@code blob}, the first 0 and the last at most {@code blob.length} + * @param nulls null for a column without a null, else one flag per row (a flagged row has an empty range and reads as null) + */ + public StringColumn(final byte[] blob, final int[] offsets, final boolean[] nulls) { + if (offsets.length < 1) + throw new IllegalArgumentException("a string column needs rows + 1 offsets, got none"); + this.rows = offsets.length - 1; + if (offsets[0] != 0) + throw new IllegalArgumentException("the first offset of a string column must be 0, got " + offsets[0]); + for (int i = 0; i < rows; i++) + if (offsets[i + 1] < offsets[i]) + throw new IllegalArgumentException("the offsets of a string column must not decrease (row " + i + ")"); + if (offsets[rows] > blob.length) + throw new IllegalArgumentException( + "the last offset " + offsets[rows] + " of a string column is beyond its " + blob.length + " bytes"); + if (nulls != null && nulls.length != rows) + throw new IllegalArgumentException("a string column has " + nulls.length + " null flags for " + rows + " rows"); + this.blob = blob; + this.offsets = offsets; + this.nulls = nulls; + } + + public int rows() { + return rows; + } + + /** The String of row {@code i}, or null for a null row. */ + public String get(final int i) { + if (nulls != null && nulls[i]) + return null; + final int from = offsets[i]; + return new String(blob, from, offsets[i + 1] - from, StandardCharsets.UTF_8); + } +} diff --git a/tests/README.md b/tests/README.md index 6d2bc8a..f1681e4 100644 --- a/tests/README.md +++ b/tests/README.md @@ -39,6 +39,8 @@ The same suite runs against the native `arcadedb` wheel (`native/wheel/parity_wh - `NotOnNativeYet`: the native code raised it because the feature needs a Java object API the library does not expose. The report says `NOTYET`, and a final release allows none. A call that is JVM-only by design (`get_java_database`, `get_java_document`, a Java object a `Schema` method returns) raises it with a message that names the `arcadedb-jvm` build; a shared test that reaches one is `NA-JVM`, or reads the value through a statement or the Python API instead. - A row of `NATIVE_NA_JVM_TESTS` in `tests/conftest.py`, keyed `::[::]`: the test reaches into the JVM on purpose (`jpype.JClass`, a Java array, `db._java_db`, counted bridge crossings). The report says `NA-JVM`. Add a row only for such a test, and give the reason. When the behaviour is shared, write a native test beside it (`test_native_*.py`, collected on the native build only). +A test file whose process dies without a result is a `CRASH` row with its exit code. On Windows the parity harness runs such a file once more (`--retry-crash`, set by `test-native.yml` on Windows legs only) and marks a retry that produced a result `FLAKY-CRASH`; the job summary and issue 295 record it. A failing test is never retried, and a crash on Linux or macOS is a failure. A process that wrote its result and then died in shutdown is only recorded (`DIRTY-EXIT`). The decision is `native/wheel/parity_crash.py`, tested by `tests/test_parity_crash_retry.py`. + A test that fails on the native build for any other reason is a failure. Fix the native code, or mark a known engine bug with a strict `xfail`. Do not add a row to hide it. ## Need Help? diff --git a/tests/conftest.py b/tests/conftest.py index bd1a24d..4a862a0 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -37,10 +37,14 @@ "test_release_preflight.py": "runs the workflows' bash step code", # runs the bash code of the plan step of test-native.yml (scripts/dev_engine_pin.py, #304) "test_dev_engine_pin.py": "runs the bash code of a workflow step", + # runs the awk program of the parity gate step of test-native.yml, which is code for the bash of a runner + "test_parity_gate_awk.py": "runs the workflows' awk step code", } if sys.platform == "win32": collect_ignore.extend(WINDOWS_NOT_APPLICABLE) if BACKEND != "native": + # the old JSON-row route that the native column frames are compared with (the module lives in the native package) + collect_ignore.append("test_native_column_frames_reference.py") # Java -> Python upcalls (Python functions in SQL) exist on the native build only collect_ignore.append("test_native_udf.py") # how find_nearest hands the query vector to the native library (the codec's side buffer) @@ -49,6 +53,18 @@ collect_ignore.append("test_native_vector_hit_lazy.py") # the columnar insert's stored records are compared between the JSON and the columnar entry points of the native library collect_ignore.append("test_native_columnar_insert.py") + # insert_columns(parallel=True) takes the columnar entry and stores what the row path stores (arcadedb_n_insert_colp with "parallel") + collect_ignore.append("test_native_columnar_parallel.py") + # the review of the native parallel loads: no silent loss, an interrupt waits for the writers and reads the failure token + collect_ignore.append("test_native_columnar_parallel_review_fixes.py") + # a column of date-times, dates, Decimals or bytes crosses as a column (kinds T D n x), observed on the native library + collect_ignore.append("test_native_columnar_temporal.py") + # the review of the temporal kinds: a value the image cannot take is refused in Python, never wrapped or thrown mid-batch + collect_ignore.append("test_native_columnar_temporal_review_fixes.py") + # a vertex load by insert_dataframe or insert_arrow crosses as columns, one crossing per batch, observed on the native library + collect_ignore.append("test_native_vertex_columnar.py") + # a document or vertex load keeps its transient Python memory to one batch and a vector column is never turned into Python floats + collect_ignore.append("test_native_columnar_memory.py") # the rows of an insert_many chunk with date-times, Decimals, bytes and numpy scalars are told apart by the entry points they cross collect_ignore.append("test_native_mixed_chunk.py") # the native build's ingest defaults (insert_many and graph_batch commit size), observed on the native library @@ -67,6 +83,10 @@ collect_ignore.append("test_native_async_callbacks.py") # the short call path of a point read (codec.encode_params, lib.exec_query, first()), observed on the native library collect_ignore.append("test_native_point_read.py") + # how a reply is read out of the library (lib._take): the copies and the frees, observed with spies on ctypes.string_at and arcadedb_free + collect_ignore.append("test_native_reply_reader.py") + # the crossings of save() and new_edge() (lib.cmd, lib.call, lib.call_bin, lib.exec_query are the native library's ctypes doors) + collect_ignore.append("test_native_one_row_crossings.py") # an RID built from an RID keeps the text, so the fast named-parameter path of codec.encode_params can encode it (#265) collect_ignore.append("test_native_rid_params.py") # to_json_list walks the rows with codec.json_native only when a reply restored a tagged value (#219), observed on the native library @@ -85,8 +105,12 @@ collect_ignore.append("test_native_point_op_glue.py") # the address table of append_samples (arcadedb_n_async_bin), observed on the native library collect_ignore.append("test_native_timeseries_binary.py") - # arcadedb._native.columns builds the columnar readers' tables from the rows; the file compares it with the rules it replaced (#312) + # the drain of a native result set (_fill(into), _drain_dicts) against row-by-row iteration (#321; the columns module it once compared is gone, #331) collect_ignore.append("test_native_columns_reference.py") + # the crossings of GraphBatch.create_vertices with float32 columns (lib.call, the codec's per-row hooks), observed on the native library + collect_ignore.append("test_native_vertex_vector_crossings.py") + # the native columnar fast path (arcadedb._native.bulk.insert_columns_fast) with a rows-dropped fault in it, found by the ingest-integrity check (#336) + collect_ignore.append("test_native_ingest_integrity_faults.py") if not os.path.isfile( os.path.join(os.path.dirname(os.path.dirname(__file__)), "docs", "index.md") ): @@ -114,6 +138,7 @@ "test_server_packaging.py": "JVM wheel contents", "test_runtime_cache.py": "JRE runtime cache of the JVM wheel", "test_aot.py": "JVM class-data cache", + "test_jvm_cds.py": "class-data-sharing archive of the JVM wheel's bundled JRE", "test_stage_jars.py": "staging of the JVM wheel's jars", "test_extract_version.py": "version of the JVM wheel's engine jar", "test_extract_version_classifiers.py": "version of the JVM wheel's engine jar", @@ -123,9 +148,14 @@ "test_query_vector_params_jvm.py": "the java.util objects the JVM build's parameter conversion builds", "test_insert_many_gc_pause.py": "the cyclic collector pause is the JVM build's (the native insert_many allocates nothing the collector tracks)", "test_graph_batch_tagged_shapes.py": "the JVM wheel's graph batch bulk path (bridge VertexBatcher and EdgeBatcher, the property-matrix fallback)", + "test_graph_batch_single_pass.py": "the JVM wheel's graph batch bulk path (GraphBatch._create_vertices_json_bulk, the property-matrix fallback)", + "test_graph_batch_vector_crossings_jvm.py": "the JPype crossings of the JVM wheel's create_vertices vector path (the bridge classes, the property-matrix fallback)", + "test_insert_columns_string_blob.py": "the bridge's StringColumn and the JVM wheel's core._column_to_java (JPype arrays, a spy on the per-element conversion)", + "test_arrow_vector_column_jvm.py": "the JPype crossings of the JVM wheel's Arrow vector column (core.convert_python_to_java, the bridge's FloatVectors.Matrix) and its Python heap", "test_async_proxy_cache.py": "the JPype proxy classes of the JVM build's async executor", "test_read_crossings.py": "crossing counts of the JVM build's read path (JPype proxies of the Java objects and the bridge classes)", "test_reader_crossings.py": "crossing counts of the JVM build's record and row readers (JPype proxies of the Java objects and the bridge classes)", + "test_jvm_ingest_integrity_faults.py": "rows-dropped faults put into the JVM build's column conversion and bulk vertex path (core._column_to_java, GraphBatch._create_vertices_json_bulk)", } if BACKEND == "native" and not os.environ.get("NATIVE_PARITY_ALL"): collect_ignore.extend(NATIVE_NOT_APPLICABLE) @@ -297,15 +327,35 @@ def pytest_configure(config): # pytest_configure, and a conftest hook otherwise runs before that. # This hook was dead from 2026-07-25 to 2026-09-29: a second # pytest_configure further down replaced it (test_jvm_args pins it now). + # + # That reason is the JVM's. The native wheel has no HotSpot in the process, so a + # fault there is a real one and the trace is what a crash leaves: with a stock run + # (faulthandler off, nothing in the log) the Windows parity legs lost a process in + # test_vector.py, test_free_threading.py and test_find_nearest_query_forms.py and + # nothing told which call it was (ArcadeData/arcadedb-python#295). native/wheel/parity_wheel.py names a + # file in ARCADEDB_TEST_FAULTHANDLER_FILE (it sets it on Windows only), and a native + # run on Windows then writes faulthandler's trace there, not to stderr. The JVM + # build ignores the variable and stays off. import sys if sys.platform == "win32": import faulthandler - if faulthandler.is_enabled(): + trace = os.environ.get(FAULT_FILE_ENV) + if trace and BACKEND == "native": + global _fault_file + # kept open: faulthandler writes to its descriptor at the crash + _fault_file = open(trace, "w") + faulthandler.enable(file=_fault_file, all_threads=True) + elif faulthandler.is_enabled(): faulthandler.disable() +# The file the native run on Windows writes a faulthandler trace to (see pytest_configure above); set by native/wheel/parity_wheel.py. +FAULT_FILE_ENV = "ARCADEDB_TEST_FAULTHANDLER_FILE" +_fault_file = None + + # A test still running this long gets every Java thread's stack on stderr, # shortly before faulthandler_timeout (600 s in pyproject.toml) dumps the # Python threads. The Python dump alone shows a test waiting inside a Java @@ -529,3 +579,30 @@ def pytest_sessionfinish(session, exitstatus): def pytest_terminal_summary(terminalreporter, exitstatus, config): """Terminal summary hook.""" pass + + +@pytest.fixture +def store(temp_db_path): + """A database on disk that a test can close and open again (the ingest-integrity tests, ArcadeData/arcadedb-python#336).""" + from tests.ingest_integrity import Store + + box = Store(temp_db_path) + yield box + box.close() + + +@pytest.fixture +def small_chunks(monkeypatch): + """insert_many reads 7 rows at a time and a graph batch crosses 7 rows at a time (the real chunks are 10,000 and 100,000).""" + from tests.ingest_integrity import CHUNK + + if BACKEND == "native": + from arcadedb._native import database, extras + + monkeypatch.setattr(database, "_INSERT_MANY_CHUNK", CHUNK) + monkeypatch.setattr(extras.NativeGraphBatch, "_BULK_CHUNK", CHUNK) + else: + from arcadedb import core + + monkeypatch.setattr(core, "_INSERT_MANY_CHUNK", CHUNK) + monkeypatch.setattr(arcadedb.GraphBatch, "_BULK_CHUNK", CHUNK) diff --git a/tests/ingest_integrity.py b/tests/ingest_integrity.py new file mode 100644 index 0000000..d41d072 --- /dev/null +++ b/tests/ingest_integrity.py @@ -0,0 +1,220 @@ +"""The one check that every ingest-integrity test calls (ArcadeData/arcadedb-python#336). + +An ingest call must not lose, duplicate or silently alter a row. After a call, ``problems_for_type`` checks three things on the +stored type, each read back through the public API: + +1. the record count, read three ways (``SELECT count(*)``, ``Database.count_type`` and the length of the rows that come back), +2. an order-independent digest of every column: the sum and the xor of a stable 64-bit hash of each row, compared with the digest + of the rows that were sent (two row readers, ``to_list()`` and iterating the result set, must each match it), +3. for a key column, every sent key present exactly once and no other key. + +``Store.reopen`` closes the database right after the call and opens it again from disk, and the same three checks run once more +(durability). A check never looks at what the path under test says it did: the expected side is the data the test generated. + +``MUTANTS`` are the faults the mutation tests inject into the rows a path is given (drop the last row, skip a middle chunk, store +one row twice, change one value, swap two values). A test family is proven able to fail when the check reports each of them. +""" + +import collections +import hashlib + +import arcadedb + +MASK = (1 << 64) - 1 + +CHUNK = 7 # rows per crossing, made small so every boundary is cheap +COMMIT = 5 # commit_every and batch_size + +#: every problem line carries this word, so a test can tell "the check found it" from "the call raised" +FOUND = "MISMATCH" + + +class IntegrityError(AssertionError): + """The stored records are not the sent records.""" + + +def canon(kind, value): + """The canonical text of one cell. ``kind`` is how the property is declared, so 5 and 5.0 are the same DOUBLE.""" + if value is None: + return "N" + if kind == "int": + return "i%d" % int(value) + if kind == "float": + number = float(value) + return "nan" if number != number else "f" + number.hex() + if kind == "bool": + return "b1" if value else "b0" + if kind == "str": + return "s" + str(value) + raise ValueError(f"no canonical form for kind {kind!r}") + + +def row_hash(schema, row): + """A stable 64-bit hash of one row: the column names sorted, each cell length-prefixed, a missing cell the same as a null.""" + parts = [] + for name in sorted(schema): + text = canon(schema[name], row.get(name)) + parts.append("%s:%d:%s" % (name, len(text), text)) + data = "|".join(parts).encode("utf-8", "surrogatepass") + return int.from_bytes(hashlib.blake2b(data, digest_size=8).digest(), "big") + + +def digest(schema, rows): + """``(row count, sum of the row hashes mod 2**64, xor of the row hashes)``: the same for any order of the rows.""" + count = total = mixed = 0 + for row in rows: + h = row_hash(schema, row) + count += 1 + total = (total + h) & MASK + mixed ^= h + return count, total, mixed + + +def _key_problems(label, view, schema, key, want_rows, rows): + kind = schema[key] + wanted = collections.Counter(canon(kind, r.get(key)) for r in want_rows) + stored = collections.Counter(canon(kind, r.get(key)) for r in rows) + missing = sorted(set(wanted) - set(stored)) + extra = sorted(set(stored) - set(wanted)) + twice = sorted(k for k, c in stored.items() if c > 1 and wanted.get(k, 0) < c) + problems = [] + if missing: + problems.append( + f"{FOUND} {label}: {len(missing)} sent key(s) not stored, first {missing[:3]} ({view})" + ) + if extra: + problems.append( + f"{FOUND} {label}: {len(extra)} stored key(s) were never sent, first {extra[:3]} ({view})" + ) + if twice: + problems.append( + f"{FOUND} {label}: {len(twice)} key(s) stored more than once, first {twice[:3]} ({view})" + ) + return problems + + +def verify(label, want_rows, schema, key, counts, readers): + """The problems (a list of text lines, empty when none) between the sent rows and what ``counts`` and ``readers`` saw.""" + want = digest(schema, want_rows) + problems = [] + for name, value in counts.items(): + if value != want[0]: + problems.append( + f"{FOUND} {label}: {name} says {value} record(s), {want[0]} were sent" + ) + first = True + for name, rows in readers.items(): + got = digest(schema, rows) + if got[0] != want[0]: + problems.append( + f"{FOUND} {label}: reading with {name} returned {got[0]} row(s), {want[0]} were sent" + ) + elif got != want: + problems.append( + f"{FOUND} {label}: reading with {name} returned {got[0]} row(s) whose digest differs from the sent data" + f" (a stored value changed)" + ) + if key is not None and first: + problems += _key_problems(label, name, schema, key, want_rows, rows) + first = False + return problems + + +def type_view(db, name, schema): + """What the public API reports for one document, vertex or time-series type: the counts and the rows two ways.""" + names = list(schema) + select = "SELECT FROM " + name # nosec B608 + count_sql = "SELECT count(*) AS n FROM " + name # nosec B608 + counts = { + "SELECT count(*)": int(db.query("sql", count_sql).to_list()[0]["n"]), + "count_type()": int(db.count_type(name)), + } + listed = db.query("sql", select).to_list() + iterated = [{n: r.get(n) for n in names} for r in db.query("sql", select)] + return counts, {"to_list()": listed, "iterating the result": iterated} + + +def problems_for_type(db, name, want_rows, schema, key, label): + counts, readers = type_view(db, name, schema) + return verify(label, want_rows, schema, key, counts, readers) + + +EDGE_SCHEMA = {"a": "int", "b": "int", "w": "float"} + + +def edge_view(db, edge_type): + """The counts of an edge type, and its edges as (source key, destination key, weight) rows.""" + count_sql = "SELECT count(*) AS n FROM " + edge_type # nosec B608 + cypher_count = "MATCH ()-[r:" + edge_type + "]->() RETURN count(r) AS c" + pairs = "MATCH (a)-[r:" + edge_type + "]->(b) RETURN a.k AS a, b.k AS b, r.w AS w" + counts = { + "SELECT count(*)": int(db.query("sql", count_sql).to_list()[0]["n"]), + "Cypher count(r)": int(db.query("opencypher", cypher_count).to_list()[0]["c"]), + } + return counts, {"Cypher MATCH": db.query("opencypher", pairs).to_list()} + + +def problems_for_edges(db, edge_type, want_rows, label): + counts, readers = edge_view(db, edge_type) + return verify(label, want_rows, EDGE_SCHEMA, None, counts, readers) + + +class Store: + """A database on disk that a test can close and open again.""" + + def __init__(self, path): + self.path = path + self.db = arcadedb.create_database(path) + + def reopen(self): + """Close straight after the last call and open the same files again.""" + self.db.close() + self.db = arcadedb.open_database(self.path) + + def close(self): + db, self.db = self.db, None + if db is not None and db.is_open(): + db.close() + + +def _mutant_drop_last(rows): + return rows[:-1] + + +def _mutant_drop_first(rows): + return rows[1:] + + +def _mutant_skip_a_chunk(rows): + third = max(1, len(rows) // 3) + return rows[:third] + rows[2 * third :] + + +def _mutant_duplicate_one(rows): + return rows + [dict(rows[len(rows) // 2])] + + +def _mutant_alter_one_value(rows): + out = [dict(r) for r in rows] + out[len(out) // 2]["val"] = out[len(out) // 2]["val"] + 0.5 + return out + + +def _mutant_swap_two_values(rows): + out = [dict(r) for r in rows] + a, b = 1, len(out) - 2 + out[a]["val"], out[b]["val"] = out[b]["val"], out[a]["val"] + return out + + +MUTANTS = { + "drop_the_last_row": _mutant_drop_last, + "drop_the_first_row": _mutant_drop_first, + "skip_a_chunk": _mutant_skip_a_chunk, + "store_one_row_twice": _mutant_duplicate_one, + "change_one_value": _mutant_alter_one_value, + "swap_two_values": _mutant_swap_two_values, +} + +#: the durability mutant: the rows go into a transaction nobody commits, so they are visible in this session and gone after a reopen +NO_COMMIT = "leave_the_transaction_open" diff --git a/tests/test_arrow_vector_column_jvm.py b/tests/test_arrow_vector_column_jvm.py new file mode 100644 index 0000000..fee897a --- /dev/null +++ b/tests/test_arrow_vector_column_jvm.py @@ -0,0 +1,428 @@ +"""insert_arrow and insert_dataframe with a vector column on the JVM wheel: the column crosses as one float32 matrix, not as one Python list of +Python floats per row, and what is stored is unchanged. + +A ``FixedSizeList`` (or list, or large list) column into an ``ARRAY_OF_FLOATS`` property went ``chunk.to_pylist()`` (one Python float +per element, 31.5 MiB of Python heap for one 10,000 x 96 batch against a 3.7 MiB flat buffer), then ``_column_to_java`` converted every row +with ``convert_python_to_java`` (an ``ArrayList`` of 96 boxed doubles per row, 100,000 conversions for the 10 x 10,000 rows of the +study) and the engine narrowed each list back to a ``float[]``. In the untimed profile that conversion was about 63 percent of the call, +and the boxed doubles are what the JVM heap held (``heap used`` 183 to 195 MiB at the end against 69 MiB live after a collection). Now a +float32 column with one length and no null stays an Arrow buffer, ``_float32_matrix`` lays it out as one 2-D float32 numpy matrix, and +``_column_to_java`` sends it as one ``float[]`` inside the bridge's ``FloatVectors.Matrix`` (the route a 2-D numpy column of +``insert_columns`` already took); the documents get a row each Java-side. + +Three tests fail on the code before (they count the per-row conversions, the matrix crossings and look at what the conversion hands over: one float32 view, not per-row lists); the others +pin what the old route stored, so the new one must store the same: count, an order-independent digest of every row's vector bytes read back +through SQL and through the result API, both again after close and reopen, the special float32 values, a null row, a null element, vectors +of different length, a wrong dimension against a vector index, float64 and integer lists (which are NOT taken: the old route refuses a +value past the float range and rounds the rest, a float32 cast would store an infinity), and the properties that are not declared +``ARRAY_OF_FLOATS`` (they keep a list). The JVM build only: it counts JPype crossings and reads Java-side values, so conftest.py does not +collect this file on the native wheel (whose vector columns stay Arrow arrays, test_native_columnar_memory.py). +""" + +import hashlib + +import numpy as np +import pyarrow as pa +import pytest + +import arcadedb +from arcadedb import core, dataframe +from arcadedb.exceptions import ArcadeDBError + +F32 = np.float32 +SHAPES = ["fixed_size", "list", "large_list"] +TYPES = ["VERTEX", "DOCUMENT"] + + +def _matrix(n, dim, start=0, specials=True): + """float32 vectors, a pure function of the row number, with the values a decimal text of 6 digits would not hold in rows 0 and 1.""" + r = np.arange(start, start + n, dtype=np.int64) + j = np.arange(dim, dtype=np.int64) + m = (((r[:, None] * 257 + j[None, :] * 17) % 1000).astype(F32)) / F32(1000.0) + if specials and start == 0 and n > 1 and dim > 3: + m[0, 0] = F32(-0.0) + m[1, 0] = np.finfo(F32).max + m[1, 1] = np.finfo(F32).tiny + m[1, 2] = np.finfo(F32).smallest_subnormal + return m + + +def _column(matrix, shape): + dim = matrix.shape[1] + flat = pa.array(matrix.reshape(-1), type=pa.float32()) + if shape == "fixed_size": + return pa.FixedSizeListArray.from_arrays(flat, dim) + kind = pa.list_ if shape == "list" else pa.large_list + rows = [row.tolist() for row in matrix] + return pa.array(rows, type=kind(pa.float32())) + + +def _table(matrix, shape, start=0): + n = matrix.shape[0] + return pa.table( + { + "k": pa.array(np.arange(start, start + n), type=pa.int32()), + "embedding": _column(matrix, shape), + } + ) + + +def _make(db, name, kind="VERTEX", embedding="ARRAY_OF_FLOATS", index=None): + db.command("sql", f"CREATE {kind} TYPE {name}") + db.command("sql", f"CREATE PROPERTY {name}.k INTEGER") + if embedding: + db.command("sql", f"CREATE PROPERTY {name}.embedding {embedding}") + if index: + db.command( + "sql", + f"CREATE INDEX ON {name} (embedding) LSM_VECTOR METADATA {{dimensions: {index}, similarity: 'COSINE'}}", + ) + + +def _count(db, name): + sql = f"SELECT count(*) AS c FROM {name}" # nosec B608 - test-owned name + return db.query("sql", sql).first().get("c") + + +def _key(k, vector_bytes): + return int.from_bytes( + hashlib.sha256(int(k).to_bytes(8, "little") + vector_bytes).digest()[:16], + "little", + ) + + +def _expected_digest(matrix, start=0): + """The digest the stored table must have, from the generator, never from the database.""" + h = 0 + for i, row in enumerate(matrix): + h ^= _key(start + i, row.tobytes()) + return h + + +def _sql_digest(db, name): + h, n = 0, 0 + sql = f"SELECT k, embedding FROM {name}" # nosec B608 - test-owned name + for row in db.query("sql", sql).to_list(): + h ^= _key( + row["k"], + np.asarray(row["embedding"], dtype=np.float64).astype(F32).tobytes(), + ) + n += 1 + return n, h + + +def _api_digest(db, name): + """The same digest through the result API (one Result per record, ``get`` per property), not through ``to_list``.""" + h, n = 0, 0 + result_set = db.query("sql", f"SELECT FROM {name}") # nosec B608 - test-owned name + for result in result_set: + h ^= _key( + result.get("k"), + np.asarray(result.get("embedding"), dtype=np.float64).astype(F32).tobytes(), + ) + n += 1 + return n, h + + +def _bits(values): + return np.asarray(values, dtype=np.float64).astype(F32).view(np.uint32).tolist() + + +@pytest.fixture +def crossings(monkeypatch): + """What one load does on the JPype side: the per-value conversions and the bridge's Matrix constructions.""" + log = {"convert": [], "matrix": []} + real_convert = core.convert_python_to_java + real_class = core._java_class + + def convert(value): + log["convert"].append(type(value).__name__) + return real_convert(value) + + class _Counting: + def __init__(self, target): + self._target = target + + def __call__(self, *args): + log["matrix"].append(len(args[0])) + return self._target(*args) + + def __getattr__(self, name): + return getattr(self._target, name) + + def java_class(name): + cls = real_class(name) + return ( + _Counting(cls) if name == "com.arcadedb.python.FloatVectors$Matrix" else cls + ) + + monkeypatch.setattr(core, "convert_python_to_java", convert) + monkeypatch.setattr(core, "_java_class", java_class) + return log + + +@pytest.mark.parametrize("kind", TYPES) +@pytest.mark.parametrize("shape", SHAPES) +def test_count_digest_and_reopen_of_a_batched_vector_load( + temp_db, temp_db_path_of, shape, kind +): + """1,300 rows in batches of 500 (500, 500, 300: the batches are slices of one table): count, SQL digest, API digest, then the same after reopen.""" + n, dim = 1_300, 24 + matrix = _matrix(n, dim) + _make(temp_db, "Vec", kind) + assert temp_db.insert_arrow("Vec", _table(matrix, shape), batch_size=500) == n + want = (n, _expected_digest(matrix)) + assert _count(temp_db, "Vec") == n + assert _sql_digest(temp_db, "Vec") == want + assert _api_digest(temp_db, "Vec") == want + path = temp_db_path_of(temp_db) + temp_db.close() + reopened = arcadedb.open_database(path) + try: + assert _count(reopened, "Vec") == n + assert _sql_digest(reopened, "Vec") == want + assert _api_digest(reopened, "Vec") == want + finally: + reopened.close() + + +@pytest.fixture +def temp_db_path_of(): + def path_of(db): + return db.get_java_database().getDatabasePath() + + return path_of + + +def test_a_load_in_several_chunks_of_one_table_keeps_every_row(temp_db): + """Two tables concatenated (a ChunkedArray of two chunks) cut into batches of 4 (offsets that start inside a chunk).""" + a, b = _matrix(3, 6, 0), _matrix(4, 6, 3) + _make(temp_db, "Vec") + table = pa.concat_tables([_table(a, "fixed_size", 0), _table(b, "fixed_size", 3)]) + assert temp_db.insert_arrow("Vec", table, batch_size=4) == 7 + assert _sql_digest(temp_db, "Vec") == ( + 7, + _expected_digest(a, 0) ^ _expected_digest(b, 3), + ) + + +@pytest.mark.parametrize("shape", SHAPES) +def test_the_special_float32_values_are_stored_bit_for_bit(temp_db, shape): + special = np.array( + [ + [ + -0.0, + np.finfo(F32).max, + np.finfo(F32).tiny, + np.finfo(F32).smallest_subnormal, + np.nan, + np.inf, + -np.inf, + 16777217.0, + ] + ], + dtype=F32, + ) + _make(temp_db, "Vec") + assert temp_db.insert_arrow("Vec", _table(special, shape)) == 1 + stored = temp_db.query("sql", "SELECT embedding FROM Vec").first().get("embedding") + assert len(stored) == 8 + got = np.asarray(stored, dtype=np.float64).astype(F32) + assert np.isnan(got[4]) + keep = [0, 1, 2, 3, 5, 6, 7] + assert ( + got.view(np.uint32)[keep].tolist() == special.view(np.uint32)[0, keep].tolist() + ) + assert got[7] == F32( + 16777216.0 + ) # 16777217.0 is not a float32; the row holds the float32 that was written + + +def test_a_null_row_stays_null_and_its_neighbours_are_exact(temp_db): + _make(temp_db, "Vec") + table = pa.table( + { + "k": pa.array([0, 1, 2], pa.int32()), + "embedding": pa.array( + [[1.0, 2.0], None, [3.0, 4.0]], pa.list_(pa.float32(), 2) + ), + } + ) + assert temp_db.insert_arrow("Vec", table) == 3 + rows = { + r["k"]: r["embedding"] + for r in temp_db.query("sql", "SELECT k, embedding FROM Vec").to_list() + } + assert rows[1] is None + assert _bits(rows[0]) == _bits([1.0, 2.0]) and _bits(rows[2]) == _bits([3.0, 4.0]) + + +def test_a_null_element_is_refused_and_the_batch_is_rolled_back(temp_db): + _make(temp_db, "Vec") + table = pa.table( + { + "k": pa.array([0, 1], pa.int32()), + "embedding": pa.array( + [[1.0, None, 3.0], [4.0, 5.0, 6.0]], pa.list_(pa.float32()) + ), + } + ) + with pytest.raises(ArcadeDBError, match="null element"): + temp_db.insert_arrow("Vec", table) + assert _count(temp_db, "Vec") == 0 + + +def test_vectors_of_different_length_are_stored_as_given_without_an_index_and_refused_with_one( + temp_db, +): + _make(temp_db, "Free") + ragged = pa.table( + { + "k": pa.array([0, 1, 2], pa.int32()), + "embedding": pa.array( + [[1.0, 2.0], [1.0, 2.0, 3.0], [4.0]], pa.list_(pa.float32()) + ), + } + ) + assert temp_db.insert_arrow("Free", ragged) == 3 + rows = { + r["k"]: r["embedding"] + for r in temp_db.query("sql", "SELECT k, embedding FROM Free").to_list() + } + assert {k: len(v) for k, v in rows.items()} == {0: 2, 1: 3, 2: 1} + _make(temp_db, "Indexed", index=2) + with pytest.raises( + ArcadeDBError, + match="Vector dimension does not match index dimension 2: got float.. of length 3", + ): + temp_db.insert_arrow("Indexed", ragged.slice(0, 2)) + assert _count(temp_db, "Indexed") == 0 + + +def test_a_fixed_size_vector_of_the_wrong_dimension_is_refused_by_the_vector_index( + temp_db, +): + _make(temp_db, "Indexed", index=2) + wrong = _table(_matrix(2, 3, specials=False), "fixed_size") + with pytest.raises( + ArcadeDBError, + match="Vector dimension does not match index dimension 2: got float.. of length 3", + ): + temp_db.insert_arrow("Indexed", wrong) + assert _count(temp_db, "Indexed") == 0 + right = _matrix(2, 2, specials=False) + assert temp_db.insert_arrow("Indexed", _table(right, "fixed_size")) == 2 + assert _sql_digest(temp_db, "Indexed") == (2, _expected_digest(right)) + + +def test_float64_and_integer_lists_keep_what_the_old_route_stored(temp_db): + """Not taken by the matrix route: a float64 value past the float range is refused (a cast would store an infinity), the rest rounds.""" + _make(temp_db, "Dbl") + ok = pa.table( + { + "k": pa.array([0], pa.int32()), + "embedding": pa.array([[0.1, 16777217.0, -2.5]], pa.list_(pa.float64())), + } + ) + assert temp_db.insert_arrow("Dbl", ok) == 1 + got = temp_db.query("sql", "SELECT embedding FROM Dbl").first().get("embedding") + assert _bits(got) == _bits([0.1, 16777217.0, -2.5]) + assert got[1] == 16777216.0 + _make(temp_db, "Over") + over = pa.table( + { + "k": pa.array([0], pa.int32()), + "embedding": pa.array([[1e39, 1.0]], pa.list_(pa.float64())), + } + ) + with pytest.raises(ArcadeDBError, match="1.0E39"): + temp_db.insert_arrow("Over", over) + assert _count(temp_db, "Over") == 0 + _make(temp_db, "Ints") + ints = pa.table( + { + "k": pa.array([0], pa.int32()), + "embedding": pa.array([[16777217, 2**40, -3]], pa.list_(pa.int64())), + } + ) + assert temp_db.insert_arrow("Ints", ints) == 1 + assert temp_db.query("sql", "SELECT embedding FROM Ints").first().get( + "embedding" + ) == [16777216.0, 1099511627776.0, -3.0] + + +@pytest.mark.parametrize( + "embedding", [None, "LIST"], ids=["undeclared", "declared_list"] +) +def test_a_property_that_is_not_array_of_floats_keeps_a_list_and_takes_no_matrix( + temp_db, crossings, embedding +): + _make(temp_db, "Free", embedding=embedding) + matrix = _matrix(5, 4, specials=False) + assert temp_db.insert_arrow("Free", _table(matrix, "fixed_size")) == 5 + assert crossings["matrix"] == [] + assert _sql_digest(temp_db, "Free") == (5, _expected_digest(matrix)) + + +def test_empty_vectors_are_stored_as_empty_lists(temp_db): + _make(temp_db, "Vec") + table = pa.table( + { + "k": pa.array([0, 1], pa.int32()), + "embedding": pa.array([[], []], pa.list_(pa.float32())), + } + ) + assert temp_db.insert_arrow("Vec", table) == 2 + assert [ + r["embedding"] + for r in temp_db.query("sql", "SELECT embedding FROM Vec ORDER BY k").to_list() + ] == [[], []] + + +# ---- the three tests that fail on the code before the fix: they count what the load does, never how long it takes + + +@pytest.mark.parametrize("shape", SHAPES) +def test_a_vector_column_converts_no_python_value_per_row(temp_db, crossings, shape): + """Before: one ``convert_python_to_java`` per row (an ArrayList of boxed doubles). After: the vector column makes none.""" + n = 600 + _make(temp_db, "Vec") + assert temp_db.insert_arrow("Vec", _table(_matrix(n, 8), shape), batch_size=n) == n + assert crossings["convert"] == [] + + +def test_a_vector_column_crosses_as_one_matrix_per_batch(temp_db, crossings): + """1,300 rows in batches of 500: three batches, so three ``FloatVectors.Matrix`` of one flat float[] each (500 x 24, 500 x 24, 300 x 24).""" + _make(temp_db, "Vec") + assert ( + temp_db.insert_arrow( + "Vec", _table(_matrix(1_300, 24), "fixed_size"), batch_size=500 + ) + == 1_300 + ) + assert crossings["matrix"] == [500 * 24, 500 * 24, 300 * 24] + + +def test_the_vector_column_reaches_the_bridge_as_one_float32_view(temp_db, monkeypatch): + """10,000 x 96 float32 is a 3.66 MiB buffer. ``_column_values`` must hand it over as ONE 2-D float32 numpy array that is a view of Arrow's + value buffer (nothing copied, no Python float per element); the per-row lists of floats it returned before were 31.5 MiB of Python heap at + their peak. The memory shape is asserted on what the conversion returns, not with tracemalloc: starting and stopping tracemalloc around a + JPype float[] conversion crashed the interpreter now and then (SIGSEGV in a JVM thread) on Python 3.11 in CI, on Linux and Windows. + """ + seen = {} + real = dataframe._column_values + + def spy(name, chunk, *args, **kwargs): + values = real(name, chunk, *args, **kwargs) + seen[name] = values + return values + + monkeypatch.setattr(dataframe, "_column_values", spy) + n, dim = 10_000, 96 + _make(temp_db, "Vec") + table = _table(_matrix(n, dim, specials=False), "fixed_size") + assert temp_db.insert_arrow("Vec", table, batch_size=n) == n + column = seen["embedding"] + assert isinstance(column, np.ndarray) + assert (column.shape, column.dtype) == ((n, dim), F32) + assert column.base is not None and not column.flags.owndata diff --git a/tests/test_bulk_json_paths.py b/tests/test_bulk_json_paths.py index 8bb7bb0..1c801d0 100644 --- a/tests/test_bulk_json_paths.py +++ b/tests/test_bulk_json_paths.py @@ -104,7 +104,7 @@ def test_a_lone_surrogate_raises_instead_of_becoming_a_question_mark( _write(db, entry, "a\ud800b") -def test_ordinary_rows_stay_on_the_bulk_path(temp_db_path): +def test_ordinary_rows_stay_on_the_bulk_path(): rows = [ {"id": i, "name": f"n{i}", "score": i * 0.5, "flag": i % 2 == 0, "note": None} for i in range(5) @@ -113,14 +113,7 @@ def test_ordinary_rows_stay_on_the_bulk_path(temp_db_path): assert all( type_conversion.json_bulk_scalar_ok(v) for row in rows for v in row.values() ) - with arcadedb.create_database(temp_db_path) as db: - _create_types(db) - with db.graph_batch(parallel_flush=False) as batch: - assert len(batch._create_vertices_json_bulk("V", rows)) == len(rows) - assert batch._create_vertices_json_bulk("V", rows + [{"id": 2**70}]) is None - assert ( - batch._create_vertices_json_bulk("V", rows + [{"x": math.nan}]) is None - ) + # the graph batch's side of this (the JVM wheel's _create_vertices_json_bulk) is in test_graph_batch_single_pass.py @pytest.mark.parametrize( diff --git a/tests/test_bulk_single_pass_validation.py b/tests/test_bulk_single_pass_validation.py new file mode 100644 index 0000000..3d12c8b --- /dev/null +++ b/tests/test_bulk_single_pass_validation.py @@ -0,0 +1,605 @@ +"""insert_many and create_vertices validate their rows in the one json.dumps pass, not in a Python loop over the rows before it. + +`json_bulk_dumps` and `json_bulk_dumps_tagged` used to walk every value of every row before json.dumps walked it again. Now json.dumps +writes the rows once and the check comes from what that pass shows (the text and the types of the rows), and a walk runs only when it +may find something. The contract is unchanged: the same text for rows that carry, and for a row that does not, the same exception type +and message as the old loop gave for the first bad row in row order. The old loop is copied here as the private reference, so these +tests fail on the old code (they spy on `_check_rows`, which only the new code has) and compare the new code with that reference. +The old messages name no row index, so the first bad row of a large batch is told from a later one by the value in its message. +""" + +import datetime as dt +import decimal +import enum +import json +import math +import random + +import numpy as np +import pytest + +import arcadedb +from arcadedb import type_conversion as tc + +INT64_MAX = 2**63 - 1 +INT64_MIN = -(2**63) + + +def _old_dumps(rows, tagged): + """The json_bulk_dumps and json_bulk_dumps_tagged of 26.11.1: the validation loop, then json.dumps.""" + for row in rows: + for key in row: + if type(key) is not str: + raise ValueError("a non-str key cannot go through the JSON bulk path") + for value in row.values(): + kind = type(value) + if kind is str: + if not value.isascii() and not tc._str_survives_json(value): + raise ValueError( + "a string with a lone surrogate cannot go through the JSON bulk path" + ) + elif kind is int: + if not INT64_MIN <= value <= INT64_MAX: + raise ValueError( + f"an integer beyond 64 bits cannot go through the JSON bulk path: {value}" + ) + elif kind is float or kind is bool or value is None: + continue + elif tagged and kind in tc._TAGGED_KINDS: + continue + elif tagged and kind is dict and tc._TAG in value: + raise ValueError( + "a map keyed by the tag marker cannot go through the JSON bulk path" + ) + elif tagged and ( + isinstance( + value, (dt.datetime, dt.date, decimal.Decimal, bytes, bytearray) + ) + or isinstance(value, np.generic) + or tc._is_pandas_null(kind) + ): + continue + else: + tc._json_bulk_check(value) + if tagged: + return tc._dumps_checked(rows, default=tc.bulk_tag_default) + return tc._dumps_checked(rows) + + +def _outcome(call, rows): + try: + return ("text", call(rows)) + except ( + BaseException + ) as e: # noqa: BLE001 - the type and the message are what is compared + return (type(e).__name__, str(e)) + + +class Colour(enum.IntEnum): + RED = 5 + + +class Name(str): + pass + + +def _scalar_rows(): + return [ + {"id": 1, "name": "alpha", "score": 0.5, "flag": True, "note": None}, + {"id": 2, "name": "", "score": -1e-30, "flag": False, "note": "x"}, + ] + + +CASES = { + "scalars": _scalar_rows(), + "empty": [], + "empty_rows": [{}, {}], + "bounds_fit": [{"a": INT64_MAX, "b": INT64_MIN, "c": 0}], + "above_max": [{"a": INT64_MAX + 1}], + "below_min": [{"a": INT64_MIN - 1}], + "big_negative": [{"a": -(10**30)}], + "huge_digits": [{"a": 10**5000}], + "non_bmp": [{"a": "emoji \U0001f600 pair", "b": "\U00010000"}], + "cjk_and_hangul": [{"a": "中文한글퟿"}], + "nul_and_controls": [{"a": "a\x00b\x1f", "b": "
"}], + "lone_high": [{"a": "a\ud800b"}], + "lone_low": [{"a": "\udc00"}], + "two_lone_halves": [{"a": "𐀀"}], + "literal_backslash_u": [{"a": "\\ud800", "b": "\\u0041"}], + "digits_in_text": [{"a": "9999999999999999999", "b": "1234567890123456789012"}], + "nan": [{"a": math.nan}], + "inf": [{"a": -math.inf}], + "subnormal": [{"a": 5e-324}], + "small_floats": [{"a": 0.00012345678901234568, "b": 1.2345678901234567e19}], + "int_key": [{1: "a"}], + "bool_key": [{True: "a"}], + "none_key": [{None: "a"}], + "list_ok": [{"a": [1, "x", None, [2.5]]}], + "list_big": [{"a": [1, 2**70]}], + "list_nan": [{"a": [1.0, math.nan]}], + "list_lone": [{"a": ["\ud800"]}], + "list_datetime": [{"a": [dt.datetime(2026, 1, 2)]}], + "dict_ok": [{"a": {"b": [1], "c": "d"}}], + "dict_int_key": [{"a": {1: 2}}], + "dict_tag_key": [{"a": {tc._TAG: 1}}], + "tuple_value": [{"a": (1, 2)}], + "datetime": [{"a": dt.datetime(2026, 1, 2, 3, 4, 5, 6)}], + "aware_datetime": [ + {"a": dt.datetime(2026, 1, 2, tzinfo=dt.timezone(dt.timedelta(hours=9)))} + ], + "date": [{"a": dt.date(2026, 1, 2)}], + "decimal": [{"a": decimal.Decimal("1.50")}], + "decimal_nan": [{"a": decimal.Decimal("NaN")}], + "bytes": [{"a": b"\x00\xffab", "b": bytearray(b"x")}], + "numpy_scalars": [ + { + "a": np.int64(5), + "b": np.float64(1.5), + "c": np.bool_(True), + "d": np.float32(2.5), + } + ], + "numpy_big": [{"a": np.uint64(2**64 - 1)}], + "numpy_nan": [{"a": np.float64(math.nan)}], + "numpy_datetime": [{"a": np.datetime64("2026-01-02T03:04:05")}], + "numpy_str": [{"a": np.str_("q")}], + "int_enum": [{"a": Colour.RED}], + "str_subclass": [{"a": Name("s")}], + "str_subclass_lone": [{"a": Name("\ud800")}], + "set_value": [{"a": {1, 2}}], + "object_value": [{"a": object()}], + "none_row": [None], + "list_row": [[1, 2]], + "string_row": ["ab"], + "mixed_good_then_bad": [{"a": 1}, {"a": 2**64}, {"a": math.nan}], +} + + +@pytest.mark.parametrize("tagged", [False, True], ids=["untagged", "tagged"]) +@pytest.mark.parametrize("name", sorted(CASES)) +def test_same_text_or_same_error_as_the_validation_loop(name, tagged): + rows = CASES[name] + new = tc.json_bulk_dumps_tagged if tagged else tc.json_bulk_dumps + assert _outcome(new, rows) == _outcome(lambda r: _old_dumps(r, tagged), rows) + + +@pytest.mark.parametrize("tagged", [False, True], ids=["untagged", "tagged"]) +def test_random_rows_agree_with_the_validation_loop(tagged): + atoms = [ + None, True, 0, -1, INT64_MAX, INT64_MIN, INT64_MAX + 1, 10**30, 1.5, 0.00012345678901234568, + 5e-324, math.nan, math.inf, "", "abc", "a\x00b", "é中", "\U0001f600", "a\ud800b", "\udc00", + "𐀀", "9999999999999999999", [1, 2], [math.nan], {"a": 1}, {1: 2}, (1, 2), + dt.datetime(2026, 1, 2), dt.date(2026, 1, 2), decimal.Decimal("1.5"), b"ab", np.int64(7), + np.float64(math.nan), Colour.RED, Name("s"), {1, 2}, object(), + ] # fmt: skip + rnd = random.Random(318) # nosec B311 - a seeded corpus, not security + new = tc.json_bulk_dumps_tagged if tagged else tc.json_bulk_dumps + for _ in range(3000): + rows = [] + for _ in range(rnd.choice([1, 2, 3, 8])): + keys = rnd.sample(["a", "b", "c", "d"], rnd.randint(0, 4)) + if rnd.random() < 0.03: + keys.append(1) + rows.append({k: rnd.choice(atoms) for k in keys}) + assert _outcome(new, rows) == _outcome(lambda r: _old_dumps(r, tagged), rows) + + +def _batch(n, bad): + """n scalar rows of the shape of a TPC-H part table; `bad` maps a row index to its replacement row.""" + rows = [ + { + "p_partkey": i, + "p_name": f"part {i}", + "p_size": i % 50, + "p_retailprice": 900.0 + (i % 1000) / 100, + "p_comment": None if i % 7 == 0 else "carefully final deposits", + } + for i in range(n) + ] + for index, row in bad.items(): + rows[index] = row + return rows + + +# kind -> (first bad row, a later bad row of another value), so the message tells which row raised +BAD_KINDS = { + "int_beyond_64_bits": ({"p_size": 2**63 + 11}, {"p_size": 2**70 + 22}), + "negative_beyond_64_bits": ({"p_size": INT64_MIN - 33}, {"p_size": 2**64 + 44}), + "lone_surrogate": ({"p_name": "a\ud800"}, {"p_size": 2**65}), + "non_str_key": ({7: "x"}, {"p_size": 2**65}), + "nested_int": ({"p_name": [1, 2**66]}, {"p_size": 2**65 + 55}), + "nan": ({"p_retailprice": math.nan}, {"p_size": 2**65}), + "subnormal": ({"p_retailprice": 5e-324}, {"p_size": 2**65}), +} + + +@pytest.mark.parametrize("tagged", [False, True], ids=["untagged", "tagged"]) +@pytest.mark.parametrize("kind", sorted(BAD_KINDS)) +def test_the_first_bad_row_of_a_large_batch_gives_the_old_error(kind, tagged): + first, later = BAD_KINDS[kind] + rows = _batch(20_000, {12_345: first, 19_000: later}) + new = tc.json_bulk_dumps_tagged if tagged else tc.json_bulk_dumps + expected = _outcome(lambda r: _old_dumps(r, tagged), rows) + assert expected[0] in ("ValueError", "TypeError"), expected + assert _outcome(new, rows) == expected + # the later row alone raises a message of its own. A NaN or a subnormal is refused by json.dumps, after the old loop had walked + # every row, so the old error there is the later row's; for every other kind the first bad row's error comes first + only_later = _batch(20_000, {19_000: later}) + assert _outcome(new, only_later) == _outcome( + lambda r: _old_dumps(r, tagged), only_later + ) + assert (_outcome(new, only_later) == expected) == (kind in ("nan", "subnormal")) + + +@pytest.mark.parametrize("tagged", [False, True], ids=["untagged", "tagged"]) +def test_the_first_bad_rows_error_carries_no_context(tagged): + # json.dumps refuses the NaN of the second row first; the error is still the walk's for the first row, as it was when the + # loop ran before json.dumps, and it does not show the NaN error as the context it was raised in + rows = [{"a": 2**70}, {"b": math.nan}] + new = tc.json_bulk_dumps_tagged if tagged else tc.json_bulk_dumps + with pytest.raises(ValueError) as caught: + new(rows) + assert str(caught.value) == _outcome(lambda r: _old_dumps(r, tagged), rows)[1] + assert caught.value.__context__ is None + # with nothing for the walk to find, json.dumps's own error is the one raised + with pytest.raises(ValueError, match="Out of range float"): + new([{"b": math.nan}]) + + +@pytest.mark.parametrize("tagged", [False, True], ids=["untagged", "tagged"]) +def test_a_clean_large_batch_is_the_plain_json_text(tagged): + rows = _batch(20_000, {}) + new = tc.json_bulk_dumps_tagged if tagged else tc.json_bulk_dumps + assert new(rows) == json.dumps(rows) + + +def test_scalar_rows_are_not_walked_when_the_text_carries_them(monkeypatch): + # the insert_many shape of the paper's l1tpc Part load and the create_vertices shape of l2, e2 and l3d: dict rows of str, int, float, + # bool and None. Numbers that look alike but fit: 19-digit ints below 2**63, floats whose text has a long run of digits after the + # point, text with digits, Hangul and CJK (BMP characters that json.dumps writes as \uXXXX, some of them as \ud...), and non-BMP + # characters (an emoji): json.dumps writes each as a surrogate PAIR (\ud83d\ude00), which the old text scan took for a lone surrogate + # and sent to the walk, a false alarm; UTF-8 encodes them, so the column screen passes them and the old loop accepted them too. + walks = [] + real = tc._check_rows + monkeypatch.setattr( + tc, + "_check_rows", + lambda rows, tagged: walks.append(tagged) or real(rows, tagged), + ) + rows = _batch(2_000, {}) + rows += [ + { + "ts": 1_700_000_000_123_456_789, + "hi": INT64_MAX, + "lo": INT64_MIN + 1, + "f": 0.00012345678901234568, + }, + { + "f": 1.2345678901234567e19, + "s": "id-123456789012345678", + "han": "한글퟿中文", + "nul": "a\x00b", + }, + { + "emoji": "\U0001f600", + "plane1": "\U00010000x\U0010ffff", + "mixed": "a\U0001f600b", + }, + {"e": "", "n": None, "b": True, "neg": -123456789012345678}, + ] + assert tc.json_bulk_dumps(rows) == json.dumps(rows) + assert tc.json_bulk_dumps_tagged(rows) == json.dumps(rows) + assert walks == [] + + +def test_tagged_kinds_are_not_walked_either(monkeypatch): + walks = [] + real = tc._check_rows + monkeypatch.setattr( + tc, + "_check_rows", + lambda rows, tagged: walks.append(tagged) or real(rows, tagged), + ) + rows = [ + {"d": dt.datetime(2026, 1, 2, 3, 4, 5), "day": dt.date(2026, 1, 2), "m": decimal.Decimal("1.50"), "b": b"ab", "n": np.int64(5), "f": np.float32(2.5)} + for _ in range(100) + ] # fmt: skip + assert tc.json_bulk_dumps_tagged(rows) + assert walks == [] + + +@pytest.mark.parametrize( + "row", + [ + {"a": 2**63}, + {"a": 10**20}, + {"a": INT64_MIN - 1}, + {"a": "a\ud800"}, + {7: "x"}, + {"a": [1]}, + {"a": 5e-324}, + {"a": Colour.RED}, + ], +) +def test_a_value_that_may_be_bad_runs_the_walk(monkeypatch, row): + walks = [] + real = tc._check_rows + monkeypatch.setattr( + tc, + "_check_rows", + lambda rows, tagged: walks.append(tagged) or real(rows, tagged), + ) + _outcome(tc.json_bulk_dumps, [{"id": 1}, row]) + assert walks, "the walk must decide a row the text cannot vouch for" + + +def test_the_column_screen_flags_exactly_the_ints_beyond_64_bits(): + # two regular expressions over the JSON text used to decide this; the text scan of the integer tokens cost 2.5 microseconds a row + # (12 ms of a 5,000-row create_vertices call of the paper's l2 shape), so the columns decide it now, in C, and the text is not scanned + rnd = random.Random(63) # nosec B311 - a seeded corpus, not security + values = [ + 0, + 1, + 10**18, + 2**63 - 2, + 2**63 - 1, + 2**63, + 2**63 + 1, + 10**19 - 1, + 10**19, + 10**20, + 9 * 10**18, + 9_999_999_999_999_999_999, + ] + values += [rnd.randint(10**18, 2**65) for _ in range(5000)] # fmt: skip + for value in values: + for number in (value, -value): + outside = not INT64_MIN <= number <= INT64_MAX + assert tc._rows_need_walk([{"a": number}], False) is outside, number + assert tc._rows_need_walk([{"a": None}, {"a": number}], False) is outside + assert tc._scalar_column_suspect([number, None]) is outside, number + # digits after the point of a float and inside an exponent are no integer token, and a float column is not screened at all + assert tc._rows_need_walk([{"f": 0.00012345678901234568}], False) is False + assert tc._rows_need_walk([{"f": 1.2345678901234567e19}], False) is False + assert tc._rows_need_walk([{"s": "id-123456789012345678901"}], False) is False + + +@pytest.mark.parametrize("tagged", [False, True], ids=["untagged", "tagged"]) +def test_a_bad_value_in_a_ragged_or_nullable_column_runs_the_walk_and_gives_the_old_error( + tagged, +): + new = tc.json_bulk_dumps_tagged if tagged else tc.json_bulk_dumps + for rows in ( + [{"a": 1}, {"b": "x"}, {"a": 2**70}], # the key is missing in the middle row + [{"a": None}, {"a": INT64_MIN - 1}], + [{"s": "ok"}, {"s": None}, {"s": "x\ud800"}], + [{"a": 1}, {"a": "text"}, {"a": 2**63}], # a column of two kinds + ): + assert tc._rows_need_walk(rows, tagged) is True, rows + expected = _outcome(lambda r: _old_dumps(r, tagged), rows) + assert expected[0] == "ValueError", expected + assert _outcome(new, rows) == expected + + +@pytest.mark.parametrize("tagged", [False, True], ids=["untagged", "tagged"]) +@pytest.mark.parametrize( + "text", + ["\U0001f600", "\U00010000", "\U0010ffff", "a\U0001f600b", "\ue000\U0001f600"], +) +def test_a_non_bmp_character_is_no_bad_value_and_is_not_walked( + monkeypatch, text, tagged +): + # the row the old text scan sent to the walk (a surrogate pair in the text) and the old loop accepted: same text, no walk + walks = [] + real = tc._check_rows + monkeypatch.setattr( + tc, + "_check_rows", + lambda rows, tagged: walks.append(tagged) or real(rows, tagged), + ) + rows = [{"id": 1}, {"a": text}] + new = tc.json_bulk_dumps_tagged if tagged else tc.json_bulk_dumps + assert _outcome(new, rows) == ("text", json.dumps(rows)) + assert _outcome(lambda r: _old_dumps(r, tagged), rows) == ("text", json.dumps(rows)) + assert walks == [] + # two lone surrogates next to each other are no pair: UTF-8 cannot encode them, the walk refuses them + bad = [{"id": 1}, {"a": "\ud83d\ude00"}] + got = _outcome(new, bad) + assert got == _outcome(lambda r: _old_dumps(r, tagged), bad) + assert got[0] == "ValueError" and walks == [tagged] + + +def test_a_nullable_column_of_a_tagged_kind_and_the_scalar_columns_are_not_walked( + monkeypatch, +): + walks = [] + real = tc._check_rows + monkeypatch.setattr( + tc, + "_check_rows", + lambda rows, tagged: walks.append(tagged) or real(rows, tagged), + ) + rows = [ + { + "id": i, + "born": None if i % 3 == 0 else dt.datetime(2026, 1, 2, 3, 4, i % 60), + "name": None if i % 5 == 0 else f"n{i}", + "w": i / 7, + } + for i in range(300) + ] + assert tc._rows_need_walk(rows, True) is False + assert tc.json_bulk_dumps_tagged(rows) + assert walks == [] + # a column of a tagged kind beside an int: the walk judges each value + assert tc._rows_need_walk([{"a": dt.date(2026, 1, 2)}, {"a": 5}], True) is True + + +def test_screen_vertex_rows_vets_a_table_of_exact_scalars(monkeypatch): + rows = _batch(2_000, {}) + rows += [ + { + "ts": 1_700_000_000_123_456_789, + "hi": INT64_MAX, + "lo": INT64_MIN, + "f": 0.00012345678901234568, + }, + {"han": "한글퟿中文", "nul": "a\x00b", "e": "", "n": None, "b": True}, + {"mixed": 1}, + {"mixed": 2.5}, # a column of two kinds: judged value by value, and it passes + {}, + ] + assert tc.screen_vertex_rows(rows, set()) == (False, True) + assert tc.screen_vertex_rows([{"a": "\U0001f600"}], set()) == (False, True) + # a vetted table is written with no check of its own: neither the type census nor the walk runs, and the text is the plain text + calls = [] + for name in ("_rows_need_walk", "_check_rows"): + real = getattr(tc, name) + monkeypatch.setattr( + tc, name, lambda *a, _n=name, _r=real: calls.append(_n) or _r(*a) + ) + assert tc.json_bulk_dumps(rows, vetted=True) == json.dumps(rows) + assert calls == [] + plain = rows[ + :2_000 + ] # no column of two kinds: the type census and the column screen decide, no walk + assert tc.json_bulk_dumps(plain) == json.dumps(plain) + assert calls == ["_rows_need_walk"], "an unvetted call still looks at its rows" + + +def test_screen_vertex_rows_does_not_vet_a_column_of_another_type(): + x = np.arange(12, dtype=np.float32).reshape(4, 3) + born = dt.datetime(2026, 1, 2) + assert tc.screen_vertex_rows([{"a": 1, "d": born}] * 3, set()) == (True, False) + assert tc.screen_vertex_rows([{"a": 1, "l": ["x", 2]}] * 3, set()) == (True, False) + # float32 arrays of an undeclared column, and the list of a declared one, are left to split_float_vectors + assert tc.screen_vertex_rows([{"v": row} for row in x], set()) == (False, False) + assert tc.screen_vertex_rows([{"v": row.tolist()} for row in x], {"v"}) == ( + False, + False, + ) + # an empty row (None counts as one) holds nothing to judge + assert tc.screen_vertex_rows([{"a": 1}, None, {}], set()) == (False, True) + + +@pytest.mark.parametrize( + "bad", [2**63, INT64_MIN - 1, math.nan, math.inf, 5e-324, "a\ud800", {1, 2}] +) +def test_screen_vertex_rows_gives_none_for_what_the_text_would_change(bad): + rows = [{"id": i, "x": 1.5} for i in range(300)] + assert tc.screen_vertex_rows(rows, set()) == (False, True) + rows[150]["x"] = bad + assert tc.screen_vertex_rows(rows, set()) is None + assert tc.classify_vertex_rows(rows, set(), True) is None + assert tc.screen_vertex_rows([{"id": 1}, {1: "a"}], set()) is None + + +# --------------------------------------------------------------------------- +# the entry points, on whichever wheel is installed + + +def _types(db): + db.command("sql", "CREATE DOCUMENT TYPE Part") + db.command("sql", "CREATE VERTEX TYPE PartV") + + +def _stored(db, type_name): + query = f"SELECT FROM {type_name}" # nosec B608 - test-controlled + rows = db.query("sql", query).to_list() + return sorted( + ( + r["p_partkey"], + r["p_name"], + r["p_size"], + r["p_retailprice"], + r.get("p_comment"), + ) + for r in rows + ) + + +def _expected(rows): + return sorted( + ( + r["p_partkey"], + r["p_name"], + r["p_size"], + r["p_retailprice"], + r.get("p_comment"), + ) + for r in rows + ) + + +def _rows_with_text(): + rows = _batch(25_000, {}) + rows[10]["p_name"] = "emoji \U0001f600 \U00010000" + rows[11]["p_name"] = "" + rows[12]["p_name"] = "한글 中文 ퟿" + rows[13]["p_size"] = INT64_MAX + rows[14]["p_size"] = INT64_MIN + return rows + + +def test_insert_many_stores_a_scalar_batch_as_before(temp_db_path): + rows = _rows_with_text() + with arcadedb.create_database(temp_db_path) as db: + _types(db) + assert db.insert_many("Part", rows) == len(rows) + assert _stored(db, "Part") == _expected(rows) + + +def test_create_vertices_stores_a_scalar_batch_as_before(temp_db_path): + rows = _rows_with_text() + with arcadedb.create_database(temp_db_path) as db: + _types(db) + with db.graph_batch(parallel_flush=False) as batch: + assert len(batch.create_vertices("PartV", rows)) == len(rows) + assert _stored(db, "PartV") == _expected(rows) + + +@pytest.mark.parametrize("entry", ["insert_many", "create_vertices"]) +@pytest.mark.parametrize("bad", [2**63 + 11, "a\ud800"], ids=["int", "surrogate"]) +def test_a_bad_row_inside_a_large_batch_is_refused(temp_db_path, entry, bad): + column = "p_size" if isinstance(bad, int) else "p_name" + rows = _batch(25_000, {12_345: {column: bad}, 19_000: {"p_size": 2**70 + 22}}) + with arcadedb.create_database(temp_db_path) as db: + _types(db) + with pytest.raises( + Exception + ): # noqa: B017 - the entry points raise their own types + if entry == "insert_many": + db.insert_many("Part", rows) + else: + with db.graph_batch(parallel_flush=False) as batch: + batch.create_vertices("PartV", rows) + + +def test_the_scalar_screen_flags_what_the_text_would_change_and_passes_the_rest(): + # the whole-input screen of create_vertices: conservative, so a false alarm is allowed, a miss is not + clean = { + "str": ["a", "é中", "\U0001f600", "", None], + "int": [1, -1, INT64_MAX, INT64_MIN, None], + "float": [0.5, -0.0, 0.0, 1e-300, 1.7e308, None], + "bool": [True, False, None], + "nones": [None, None], + } + for name, column in clean.items(): + assert tc._scalar_column_suspect(list(column)) is False, name + bad = { + "surrogate": ["ok", "a\ud800b"], + "above_max": [1, INT64_MAX + 1], + "below_min": [INT64_MIN - 1, 1], + "nan": [1.5, math.nan], + "inf": [math.inf], + "subnormal": [1.5, 5e-324], + "mixed": [1, "a"], + "int_float": [1, 2.5], + } + for name, column in bad.items(): + assert tc._scalar_column_suspect(list(column)) is True, name + # the screen runs over blocks: a bad value in the last block of a large column is found + column = [{"x": i} for i in range(250_000)] + column[249_999] = {"x": INT64_MAX + 1} + assert tc.classify_vertex_rows(column, set(), True) is None + assert tc.classify_vertex_rows(column, set(), False) is False diff --git a/tests/test_column_frames.py b/tests/test_column_frames.py new file mode 100644 index 0000000..98de315 --- /dev/null +++ b/tests/test_column_frames.py @@ -0,0 +1,713 @@ +"""The column frame decoder both wheels share (``arcadedb._columnar``), and the columnar readers that use it (refs #312). + +The first half builds frames by hand, in the layout of ``ColumnBatcher`` (JVM wheel) and ``arcadedb_n_next_col`` (native wheel), and +decodes them: no database, so a decoder change is checked on its own. The second half reads one stored table through ``to_arrow``, +``to_columns``, ``to_dataframe`` and ``to_polars`` and compares each value with what was stored, on whichever wheel runs the tests: +nulls kept in Arrow, one type per column whatever the batch size, a result of more than one frame, column subsets, an empty result, +one row, many columns, vectors of several lengths. There is no branch on ``arcadedb.__backend__``: a literal that holds on both wheels +is the parity check. +""" + +import json +import math +import struct +from datetime import datetime + +import numpy as np +import pandas as pd +import pyarrow as pa +import pytest + +from arcadedb import _columnar + + +def _json_values(data, side, count): + return json.loads(bytes(data)) + + +def _bits(count, nulls): + out = bytearray((count + 7) // 8) + for i in nulls: + out[i >> 3] |= 1 << (i & 7) + return bytes(out) + + +def _frame(count, columns): + """``columns``: ``(name, type, null_rows, data_bytes[, dim])`` each; the frame as ColumnBatcher writes it.""" + cols, body = [], b"" + for name, ctype, null_rows, data, *dim in columns: + nulls = _bits(count, null_rows) + head = {"name": name, "type": ctype, "nulls": len(nulls), "bytes": len(data)} + if dim: + head["dim"] = dim[0] + cols.append(head) + body += nulls + data + header = json.dumps({"count": count, "cols": cols}).encode("utf-8") + return memoryview(struct.pack(" 1 or t.num_rows == 8000 + + +def test_to_columns_of_the_stored_rows(table): + db, rows = table + c = db.query("sql", _sel()).to_columns() + assert c["big"].dtype == np.int64 and list(c["big"]) == [r["big"] for r in rows] + assert c["n"].dtype == np.float64 # an int column with nulls + assert c["d"].dtype == np.float64 and c["ts"].dtype == np.dtype("datetime64[ms]") + assert c["v9"].dtype == np.float32 and c["v9"].shape == (5000, 9) + assert c["v128"].shape == (5000, 128) + np.testing.assert_array_equal( + c["v9"], np.array([r["v9"] for r in rows], dtype=np.float32) + ) + assert c["s"] == [r["s"] for r in rows] + assert c["ok"] == [r["ok"] for r in rows] # a bool column with nulls is a list + + +def test_column_subset_and_order(table): + db, rows = table + t = db.query("sql", _sel()).to_arrow(columns=["s", "big"]) + assert t.column_names == ["s", "big"] + assert t.column("big").to_pylist() == [r["big"] for r in rows] + c = db.query("sql", _sel()).to_columns(columns=["v3"]) + assert list(c) == ["v3"] and c["v3"].shape == (5000, 3) + + +def test_empty_and_one_row_results(table): + db, rows = table + empty = db.query("sql", "SELECT n, s FROM T WHERE big < 0").to_arrow() + assert empty.num_rows == 0 + assert db.query("sql", "SELECT n, s FROM T WHERE big < 0").to_columns() == {} + one = db.query( + "sql", "SELECT n, s, v3 FROM T WHERE big = ?", rows[1]["big"] + ).to_arrow() + assert one.num_rows == 1 + assert one.column("s").to_pylist() == [rows[1]["s"]] + assert one.column("v3").to_pylist() == [rows[1]["v3"]] + assert db.query("sql", "SELECT FROM T WHERE big < 0").to_arrow().num_rows == 0 + + +def test_a_cursor_read_by_row_first_then_to_arrow(table): + db, rows = table + rs = db.query("sql", _sel()) + first = [next(rs) for _ in range(10)] + assert [r.get("big") for r in first] == [r["big"] for r in rows[:10]] + t = rs.to_arrow() + assert t.column("big").to_pylist() == [r["big"] for r in rows[10:]] + + +def test_many_columns(temp_db): + db = temp_db + db.command("sql", "CREATE DOCUMENT TYPE Many") + names = [f"c{k}" for k in range(80)] + for k, name in enumerate(names): + db.command( + "sql", + f"CREATE PROPERTY Many.{name} {['INTEGER', 'STRING', 'DOUBLE'][k % 3]}", + ) + rows = [ + { + name: (i + k if k % 3 == 0 else f"{i}-{k}" if k % 3 == 1 else (i + k) / 4) + for k, name in enumerate(names) + } + for i in range(600) + ] + db.insert_many("Many", rows) + t = db.query("sql", "SELECT FROM Many ORDER BY c0").to_arrow(columns=names) + assert t.column_names == names + for name in ("c0", "c1", "c2", "c79"): + assert t.column(name).to_pylist() == [r[name] for r in rows] + + +def test_to_dataframe_and_to_polars(table): + db, rows = table + df = db.query("sql", _sel()).to_dataframe() + assert list(df.columns) == [n for n, _ in TYPES] + assert list(df["big"]) == [r["big"] for r in rows] + # pandas decides how a missing string is shown (None in an object column, nan in the string dtype of pandas 3): the same on + # both wheels, since both build the frame from a list with None + assert [None if pd.isna(v) else v for v in df["s"].tolist()] == [ + r["s"] for r in rows + ] + assert [list(v) for v in df["v3"]] == [r["v3"] for r in rows] + assert df["ts"].iloc[1] == datetime(2026, 1, 2, 3, 4, 5, 1000) + assert str(df["ts"].dtype).startswith( + "datetime64[us" + ) # microseconds on both wheels, whatever pandas infers from datetimes + # a vector cell is a numpy float32 array, not a Python list (documented on to_dataframe) + assert all(isinstance(v, np.ndarray) and v.dtype == np.float32 for v in df["v3"]) + import polars as pl + + p = db.query("sql", _sel()).to_polars() + assert isinstance(p, pl.DataFrame) and p.height == 5000 + assert p.schema["v3"] == pl.Array(pl.Float32, 3) + assert p["big"].to_list() == [r["big"] for r in rows] + + +# --- what a column's type depends on: the whole result, not the size of the result or where a batch ends --------------------- + + +@pytest.mark.parametrize("batch_size", [2**31 - 1, 2**31, 2**32, 10**10]) +def test_a_huge_batch_size_means_as_many_as_one_batch_takes(table, batch_size): + db, rows = table + t = db.query("sql", _sel()).to_arrow(batch_size=batch_size) + assert t.num_rows == 5000 + c = db.query("sql", _sel()).to_columns(batch_size=batch_size) + assert len(c["big"]) == 5000 + + +@pytest.mark.parametrize("batch_size", [0, -1]) +def test_a_batch_size_below_one_is_refused(table, batch_size): + db, _ = table + with pytest.raises(ValueError): + db.query("sql", _sel()).to_arrow(batch_size=batch_size) + with pytest.raises(ValueError): + db.query("sql", _sel()).to_columns(batch_size=batch_size) + + +def test_no_rows_with_columns_given_returns_those_columns_empty(table): + db, _ = table + sql = "SELECT n, s FROM T WHERE big < 0" + t = db.query("sql", sql).to_arrow(columns=["s", "n"]) + assert t.column_names == ["s", "n"] and t.num_rows == 0 + assert db.query("sql", sql).to_columns(columns=["s", "n"]) == {"s": [], "n": []} + assert db.query("sql", sql).to_columns() == {} + assert db.query("sql", sql).to_arrow().num_columns == 0 + + +@pytest.mark.parametrize("limit", [1, 10, 300]) +def test_to_columns_arrays_are_writable(table, limit): + db, _ = table + c = db.query( + "sql", + f"SELECT big, d, ts, v3 FROM T ORDER BY big LIMIT {limit}", # nosec B608 - an integer of the test + ).to_columns() + for name in ("big", "d", "ts", "v3"): + assert c[name].flags.writeable, (name, limit) + c[name][0] = c[name][0] + + +@pytest.mark.parametrize("rows", [255, 256, 257, 511, 512, 513]) +def test_sizes_around_the_first_reply_and_two_of_them(table, rows): + db, stored = table + t = db.query("sql", _sel() + f" LIMIT {rows}").to_arrow() + assert t.num_rows == rows + assert t.column("big").to_pylist() == [r["big"] for r in stored[:rows]] + assert t.column("v9").to_pylist() == [r["v9"] for r in stored[:rows]] + assert t.schema.field("v9").type == pa.list_(pa.float32(), 9) + assert t.schema.field("s").type == pa.string() + + +def test_two_cursors_open_at_once_and_a_cursor_closed_early(table): + db, rows = table + one = db.query("sql", _sel()) + two = db.query("sql", _sel()) + assert next(one).get("big") == rows[0]["big"] + assert next(two).get("big") == rows[0]["big"] + t_two = two.to_arrow() + t_one = one.to_arrow() + assert t_one.column("big").to_pylist() == [r["big"] for r in rows[1:]] + assert t_two.column("big").to_pylist() == [r["big"] for r in rows[1:]] + partly = db.query("sql", _sel()) + for _ in range(300): + next(partly) + partly.close() + fresh = db.query("sql", _sel() + " LIMIT 400").to_arrow() + assert fresh.num_rows == 400 + + +def test_a_cursor_with_nothing_read_closed_many_times_over(table): + db, _ = table + # No ORDER BY: a sorted query holds its whole sorted buffer in the engine's query heap budget (2 GB by default) from the moment + # it starts, so 300 of them open at once exhausted it (macOS arm64 CI and a laptop, 2026-11-10). This test is about cursors that + # were never read, so the query streams. + unsorted = "SELECT n, big, d, f, s, ok, ts, v3, v9, v128 FROM T" + cursors = [db.query("sql", unsorted) for _ in range(300)] + for cursor in cursors: + cursor.close() + assert db.query("sql", _sel() + " LIMIT 3").to_arrow().num_rows == 3 + + +@pytest.fixture +def declared(temp_db): + db = temp_db + db.command("sql", "CREATE DOCUMENT TYPE Dec") + db.command("sql", "CREATE PROPERTY Dec.i INTEGER") + db.command("sql", "CREATE PROPERTY Dec.arr ARRAY_OF_FLOATS") + db.command("sql", "CREATE PROPERTY Dec.dbl ARRAY_OF_DOUBLES") + db.command("sql", "CREATE PROPERTY Dec.lst LIST") + rows = [ + { + "i": i, + "arr": [0.5 * i, 1.0, 2.0], + "dbl": [0.5 * i, 1.5, 2.5], + "lst": [0.5 * i, 1.0, 2.0], + } + for i in range(3000) + ] + db.insert_many("Dec", rows) + return db, rows + + +@pytest.mark.parametrize("rows", [10, 300, 3000]) +def test_a_vector_column_is_a_declared_array_whatever_the_result_size(declared, rows): + db, stored = declared + sql = f"SELECT i, arr, dbl, lst FROM Dec ORDER BY i LIMIT {rows}" # nosec B608 - an integer of the test + c = db.query("sql", sql).to_columns() + assert isinstance(c["arr"], np.ndarray) and c["arr"].dtype == np.float32 + assert isinstance(c["dbl"], np.ndarray) and c["dbl"].dtype == np.float64 + assert isinstance(c["lst"], list), type( + c["lst"] + ) # a LIST of doubles is lists, not a float32 matrix that happens to fit + assert c["lst"] == [r["lst"] for r in stored[:rows]] + t = db.query("sql", sql).to_arrow() + assert t.schema.field("arr").type == pa.list_(pa.float32(), 3) + assert t.schema.field("dbl").type == pa.list_(pa.float64(), 3) + assert t.schema.field("lst").type == pa.list_(pa.float64()) + assert t.column("dbl").to_pylist() == [r["dbl"] for r in stored[:rows]] + + +def test_a_double_array_stays_double_after_rows_read_past_the_first_reply(declared): + db, stored = declared + rs = db.query("sql", "SELECT i, dbl, arr FROM Dec ORDER BY i") + for _ in range(300): + next(rs) + t = rs.to_arrow() + assert t.schema.field("dbl").type == pa.list_(pa.float64(), 3) + assert t.schema.field("arr").type == pa.list_(pa.float32(), 3) + assert t.column("dbl").to_pylist() == [r["dbl"] for r in stored[300:]] + + +def test_a_property_that_is_numeric_from_row_30000_on_has_one_type(temp_db): + db = temp_db + db.command("sql", "CREATE DOCUMENT TYPE Late") + db.command("sql", "CREATE PROPERTY Late.i INTEGER") + rows = [{"i": i} for i in range(30000)] + [ + {"i": i, "late": i * 2} for i in range(30000, 30100) + ] + db.insert_many("Late", rows) + sql = "SELECT i, late FROM Late ORDER BY i" + t = db.query("sql", sql).to_arrow(batch_size=25_000, columns=["i", "late"]) + assert t.schema.field("late").type == pa.int64() + late = t.column("late").to_pylist() + assert late[:30000] == [None] * 30000 and late[30000:] == [ + i * 2 for i in range(30000, 30100) + ] + c = db.query("sql", sql).to_columns(batch_size=25_000, columns=["i", "late"]) + assert isinstance(c["late"], np.ndarray) and c["late"].dtype == np.float64 + assert np.isnan(c["late"][:30000]).all() + assert list(c["late"][30000:]) == [i * 2 for i in range(30000, 30100)] + + +@pytest.mark.parametrize("rows", [10, 300]) +def test_an_integer_beyond_2_53_next_to_a_float_stays_exact(temp_db, rows): + db = temp_db + db.command("sql", "CREATE DOCUMENT TYPE Mix") + db.command("sql", "CREATE PROPERTY Mix.i INTEGER") + big = 2**60 + 1 + db.insert_many( + "Mix", [{"i": i, "x": big if i % 2 == 0 else 1.5} for i in range(rows)] + ) + c = db.query("sql", "SELECT i, x FROM Mix ORDER BY i").to_columns() + want = [big if i % 2 == 0 else 1.5 for i in range(rows)] + assert list(c["x"]) == want + assert all(type(a) is type(b) for a, b in zip(c["x"], want)) + t = db.query("sql", "SELECT i, x FROM Mix ORDER BY i").to_arrow() + assert t.column("x").to_pylist() == [ + str(v) for v in want + ] # no common numeric type: exact text, as across batches + + +# --- to_dataframe and an integer beyond 2**53 (#340) ----------------------------------------------------------------------- +# to_columns() gives the exact Python list and to_arrow() the exact text; to_dataframe() handed that list to pandas, which infers +# float64 for ints and floats and rounded the integer. Each test reads the frame, so the same literal holds on both wheels. + + +def _mix_frame(db, values): + """The ``to_dataframe()`` of ``values`` stored in property ``x`` of one document each, in order (None = the row lacks ``x``).""" + db.command("sql", "CREATE DOCUMENT TYPE Mix") + db.command("sql", "CREATE PROPERTY Mix.i INTEGER") + db.insert_many( + "Mix", + [{"i": i} if v is None else {"i": i, "x": v} for i, v in enumerate(values)], + ) + return db.query("sql", "SELECT i, x FROM Mix ORDER BY i").to_dataframe() + + +def _same_values_and_types(have, want): + return list(have) == list(want) and all( + type(a) is type(b) for a, b in zip(have, want) + ) + + +@pytest.mark.parametrize("layout", ["alternating", "blocks"]) +@pytest.mark.parametrize("rows", [10, 300]) +def test_to_dataframe_keeps_an_integer_beyond_2_53_next_to_a_float( + temp_db, rows, layout +): + big = 2**60 + 1 + if layout == "alternating": + want = [big if i % 2 == 0 else 1.5 for i in range(rows)] + else: + want = [big if i < rows // 2 else 1.5 for i in range(rows)] + df = _mix_frame(temp_db, want) + assert df["x"].dtype == object # float64 would round the integer + assert _same_values_and_types(df["x"].tolist(), want) # an int stays an int + assert df["i"].dtype == np.int64 and df["i"].tolist() == list(range(rows)) + + +@pytest.mark.parametrize( + "big", [2**53 + 1, -(2**53) - 1, 2**63 - 1, -(2**63) + 1, -(2**63)] +) +def test_to_dataframe_keeps_the_ends_of_int64_next_to_a_float(temp_db, big): + want = [big, 1.5, big, -0.25] + df = _mix_frame(temp_db, want) + assert df["x"].dtype == object + assert _same_values_and_types(df["x"].tolist(), want) + + +def test_to_dataframe_mixed_column_with_a_null_keeps_the_null_as_none(temp_db): + big = 2**60 + 1 + df = _mix_frame(temp_db, [big, None, 1.5, -(2**63)]) + assert df["x"].dtype == object + assert df["x"].tolist() == [big, None, 1.5, -(2**63)] + assert df["x"][1] is None # not NaN: a missing value is None as in a text column + assert type(df["x"][0]) is int and type(df["x"][2]) is float + + +def test_to_dataframe_integer_beyond_2_53_next_to_text_and_a_float(temp_db): + want = [2**60 + 1, "text", 1.5, -(2**63)] + df = _mix_frame(temp_db, want) + assert df["x"].dtype == object + assert _same_values_and_types(df["x"].tolist(), want) + + +def test_to_dataframe_integers_alone_stay_int64_even_beyond_2_53(temp_db): + want = [2**60 + 1, -(2**63), 2**63 - 1, 7] + df = _mix_frame(temp_db, want) + assert df["x"].dtype == np.int64 # unchanged: exact, no float in the column + assert df["x"].tolist() == want + + +def test_to_dataframe_integers_up_to_2_53_next_to_floats_stay_float64(temp_db): + # every integer here is an exact double, so the usual numeric column is kept (the dtype does not change for this shape) + df = _mix_frame(temp_db, [1, 1.5, 2**53, -(2**53)]) + assert df["x"].dtype == np.float64 + assert df["x"].tolist() == [1.0, 1.5, 2.0**53, -(2.0**53)] + + +def test_to_dataframe_of_an_empty_result_is_an_empty_frame(temp_db): + _mix_frame(temp_db, [2**60 + 1, 1.5]) + df = temp_db.query("sql", "SELECT i, x FROM Mix WHERE i < 0").to_dataframe() + assert len(df) == 0 and list(df.columns) == [] + + +def test_frames_of_an_integer_beyond_2_53_and_frames_of_floats_merge_exactly(): + # the column's type belongs to the whole result: two frames, one of int64 (beyond 2**53) and one of float64, must not go + # through float64 when they merge (np.concatenate would round the integers), and to_dataframe must keep the exact values + ints = _frame(2, [("n", "i8", [], struct.pack("<2q", 2**60 + 1, -(2**63)))]) + floats = _frame(2, [("n", "f8", [], struct.pack("<2d", 0.5, 1.5))]) + columns = _columnar.merge_columns(["n"], [_columns(ints), _columns(floats)]) + want = [2**60 + 1, -(2**63), 0.5, 1.5] + assert _same_values_and_types(columns["n"], want) + df = _columnar.dataframe_from_columns(columns) + assert df["n"].dtype == object + assert _same_values_and_types(df["n"].tolist(), want) + # integers within 2**53 next to floats keep the numeric column of the merge (float64), and so does the frame + small = _frame(2, [("n", "i8", [], struct.pack("<2q", 1, -(2**53)))]) + columns = _columnar.merge_columns(["n"], [_columns(small), _columns(floats)]) + assert columns["n"].dtype == np.float64 + assert _columnar.dataframe_from_columns(columns)["n"].dtype == np.float64 + + +def test_vectors_whose_dimension_changes_between_frames_are_one_list_column(temp_db): + db = temp_db + db.command("sql", "CREATE DOCUMENT TYPE Dim") + db.command("sql", "CREATE PROPERTY Dim.i INTEGER") + db.command("sql", "CREATE PROPERTY Dim.v ARRAY_OF_FLOATS") + db.insert_many( + "Dim", + [{"i": i, "v": [0.5, 1.0, 2.0] if i < 300 else [0.5, 1.0]} for i in range(600)], + ) + t = db.query("sql", "SELECT i, v FROM Dim ORDER BY i").to_arrow(batch_size=100) + assert t.schema.field("v").type == pa.list_( + pa.float64() + ) # a lossless widening, wherever the dimension changes + assert t.column("v").to_pylist() == [[0.5, 1.0, 2.0]] * 300 + [[0.5, 1.0]] * 300 + + +def test_vector_types_unify_by_value_and_dimension_never_to_text(): + def vec(dim, ctype, dtype, rows=2): + data = np.arange(rows * dim, dtype=dtype).reshape(rows, dim).tobytes() + return _frame(rows, [("v", ctype, [], data, dim)]) + + f3, f2 = _arrow(vec(3, "f4v", " counts[1000] >= 1, (reader, counts) def test_an_empty_result_has_no_columns(temp_db): diff --git a/tests/test_graph_batch_single_pass.py b/tests/test_graph_batch_single_pass.py new file mode 100644 index 0000000..a6aa6a9 --- /dev/null +++ b/tests/test_graph_batch_single_pass.py @@ -0,0 +1,201 @@ +"""GraphBatch.create_vertices (the JVM wheel) decides its rows by their types and writes each chunk once. + +Before, `_create_vertices_json_bulk` called `json_bulk_scalar_ok` on every value of every row before the first chunk was written, and +the chunk was then walked again by json.dumps. Now `type_conversion.classify_vertex_rows` takes a census of the types in C, and the +value checks come from the one json.dumps pass of each chunk (`json_bulk_dumps`, as `json_bulk_dumps_tagged` already did for the tagged +text). The rule that keeps the stored type of a property from depending on which chunk held a bad value: the columns of exact scalars are +screened in C over the whole input, one chunk or several (`type_conversion.screen_vertex_rows`), and a bad scalar (a NaN, an integer beyond +64 bits, a lone surrogate, a subnormal) anywhere sends the WHOLE input the property-matrix way, as it did before. A name that is not a str +or a type the text does not carry does the same (`_create_vertices_json_bulk` returns None). An input of exact scalars only has then been +looked at once, as columns, so its chunks are written with no check of their own (no text scan, no walk): the 5,000-row call of the l2 +Person load spent 12 ms of 43 in two regular expressions over the JSON text before this (#318 follow-up, the g1 regression of candidate 3). +""" + +import pytest + +import arcadedb +from arcadedb import graph_batch as gbm +from arcadedb import type_conversion as tc +from arcadedb.graph_batch import GraphBatch + + +def _rows(n): + return [ + {"id": i, "name": f"n{i}", "score": i * 0.5, "flag": i % 2 == 0, "note": None} + for i in range(n) + ] + + +@pytest.mark.parametrize("chunk", [100_000, 500], ids=["one_chunk", "screened"]) +def test_scalar_rows_are_not_judged_value_by_value(temp_db_path, monkeypatch, chunk): + def refuse(value): + raise AssertionError("a per-value gate ran over a scalar table") + + monkeypatch.setattr(GraphBatch, "_BULK_CHUNK", chunk) + monkeypatch.setattr(tc, "json_bulk_scalar_ok", refuse) + monkeypatch.setattr(gbm, "json_bulk_scalar_ok", refuse) + monkeypatch.setattr(tc, "json_bulk_shape_ok", refuse) + monkeypatch.setattr(gbm, "json_bulk_shape_ok", refuse) + rows = _rows(2_000) + with arcadedb.create_database(temp_db_path) as db: + db.command("sql", "CREATE VERTEX TYPE V") + with db.graph_batch(parallel_flush=False) as batch: + assert len(batch._create_vertices_json_bulk("V", rows)) == len(rows) + count = db.query("sql", "SELECT count(*) AS c FROM V").first().get("c") + assert count == len(rows) + + +def test_the_whole_input_still_goes_the_matrix_way_for_a_name_or_type_the_text_does_not_carry( + temp_db_path, +): + rows = _rows(5) + with arcadedb.create_database(temp_db_path) as db: + db.command("sql", "CREATE VERTEX TYPE V") + with db.graph_batch(parallel_flush=False) as batch: + assert len(batch._create_vertices_json_bulk("V", rows)) == len(rows) + assert batch._create_vertices_json_bulk("V", rows + [{1: "a"}]) is None + assert batch._create_vertices_json_bulk("V", rows + [{"k": {1, 2}}]) is None + + +@pytest.mark.parametrize("value", [5e-324, float("nan"), 2**70, "a\ud800b"]) +@pytest.mark.parametrize("chunks", [1, 3], ids=["one_chunk", "three_chunks"]) +def test_a_scalar_the_text_changes_sends_the_whole_input_through_the_matrix_path( + temp_db_path, monkeypatch, value, chunks +): + monkeypatch.setattr(GraphBatch, "_BULK_CHUNK", 100 if chunks == 3 else 1_000) + calls = [] + real = GraphBatch._to_java_property_matrix + + def spy(self, rows): + calls.append(len(rows)) + return real(self, rows) + + monkeypatch.setattr(GraphBatch, "_to_java_property_matrix", spy) + rows = [{"id": i, "x": 1.5} for i in range(250)] + rows[150] = {"id": 150, "x": value} + with arcadedb.create_database(temp_db_path) as db: + db.command("sql", "CREATE VERTEX TYPE V") + with db.graph_batch(parallel_flush=False) as batch: + try: + rids = batch.create_vertices("V", rows) + except ( + Exception + ): # noqa: BLE001 - the matrix path refuses a value that no entry point stores + rids = None + assert calls == [250], "the whole input takes the matrix path, once" + if value == 5e-324: + # stored as the double every other entry point stores; the text of a subnormal was read as a DECIMAL + assert len(rids) == 250 + stored = db.query("sql", "SELECT x FROM V WHERE id = 150").first().get("x") + assert stored == 5e-324 and type(stored) is float + + +@pytest.mark.parametrize( + "bad_row", [5, 150_000, 249_999], ids=["first_chunk", "middle_chunk", "last_chunk"] +) +def test_the_stored_type_does_not_depend_on_which_chunk_held_a_bad_value( + temp_db_path, monkeypatch, bad_row +): + """An undeclared int property is stored as the same Java type in every chunk, whichever chunk holds a NaN in another property + (the 100,000-row chunking of the real class): the NaN sends the whole input the matrix way, so no chunk is stored from the JSON text. + """ + n = 250_000 + rows = [{"id": i, "k": i % 1000} for i in range(n)] + rows[bad_row] = {"id": bad_row, "k": 1, "x": float("nan")} + probes = {7: "chunk 1", 150_001: "chunk 2", 249_999: "chunk 3"} + probes.pop(bad_row, None) + calls = [] + real = GraphBatch._to_java_property_matrix + + def spy(self, chunk): + calls.append(len(chunk)) + return real(self, chunk) + + monkeypatch.setattr(GraphBatch, "_to_java_property_matrix", spy) + with arcadedb.create_database(temp_db_path) as db: + db.command("sql", "CREATE VERTEX TYPE V") + with db.graph_batch(parallel_flush=False) as batch: + assert len(batch.create_vertices("V", rows)) == n + assert calls == [n], "the whole input takes the matrix path, once" + types = _stored_types(db, probes) + assert ( + len(set(types.values())) == 1 + ), f"the stored type depends on the chunk: {types}" + + +def test_a_clean_input_of_several_chunks_is_stored_alike_in_every_chunk( + temp_db_path, monkeypatch +): + n = 250_000 + rows = [{"id": i, "k": i % 1000} for i in range(n)] + probes = {7: "chunk 1", 150_001: "chunk 2", 249_999: "chunk 3"} + calls = [] + real = GraphBatch._to_java_property_matrix + monkeypatch.setattr( + GraphBatch, + "_to_java_property_matrix", + lambda self, chunk: calls.append(len(chunk)) or real(self, chunk), + ) + with arcadedb.create_database(temp_db_path) as db: + db.command("sql", "CREATE VERTEX TYPE V") + with db.graph_batch(parallel_flush=False) as batch: + assert len(batch.create_vertices("V", rows)) == n + assert calls == [], "a clean input never takes the matrix path" + types = _stored_types(db, probes) + assert ( + len(set(types.values())) == 1 + ), f"the stored type depends on the chunk: {types}" + + +def _stored_types(db, probes): + ids = ", ".join(str(i) for i in probes) + query = f"SELECT id, k, k.javaType() AS t FROM V WHERE id IN [{ids}]" # nosec B608 + got = db.query("sql", query).to_list() + by_id = {r["id"]: (r["k"], str(r["t"])) for r in got} + assert set(by_id) == set(probes) + assert {i: by_id[i][0] for i in probes} == {i: i % 1000 for i in probes} + return {probes[i]: by_id[i][1] for i in probes} + + +@pytest.mark.parametrize("chunk", [100_000, 500], ids=["one_chunk", "screened"]) +def test_a_screened_scalar_table_is_written_with_no_check_per_chunk( + temp_db_path, monkeypatch, chunk +): + # the l2 Person shape: the input was screened once as columns, so no chunk runs the census again or the walk (candidate 3 ran two + # regular expressions over every chunk's text here: 31 percent more wall for the 5,000-row call) + calls = [] + for name in ("_rows_need_walk", "_check_rows"): + real = getattr(tc, name) + monkeypatch.setattr( + tc, name, lambda *a, _n=name, _r=real: calls.append(_n) or _r(*a) + ) + monkeypatch.setattr(GraphBatch, "_BULK_CHUNK", chunk) + rows = _rows(2_000) + with arcadedb.create_database(temp_db_path) as db: + db.command("sql", "CREATE VERTEX TYPE V") + with db.graph_batch(parallel_flush=False) as batch: + assert len(batch.create_vertices("V", rows)) == len(rows) + assert db.query("sql", "SELECT count(*) AS c FROM V").first().get("c") == 2_000 + assert calls == [] + + +def test_rows_with_another_kind_of_column_are_still_checked_chunk_by_chunk( + temp_db_path, monkeypatch +): + from datetime import datetime + + walks = [] + real = tc._rows_need_walk + monkeypatch.setattr( + tc, + "_rows_need_walk", + lambda rows, tagged: walks.append(len(rows)) or real(rows, tagged), + ) + monkeypatch.setattr(GraphBatch, "_BULK_CHUNK", 100) + rows = [{"id": i, "born": datetime(2026, 1, 2, 3, 4, 5)} for i in range(250)] + with arcadedb.create_database(temp_db_path) as db: + db.command("sql", "CREATE VERTEX TYPE V") + with db.graph_batch(parallel_flush=False) as batch: + assert len(batch.create_vertices("V", rows)) == 250 + assert db.query("sql", "SELECT count(*) AS c FROM V").first().get("c") == 250 + assert walks == [100, 100, 50] diff --git a/tests/test_graph_batch_tagged_shapes.py b/tests/test_graph_batch_tagged_shapes.py index 2da99c1..560b2d9 100644 --- a/tests/test_graph_batch_tagged_shapes.py +++ b/tests/test_graph_batch_tagged_shapes.py @@ -438,6 +438,8 @@ def shape(*a, **k): monkeypatch.setattr(gbm, "json_bulk_dumps_tagged", dumps) monkeypatch.setattr(gbm, "json_bulk_shape_ok", shape) + # create_vertices decides the shapes in type_conversion.classify_vertex_rows, which calls the gate there + monkeypatch.setattr(type_conversion, "json_bulk_shape_ok", shape) return calls diff --git a/tests/test_graph_batch_vector_crossings_jvm.py b/tests/test_graph_batch_vector_crossings_jvm.py new file mode 100644 index 0000000..a86b89b --- /dev/null +++ b/tests/test_graph_batch_vector_crossings_jvm.py @@ -0,0 +1,322 @@ +"""GraphBatch.create_vertices (the JVM wheel): a Java float[] or float32 array crosses JPype as one matrix per property and chunk (#337). + +Before, a Java ``float[]`` (what ``to_java_float_array`` returns) was no vector candidate: ``classify_vertex_rows`` gave up on the first +row, and the whole load went through ``_to_java_property_matrix``, which converts every value of every row with ``convert_python_to_java``, +builds one ``Object[]`` per row and one ``Object[][]`` for the call (an unwrapped Python loop that was 59 to 61 percent of the wall of the +e2 and l3d loads in the untimed ingest profile of 26.11.1). A float32 numpy array on a property that is not declared ``ARRAY_OF_FLOATS`` +took the same way. Now ``take_vector_columns`` stacks the column once (numpy reads a Java array through the buffer protocol), one +``float[]`` copy and one ``FloatVectors.Matrix`` cross per property, and one ``VertexBatcher.createVerticesJson`` call per chunk. + +The JPype crossings are counted by wrapping the bridge classes and the Python-side conversion; the stored values are in +test_vertex_vector_loads.py. The JPype bridge does not exist on the native wheel (its counts are in test_native_vertex_vector_crossings.py), +so conftest.py does not collect this file there. +""" + +import jpype +import numpy as np +import pytest + +import arcadedb +from arcadedb import graph_batch as gbm +from arcadedb import type_conversion as tc +from arcadedb.graph_batch import GraphBatch + +ROWS, CHUNK, DIM = 250, 100, 16 # three chunks: 100, 100, 50 + + +class _Counting: + """Forwards every call of a Java class (or of the bridge's FloatVectors) and records its name.""" + + def __init__(self, target, log): + self._target, self._log = target, log + + def __getattr__(self, name): + attribute = getattr(self._target, name) + if not callable(attribute): + return attribute + + def call(*args): + self._log.append(name) + return attribute(*args) + + return call + + +@pytest.fixture +def crossings(monkeypatch): + """What the load does: property-matrix fallbacks, per-value conversions, bridge calls, Matrix constructions, per-row inserts.""" + log = {"fallback": [], "convert": [], "bridge": [], "matrix": [], "per_row": []} + + real_matrix_path = GraphBatch._to_java_property_matrix + + def fallback(self, rows): + log["fallback"].append(len(rows)) + return real_matrix_path(self, rows) + + monkeypatch.setattr(GraphBatch, "_to_java_property_matrix", fallback) + + real_convert = tc.convert_python_to_java + + def convert(value): + log["convert"].append(type(value).__name__) + return real_convert(value) + + monkeypatch.setattr(tc, "convert_python_to_java", convert) + monkeypatch.setattr(gbm, "convert_python_to_java", convert) + + real_jclass = jpype.JClass + + def jclass(name, *args, **kwargs): + cls = real_jclass(name, *args, **kwargs) + if name == "com.arcadedb.python.VertexBatcher": + return _Counting(cls, log["bridge"]) + return cls + + monkeypatch.setattr(jpype, "JClass", jclass) + + real_vectors = tc._float_vectors_class + + def vectors_class(): + vectors = real_vectors() + return None if vectors is None else _Counting(vectors, log["matrix"]) + + monkeypatch.setattr(tc, "_float_vectors_class", vectors_class) + + from arcadedb import core + + real_per_row = core.Database._insert_many_per_row + + def per_row(self, type_name, rows): + log["per_row"].append(len(rows)) + return real_per_row(self, type_name, rows) + + monkeypatch.setattr(core.Database, "_insert_many_per_row", per_row) + monkeypatch.setattr(GraphBatch, "_BULK_CHUNK", CHUNK) + return log + + +@pytest.fixture +def db(temp_db): + for name in ("Declared", "Free", "Doc"): + kind = "DOCUMENT" if name == "Doc" else "VERTEX" + temp_db.command("sql", f"CREATE {kind} TYPE {name}") + temp_db.command("sql", f"CREATE PROPERTY {name}.id LONG") + temp_db.command("sql", "CREATE PROPERTY Declared.embedding ARRAY_OF_FLOATS") + temp_db.command("sql", "CREATE PROPERTY Doc.embedding ARRAY_OF_FLOATS") + return temp_db + + +def _x(n=ROWS, dim=DIM): + return np.random.default_rng(7).standard_normal((n, dim)).astype(np.float32) + + +def _floats(x): + return [jpype.JArray(jpype.JFloat)(row) for row in x] + + +@pytest.mark.parametrize("type_name", ["Declared", "Free"]) +@pytest.mark.parametrize("shape", ["java_float_array", "float32_ndarray"]) +def test_a_float32_vector_crosses_as_one_matrix_per_chunk( + db, crossings, type_name, shape +): + x = _x() + vectors = _floats(x) if shape == "java_float_array" else list(x) + rows = [{"id": i, "embedding": vectors[i]} for i in range(ROWS)] + with db.graph_batch(parallel_flush=False) as gb: + assert len(gb.create_vertices(type_name, rows)) == ROWS + assert ( + crossings["fallback"] == [] + ), "the property-matrix path ran (a JPype conversion per value and per row)" + assert crossings["convert"] == [], "a value was converted one at a time" + assert ( + crossings["bridge"] == ["createVerticesJson"] * 3 + ) # one crossing per chunk of 100 + # one float32 matrix per chunk: `declared` is asked once per call, `Matrix` once per chunk + assert crossings["matrix"].count("Matrix") == 3 + sql = f"SELECT id, embedding FROM {type_name}" # nosec B608 - test-owned name + stored = {r["id"]: r["embedding"] for r in db.query("sql", sql).to_list()} + for i in range(ROWS): + assert np.array(stored[i], dtype=np.float32).tobytes() == x[i].tobytes() + + +def test_two_vector_properties_cross_as_two_matrices_per_chunk(db, crossings): + x, y = _x(), _x(dim=8) + rows = [ + {"id": i, "embedding": jpype.JArray(jpype.JFloat)(x[i]), "second": y[i]} + for i in range(ROWS) + ] + with db.graph_batch(parallel_flush=False) as gb: + gb.create_vertices("Declared", rows) + assert crossings["fallback"] == [] and crossings["convert"] == [] + assert crossings["bridge"] == ["createVerticesJson"] * 3 + assert crossings["matrix"].count("Matrix") == 6 + + +def test_a_ragged_column_still_takes_the_property_matrix_path_for_its_chunk( + db, crossings +): + x = _x() + vectors = _floats(x) + rows = [{"id": i, "embedding": vectors[i]} for i in range(ROWS)] + rows[120] = {"id": 120, "embedding": jpype.JArray(jpype.JFloat)(x[120][:5])} + with db.graph_batch(parallel_flush=False) as gb: + assert len(gb.create_vertices("Declared", rows)) == ROWS + assert crossings["fallback"] == [100], "only the chunk of row 120" + assert crossings["bridge"] == ["createVerticesJson"] * 2 + stored = { + r["id"]: r["embedding"] + for r in db.query("sql", "SELECT id, embedding FROM Declared").to_list() + } + assert np.array(stored[120], dtype=np.float32).tobytes() == x[120][:5].tobytes() + assert np.array(stored[0], dtype=np.float32).tobytes() == x[0].tobytes() + + +def test_a_java_double_array_is_a_vector_only_on_a_declared_property(db, crossings): + x64 = np.random.default_rng(8).random((ROWS, DIM)) * 1000.0 + 1e-3 + rows = [ + {"id": i, "embedding": jpype.JArray(jpype.JDouble)(x64[i])} for i in range(ROWS) + ] + with db.graph_batch(parallel_flush=False) as gb: + gb.create_vertices("Declared", rows) + assert ( + crossings["fallback"] == [] + and crossings["bridge"] == ["createVerticesJson"] * 3 + ) + declared = { + r["id"]: r["embedding"] + for r in db.query("sql", "SELECT id, embedding FROM Declared").to_list() + } + for i in ( + 0, + 99, + 100, + ROWS - 1, + ): # the engine stores floats for the declared property + want = x64[i].astype(np.float32).tobytes() + assert np.array(declared[i], dtype=np.float32).tobytes() == want + del crossings["fallback"][:], crossings["bridge"][:] + with db.graph_batch(parallel_flush=False) as gb: + gb.create_vertices("Free", rows) + assert crossings["fallback"] == [ + ROWS + ], "undeclared: the double[] keeps the matrix path" + free = { + r["id"]: r["embedding"] + for r in db.query("sql", "SELECT id, embedding FROM Free").to_list() + } + assert np.array(free[5], dtype=np.float64).tolist() == x64[5].tolist() + + +def test_insert_many_sends_a_java_float_array_column_through_the_matrix(db, crossings): + x = _x(50) + arrays = _floats(x) + rows = [{"id": i, "embedding": arrays[i]} for i in range(50)] + assert db.insert_many("Doc", rows) == 50 + assert crossings["per_row"] == [], "the column went row by row" + assert crossings["matrix"].count("Matrix") == 1 + stored = { + r["id"]: r["embedding"] + for r in db.query("sql", "SELECT id, embedding FROM Doc").to_list() + } + for i in range(50): + assert np.array(stored[i], dtype=np.float32).tobytes() == x[i].tobytes() + + +def test_take_vector_columns_reads_java_arrays_through_one_stack(db): + float_array, double_array = jpype.JArray(jpype.JFloat), jpype.JArray(jpype.JDouble) + x = _x(6, 5) + floats = [{"v": float_array(row), "k": i} for i, row in enumerate(x)] + names, matrices, rest = tc.take_vector_columns(floats, (), undeclared=True) + assert names == ["v"] and matrices[0].dtype == np.float32 + assert matrices[0].tobytes() == x.tobytes() and rest == [{"k": i} for i in range(6)] + doubles = [{"v": double_array(row.astype(np.float64))} for row in x] + assert tc.take_vector_columns(doubles, (), undeclared=True) is None + assert tc.take_vector_columns(doubles, ["v"])[1][0].tobytes() == x.tobytes() + ints = [{"v": jpype.JArray(jpype.JInt)([1, 2, 3])} for _ in range(3)] + assert tc.take_vector_columns(ints, ["v"], undeclared=True) is None + assert not tc.is_vector_candidate(ints[0]["v"]) + assert tc.is_vector_candidate(floats[0]["v"]) and tc.is_vector_candidate( + doubles[0]["v"] + ) + assert isinstance(arcadedb.to_java_float_array(x[0]), float_array) + + +def test_rows_with_vectors_cross_in_blocks_sized_by_their_width( + db, crossings, monkeypatch +): + # about _VECTOR_BLOCK_FLOATS floats per crossing over all the vector columns, never fewer than _VECTOR_BLOCK_MIN_ROWS rows + monkeypatch.setattr(GraphBatch, "_BULK_CHUNK", 100_000) + monkeypatch.setattr(GraphBatch, "_VECTOR_BLOCK_FLOATS", 640) # 40 rows of 16 floats + monkeypatch.setattr(GraphBatch, "_VECTOR_BLOCK_MIN_ROWS", 10) + x = _x() + arrays = _floats(x) + rows = [{"id": i, "embedding": arrays[i]} for i in range(ROWS)] + with db.graph_batch(parallel_flush=False) as gb: + assert len(gb.create_vertices("Declared", rows)) == ROWS + assert crossings["fallback"] == [] and crossings["convert"] == [] + assert ( + crossings["bridge"] == ["createVerticesJson"] * 7 + ) # 6 blocks of 40, one of 10 + assert crossings["matrix"].count("Matrix") == 7 + sql = "SELECT id, embedding FROM Declared" + stored = {r["id"]: r["embedding"] for r in db.query("sql", sql).to_list()} + assert len(stored) == ROWS + for i in range(ROWS): + assert np.array(stored[i], dtype=np.float32).tobytes() == x[i].tobytes() + # the floor: a block is never smaller than _VECTOR_BLOCK_MIN_ROWS rows + crossings["bridge"].clear() + monkeypatch.setattr(GraphBatch, "_VECTOR_BLOCK_FLOATS", 16) + rows = [{"id": 1000 + i, "embedding": arrays[i]} for i in range(25)] + with db.graph_batch(parallel_flush=False) as gb: + gb.create_vertices("Free", rows) + assert crossings["bridge"] == ["createVerticesJson"] * 3 # 10, 10, 5 + # rows without a vector keep the whole _BULK_CHUNK + crossings["bridge"].clear() + with db.graph_batch(parallel_flush=False) as gb: + gb.create_vertices("Free", [{"id": 2000 + i} for i in range(ROWS)]) + assert crossings["bridge"] == ["createVerticesJson"] + + +def test_vector_width_counts_the_vector_columns_of_the_first_row(db): + x = _x(3, 16) + arrays = _floats(x) + both = [{"a": 1, "v": arrays[0], "w": x[1], "d": x[2].astype(np.float64)}] + assert ( + tc.vector_width(both, set()) == 16 + 16 + ) # a float32 array and a Java float[] of any property; the float64 array is not one + assert tc.vector_width(both, {"d"}) == 16 + 16 + 16 # on a declared property it is + assert tc.vector_width([{"a": 1, "l": [1.0, 2.0]}], {"l"}) == 2 + assert tc.vector_width([{"a": 1, "l": [1.0, 2.0]}], set()) == 0 + assert tc.vector_width([], {"l"}) == 0 and tc.vector_width([None], {"l"}) == 0 + assert tc.vector_width([{"v": np.float32(1.5)}], set()) == 0 + + +def test_the_previous_chunks_rows_are_released_before_the_next_chunk_is_split( + db, crossings, monkeypatch +): + """Two chunks' float[] buffers and rest rows were alive at once: `split` still held the previous chunk's while the next was built.""" + import weakref + + class Rest(list): # a list that can be weakly referenced + pass + + alive = ( + [] + ) # for every call of the split: whether the rest rows of each earlier call are still alive + refs = [] + real = gbm.split_float_vectors + + def spy(*args, **kwargs): + alive.append([ref() is not None for ref in refs]) + out = real(*args, **kwargs) + rest = Rest(out[0]) + refs.append(weakref.ref(rest)) + return (rest,) + tuple(out[1:]) + + monkeypatch.setattr(gbm, "split_float_vectors", spy) + x = _x() + arrays = _floats(x) + rows = [{"id": i, "embedding": arrays[i]} for i in range(ROWS)] + with db.graph_batch(parallel_flush=False) as gb: + assert len(gb.create_vertices("Declared", rows)) == ROWS + assert alive == [[], [False], [False, False]] diff --git a/tests/test_ingest_integrity.py b/tests/test_ingest_integrity.py new file mode 100644 index 0000000..006d992 --- /dev/null +++ b/tests/test_ingest_integrity.py @@ -0,0 +1,988 @@ +"""Ingest integrity (ArcadeData/arcadedb-python#336): no ingest path may lose, duplicate or silently alter a row, and a test must be +able to catch it. + +``tests/ingest_integrity.py`` holds the one check. After every ingest call it compares the stored type with the rows that were sent: +the record count (read three ways), an order-independent digest of every column (read two ways), the key of every row exactly once, +and all of it again after the database is closed straight after the call and opened from disk. Both wheels run this file: the API is +the same, only the seams in ``test_native_ingest_integrity_faults.py`` and ``test_jvm_ingest_integrity_faults.py`` differ. + +* ``test_every_size_at_every_boundary`` loads each ingest path (``insert_many`` from a list and from a generator, ``insert_columns``, + ``insert_arrow``, ``insert_dataframe``, ``graph_batch().create_vertices``, ``create_record`` and ``command`` on the async executor, + ``append_samples``; each sequential and, where it exists, with ``parallel=True``) at 0, 1 and 2 rows and one either side of every + multiple of the chunk (7 rows) and of ``commit_every`` / ``batch_size`` (5 rows), and at 255, 256 and 257 rows (the first reply of + the native read path). The chunk sizes are made small so the boundaries are cheap; one load at the real 100,000 chunk is separate. +* the edge, thread, failure, async and ``close()`` tests below check the cases the issue lists, with the documented outcome exact. +* ``test_mutation_*`` run the same code with a rows-dropped (or duplicated, or altered, or uncommitted) fault in the path and assert that the check + reports it, so each family is shown able to fail. + +``wait_completion()`` can return while the executor still reports itself as processing (#210), so no test here asserts idleness: +they assert the stored data. +""" + +import dataclasses +import threading +import time +from typing import Any, Callable, Optional + +import numpy as np +import pandas as pd +import pyarrow as pa +import pytest + +import arcadedb +from arcadedb.exceptions import ArcadeDBError +from tests.ingest_integrity import ( + CHUNK, + COMMIT, + FOUND, + MUTANTS, + NO_COMMIT, + IntegrityError, + problems_for_edges, + problems_for_type, + verify, +) + +FIRST_REPLY = 256 # rows in the first reply of the native read path +MUTATION_SIZE = 25 + +SCHEMA = {"k": "int", "val": "float", "flag": "bool", "txt": "str"} +PROPS = {"k": "LONG", "val": "DOUBLE", "flag": "BOOLEAN", "txt": "STRING"} +TS_SCHEMA = {"host": "str", "val": "float"} + + +def ladder(units, extra=()): + sizes = {0, 1, 2, *extra} + for unit in units: + for k in (1, 2, 3): + sizes.update((k * unit - 1, k * unit, k * unit + 1)) + return sorted(sizes) + + +SIZES = ladder((CHUNK, COMMIT), extra=(FIRST_REPLY - 1, FIRST_REPLY, FIRST_REPLY + 1)) + + +def make_rows(n, start=0): + return [ + { + "k": i, + "val": (i * 37 % 1009) / 7.0 + i, + "flag": i % 3 == 0, + "txt": None if i % 7 == 3 else "row-%d-%s" % (i, "é" * (i % 4)), + } + for i in range(start, start + n) + ] + + +def make_ts_rows(n): + return [{"host": "h%d" % (i % 3), "val": float(i)} for i in range(n)] + + +def make_type(db, name, kind="DOCUMENT", buckets=None, unique=False): + ddl = f"CREATE {kind} TYPE {name}" + if buckets: + ddl += f" BUCKETS {buckets}" + db.command("sql", ddl) + for prop, declared in PROPS.items(): + db.command("sql", f"CREATE PROPERTY {name}.{prop} {declared}") + if unique: + db.command("sql", f"CREATE INDEX ON {name} (k) UNIQUE") + + +def make_ts_type(db, name): + db.command( + "sql", + f"CREATE TIMESERIES TYPE {name} TIMESTAMP ts TAGS (host STRING) FIELDS (val DOUBLE) SHARDS 2", + ) + + +# -- the ingest paths --------------------------------------------------------------------------------------------------------- + + +def _numpy_columns(rows): + return { + "k": np.array([r["k"] for r in rows], dtype=np.int64), + "val": np.array([r["val"] for r in rows], dtype=np.float64), + "flag": np.array([r["flag"] for r in rows], dtype=np.bool_), + "txt": [r["txt"] for r in rows], + } + + +def _list_columns(rows): + return {n: [r[n] for r in rows] for n in SCHEMA} + + +ARROW_SCHEMA = pa.schema( + [ + ("k", pa.int64()), + ("val", pa.float64()), + ("flag", pa.bool_()), + ("txt", pa.string()), + ] +) + + +def _arrow(rows): + return pa.Table.from_pylist(list(rows), schema=ARROW_SCHEMA) + + +def _frame(rows): + return pd.DataFrame( + { + "k": np.array([r["k"] for r in rows], dtype=np.int64), + "val": np.array([r["val"] for r in rows], dtype=np.float64), + "flag": np.array([r["flag"] for r in rows], dtype=np.bool_), + "txt": pd.array([r["txt"] for r in rows], dtype="string"), + } + ) + + +def _doc_of(db, name, row): + doc = db.new_document(name) + for key, value in row.items(): + if value is not None: + doc.set(key, value) + return doc + + +@dataclasses.dataclass +class Ctx: + """The async executor of a database (there is one per database) and what its executor-wide handler was told.""" + + ex: Any + errors: list + + +def _executor(level=None): + def setup(db): + ex = db.async_executor() + if level: + ex.set_parallel_level(level) + ex.set_commit_every(COMMIT) + errors = [] + ex.on_error(errors.append) + return Ctx(ex, errors) + + return setup + + +def _no_setup(db): + return None + + +def _teardown(ctx): + if ctx is not None: + ctx.ex.wait_completion() + ctx.ex.close() + + +def _ing_many(ctx, db, name, rows): + return db.insert_many(name, list(rows), commit_every=COMMIT) + + +def _ing_many_gen(ctx, db, name, rows): + return db.insert_many(name, (r for r in rows), commit_every=COMMIT) + + +def _ing_many_par(ctx, db, name, rows): + return db.insert_many(name, list(rows), parallel=True) + + +def _ing_cols(ctx, db, name, rows): + return db.insert_columns(name, _numpy_columns(rows), commit_every=COMMIT) + + +def _ing_cols_par(ctx, db, name, rows): + return db.insert_columns(name, _numpy_columns(rows), parallel=True) + + +def _ing_arrow(ctx, db, name, rows): + return db.insert_arrow(name, _arrow(rows), batch_size=COMMIT) + + +def _ing_arrow_par(ctx, db, name, rows): + return db.insert_arrow(name, _arrow(rows), batch_size=COMMIT, parallel=True) + + +def _ing_df(ctx, db, name, rows): + return db.insert_dataframe(name, _frame(rows), batch_size=COMMIT) + + +def _ing_df_par(ctx, db, name, rows): + return db.insert_dataframe(name, _frame(rows), batch_size=COMMIT, parallel=True) + + +def _ing_async_create(ctx, db, name, rows): + for row in rows: + ctx.ex.create_record(_doc_of(db, name, row)) + ctx.ex.wait_completion() + return len(rows) + + +def _ing_async_command(ctx, db, name, rows): + head = "INSERT INTO " + name + with_text = head + " SET k = :k, val = :val, flag = :flag, txt = :txt" + without_text = head + " SET k = :k, val = :val, flag = :flag" + for row in rows: + if row["txt"] is None: + ctx.ex.command( + "sql", without_text, k=row["k"], val=row["val"], flag=row["flag"] + ) + else: + ctx.ex.command("sql", with_text, **row) + ctx.ex.wait_completion() + return len(rows) + + +def _ing_vertices(ctx, db, name, rows): + with db.graph_batch(commit_every=CHUNK, batch_size=CHUNK) as batch: + rids = batch.create_vertices(name, [dict(r) for r in rows]) + if len(rids) != len(rows) or len(set(rids)) != len(rids): + raise AssertionError( + f"create_vertices returned {len(rids)} RIDs ({len(set(rids))} distinct) for {len(rows)} rows" + ) + return len(rids) + + +def _ing_timeseries(ctx, db, name, rows): + n = len(rows) + ts = (np.arange(n, dtype=np.int64) + 1) * 1000 + ctx.ex.append_samples( + name, + ts, + [r["host"] for r in rows], + np.array([r["val"] for r in rows], dtype=np.float64), + ) + ctx.ex.wait_completion() + return n + + +@dataclasses.dataclass(frozen=True) +class Family: + id: str + create: Callable + ingest: Callable + setup: Callable = _no_setup + schema: Optional[dict] = None + rows: Callable = make_rows + key: str = "k" + min_n: int = 0 + sync_tx: bool = ( + False # joins a transaction the caller opened (the durability mutant needs one) + ) + + def __post_init__(self): + if self.schema is None: + object.__setattr__(self, "schema", SCHEMA) + + +def _doc(buckets=None): + return lambda db, name: make_type(db, name, buckets=buckets) + + +def _vertex(db, name): + make_type(db, name, kind="VERTEX") + + +FAMILIES = [ + Family("many", _doc(), _ing_many, sync_tx=True), + Family("many_gen", _doc(), _ing_many_gen, sync_tx=True), + Family("many_par", _doc(4), _ing_many_par, setup=_executor()), + Family("cols", _doc(), _ing_cols, sync_tx=True), + Family("cols_par", _doc(4), _ing_cols_par, setup=_executor()), + Family("arrow", _doc(), _ing_arrow, sync_tx=True), + Family("arrow_par", _doc(4), _ing_arrow_par, setup=_executor()), + Family("arrow_vertex", _vertex, _ing_arrow), + Family("df", _doc(), _ing_df, sync_tx=True), + Family("df_par", _doc(4), _ing_df_par, setup=_executor()), + Family("async_create", _doc(), _ing_async_create, setup=_executor(1)), + Family("async_create_par", _doc(4), _ing_async_create, setup=_executor(4)), + Family("async_command", _doc(), _ing_async_command, setup=_executor(1)), + Family("vertices", _vertex, _ing_vertices), + Family( + "timeseries", + lambda db, name: make_ts_type(db, name), + _ing_timeseries, + setup=_executor(), + schema=TS_SCHEMA, + rows=make_ts_rows, + key="val", + min_n=1, + ), +] +BY_ID = {f.id: f for f in FAMILIES} + + +def run_cases(store, fam, sizes, mutant=None): + """Load one type per size through ``fam``, check each after the call, close and reopen, check each again. + + ``mutant`` names a fault: one of ``MUTANTS`` rewrites the rows the path is given (the check still compares with the rows that + were generated), ``NO_COMMIT`` loads inside a transaction nobody commits. Raises ``IntegrityError`` with every problem found. + """ + problems, cases = [], [] + db = store.db + ctx = fam.setup(db) + try: + for n in sizes: + if n < fam.min_n: + continue + name = f"{fam.id}_{n}" + rows = fam.rows(n) + sent = MUTANTS[mutant](rows) if mutant in MUTANTS else rows + fam.create(db, name) + if mutant == NO_COMMIT: + db.begin() + try: + got = fam.ingest(ctx, db, name, sent) + if got != len(sent): + problems.append( + f"{FOUND} n={n}: the call returned {got}, {len(sent)} rows were given" + ) + except ( + Exception + ) as exc: # noqa: BLE001 - reported with the size, the other sizes still run + problems.append(f"RAISED n={n}: {type(exc).__name__}: {exc}") + if db.is_transaction_active() and mutant != NO_COMMIT: + db.rollback() + if ctx is not None and ctx.errors: + problems.append( + f"{FOUND} n={n}: the executor's error handler was told {len(ctx.errors)} time(s): {ctx.errors[:2]}" + ) + ctx.errors.clear() + problems += problems_for_type( + db, name, rows, fam.schema, fam.key, f"n={n} after the call" + ) + cases.append((name, n, rows)) + finally: + _teardown(ctx) + store.reopen() + for name, n, rows in cases: + problems += problems_for_type( + store.db, name, rows, fam.schema, fam.key, f"n={n} after reopen" + ) + if problems: + raise IntegrityError(f"{fam.id}: " + "\n".join(problems)) + + +# -- sizes at every boundary -------------------------------------------------------------------------------------------------- + + +@pytest.mark.parametrize("fam", FAMILIES, ids=lambda f: f.id) +def test_every_size_at_every_boundary(store, small_chunks, fam): + run_cases(store, fam, SIZES) + + +def test_one_load_across_the_real_100000_chunk(store): + db = store.db + make_type(db, "Big", kind="VERTEX") + n = 100_001 + rows = make_rows(n) + with db.graph_batch() as batch: + rids = batch.create_vertices("Big", rows) + assert len(rids) == n and len(set(rids)) == n + problems = problems_for_type(db, "Big", rows, SCHEMA, "k", "after the call") + store.reopen() + problems += problems_for_type(store.db, "Big", rows, SCHEMA, "k", "after reopen") + assert not problems, "\n".join(problems) + + +# -- reading back at every batch size (the read path found the loss of #331) ---------------------------------------------- + + +@pytest.mark.parametrize("batch_size", [1, 2, 7, 255, 256, 257, 600, 25_000]) +def test_reading_back_every_row_at_every_batch_size(store, batch_size): + db = store.db + make_type(db, "Wide") + rows = make_rows(600) + assert db.insert_many("Wide", rows) == 600 + names = list(SCHEMA) + select = "SELECT FROM Wide" + cols = db.query("sql", select).to_columns(batch_size=batch_size, columns=names) + from_columns = [dict(zip(names, vals)) for vals in zip(*(cols[n] for n in names))] + table = db.query("sql", select).to_arrow(batch_size=batch_size, columns=names) + problems = verify( + f"batch_size={batch_size}", + rows, + SCHEMA, + "k", + {"to_columns rows": len(from_columns), "to_arrow rows": table.num_rows}, + {"to_columns()": from_columns, "to_arrow()": table.to_pylist()}, + ) + assert not problems, "\n".join(problems) + + +# -- edges: count and endpoint pairs ------------------------------------------------------------------------------------------ + +EDGE_SIZES = ladder((CHUNK, COMMIT)) +VERTICES = 11 + + +def _endpoints(m): + return [(i % VERTICES, (3 * i + 1) % VERTICES) for i in range(m)] + + +def _load_edges(db, rids, etype, pairs, weights): + batch = db.graph_batch(commit_every=CHUNK, batch_size=CHUNK) + try: + batch.new_edges( + [rids[a] for a, _ in pairs], + etype, + [rids[b] for _, b in pairs], + properties=None if weights is None else [{"w": w} for w in weights], + ) + finally: + batch.close() + + +def _vertex_rids(db): + make_type(db, "Node", kind="VERTEX") + with db.graph_batch(commit_every=CHUNK, batch_size=CHUNK) as batch: + return batch.create_vertices("Node", make_rows(VERTICES)) + + +@pytest.mark.parametrize("with_weights", [False, True], ids=["plain", "weighted"]) +def test_new_edges_store_every_edge_between_its_endpoints( + store, small_chunks, with_weights +): + db = store.db + rids = _vertex_rids(db) + problems, cases = [], [] + for m in EDGE_SIZES: + etype = f"Link{m}" + db.command("sql", f"CREATE EDGE TYPE {etype}") + pairs = _endpoints(m) + weights = [i + 0.25 for i in range(m)] if with_weights else None + _load_edges(db, rids, etype, pairs, weights) + want = [ + {"a": a, "b": b, "w": None if weights is None else weights[i]} + for i, (a, b) in enumerate(pairs) + ] + problems += problems_for_edges(db, etype, want, f"m={m} after close()") + cases.append((etype, m, want)) + store.reopen() + for etype, m, want in cases: + problems += problems_for_edges(store.db, etype, want, f"m={m} after reopen") + assert not problems, "\n".join(problems) + + +@pytest.mark.parametrize( + "fault", ["drop_the_last_edge", "swap_two_destinations", "store_one_edge_twice"] +) +def test_mutation_edges(store, small_chunks, fault): + db = store.db + rids = _vertex_rids(db) + db.command("sql", "CREATE EDGE TYPE Mut") + m = MUTATION_SIZE + pairs = _endpoints(m) + weights = [i + 0.25 for i in range(m)] + sent = list(pairs) + sent_weights = list(weights) + if fault == "drop_the_last_edge": + sent, sent_weights = sent[:-1], sent_weights[:-1] + elif fault == "store_one_edge_twice": + sent.append(sent[4]) + sent_weights.append(sent_weights[4]) + else: + (a1, b1), (a2, b2) = sent[1], sent[8] + sent[1], sent[8] = (a1, b2), (a2, b1) + _load_edges(db, rids, "Mut", sent, sent_weights) + want = [{"a": a, "b": b, "w": weights[i]} for i, (a, b) in enumerate(pairs)] + problems = problems_for_edges(db, "Mut", want, "after close()") + assert problems and all(FOUND in p for p in problems), problems + + +# -- the mutation check: every family, every fault --------------------------------------------------------------------------- + + +def _mutation_cases(): + for fam in FAMILIES: + for mutant in MUTANTS: + yield pytest.param(fam, mutant, id=f"{fam.id}-{mutant}") + if fam.sync_tx: + yield pytest.param(fam, NO_COMMIT, id=f"{fam.id}-{NO_COMMIT}") + + +@pytest.mark.parametrize("fam, mutant", list(_mutation_cases())) +def test_mutation_the_check_reports_the_fault(store, small_chunks, fam, mutant): + with pytest.raises(IntegrityError) as found: + run_cases(store, fam, [MUTATION_SIZE], mutant=mutant) + text = str(found.value) + assert FOUND in text, text + if mutant == NO_COMMIT: + # the rows are visible in this session, so only the reopen can tell + assert "after the call" not in text, text + assert "after reopen" in text, text + else: + assert "after the call" in text and "after reopen" in text, text + + +def _chunk_seam(): + """The function that stores one chunk of insert_many on this wheel: (class, name, arguments before the chunk).""" + if arcadedb.__backend__ == "native": + from arcadedb._native import database + + return database.NativeDatabase, "_insert_chunk" + from arcadedb import core + + return core.Database, "_insert_many_chunk" + + +@pytest.mark.parametrize( + "fault", + ["drop_the_last_row_of_a_chunk", "skip_a_chunk", "store_a_row_twice"], +) +def test_mutation_inside_insert_many(store, small_chunks, monkeypatch, fault): + owner, name = _chunk_seam() + real = getattr(owner, name) + seen = [] + + def faulty(self, type_name, *args): + *lead, chunk = args + seen.append(len(chunk)) + if len(seen) == 2: + if fault == "skip_a_chunk": + return None + if fault == "drop_the_last_row_of_a_chunk": + chunk = chunk[:-1] + else: + chunk = chunk + [chunk[-1]] + return real(self, type_name, *lead, chunk) + + monkeypatch.setattr(owner, name, faulty) + with pytest.raises(IntegrityError) as found: + run_cases(store, BY_ID["many"], [MUTATION_SIZE]) + assert FOUND in str(found.value), found.value + assert len(seen) >= 3, f"the fault was not in the path: {seen}" + + +def test_mutation_an_async_load_that_returns_before_the_wait(store): + """The executor is parked inside a callback, so the queued records cannot have run when the check looks.""" + db = store.db + make_type(db, "Early") + rows = make_rows(MUTATION_SIZE) + ex = db.async_executor().set_parallel_level(1).set_commit_every(COMMIT) + started, release = threading.Event(), threading.Event() + + def park(_row): + started.set() + release.wait(60) + + ex.query("sql", "SELECT 1 AS one", park) + assert started.wait(60), "the parking callback never ran" + try: + for row in rows: + ex.create_record(_doc_of(db, "Early", row)) + # no wait_completion(): this is the fault + problems = problems_for_type(db, "Early", rows, SCHEMA, "k", "before the wait") + finally: + release.set() + ex.wait_completion() + ex.close() + assert problems and all(FOUND in p for p in problems), problems + # and once the wait is done the same check passes, so the fault was the missing wait + assert not problems_for_type(db, "Early", rows, SCHEMA, "k", "after the wait") + + +# -- several writer threads --------------------------------------------------------------------------------------------------- + + +def _run_threads(jobs): + errors = [] + + def run(job): + try: + job() + except BaseException as exc: # noqa: BLE001 - handed to the test thread + errors.append(exc) + + threads = [threading.Thread(target=run, args=(job,)) for job in jobs] + for t in threads: + t.start() + for t in threads: + t.join(180) + assert not [t for t in threads if t.is_alive()], "a writer thread never finished" + return errors + + +class Reader: + """Counts the types in a loop while writers run; the counts of one type may only grow.""" + + def __init__(self, db, names): + self.db = db + self.series = {n: [] for n in names} + self.stop = threading.Event() + self.error = None + self.thread = threading.Thread(target=self.run) + + def count(self, name): + sql = "SELECT count(*) AS n FROM " + name # nosec B608 + return int(self.db.query("sql", sql).to_list()[0]["n"]) + + def run(self): + try: + while not self.stop.is_set(): + for name, values in self.series.items(): + values.append(self.count(name)) + time.sleep(0.002) + except BaseException as exc: # noqa: BLE001 - handed to the test thread + self.error = exc + + def __enter__(self): + self.thread.start() + return self + + def __exit__(self, *exc_info): + self.stop.set() + self.thread.join(60) + + def problems(self, totals): + found = [] + if self.error is not None: + found.append(f"RAISED the reader thread: {self.error!r}") + for name, values in self.series.items(): + if any(b < a for a, b in zip(values, values[1:])): + found.append(f"{FOUND} the count of {name} went down: {values}") + if values and values[-1] > totals[name]: + found.append( + f"{FOUND} the count of {name} reached {values[-1]}, above the {totals[name]} sent" + ) + if self.count(name) != totals[name]: + found.append( + f"{FOUND} the final count of {name} is {self.count(name)}, {totals[name]} were sent" + ) + return found + + +def test_writer_threads_on_one_type(store, small_chunks): + db = store.db + make_type(db, "OneType", buckets=4) + writers, per = 4, 60 + parts = [make_rows(per, start=w * per) for w in range(writers)] + + def job(rows): + return lambda: db.run_in_transaction( + lambda: db.insert_many("OneType", rows), retries=200, backoff_s=0.002 + ) + + with Reader(db, ["OneType"]) as reader: + errors = _run_threads([job(rows) for rows in parts]) + assert not errors, errors + everything = [r for rows in parts for r in rows] + problems = reader.problems({"OneType": len(everything)}) + problems += problems_for_type(db, "OneType", everything, SCHEMA, "k", "after join") + store.reopen() + problems += problems_for_type( + store.db, "OneType", everything, SCHEMA, "k", "after reopen" + ) + assert not problems, "\n".join(problems) + + +def test_writer_threads_on_different_types_and_paths(store, small_chunks): + db = store.db + paths = [BY_ID["many"], BY_ID["cols"], BY_ID["arrow"], BY_ID["df"]] + names = [f"Thread{i}" for i in range(len(paths))] + sent = {name: make_rows(90, start=1000 * i) for i, name in enumerate(names)} + for name in names: + make_type(db, name) + + def job(fam, name): + return lambda: fam.ingest(None, db, name, sent[name]) + + with Reader(db, names) as reader: + errors = _run_threads([job(f, n) for f, n in zip(paths, names)]) + assert not errors, errors + problems = reader.problems({n: len(sent[n]) for n in names}) + for name in names: + problems += problems_for_type(db, name, sent[name], SCHEMA, "k", "after join") + store.reopen() + for name in names: + problems += problems_for_type( + store.db, name, sent[name], SCHEMA, "k", "after reopen" + ) + assert not problems, "\n".join(problems) + + +# -- a bad row in the middle of a batch --------------------------------------------------------------------------------------- + +BAD_ROWS = { + "duplicate_key": lambda rows, p: {**rows[p], "k": rows[0]["k"]}, + "wrong_declared_type": lambda rows, p: {**rows[p], "val": "not a number"}, + "refused_value": lambda rows, p: {**rows[p], "extra": 2**70}, +} +POSITIONS = [1, 4, 5, 7, 13] # 14 rows, commit_every 5: batches [0-4] [5-9] [10-13] + +FAIL_CALLS = { + "insert_many": lambda db, name, sent: db.insert_many( + name, sent, commit_every=COMMIT + ), + "insert_columns": lambda db, name, sent: db.insert_columns( + name, _list_columns(sent), commit_every=COMMIT + ), +} + + +def _bad_cases(): + for path in FAIL_CALLS: + for bad in BAD_ROWS: + if bad == "refused_value" and path != "insert_many": + continue # the row's extra property has no column to go into + for p in POSITIONS: + yield pytest.param(path, bad, p, id=f"{path}-{bad}-row{p}") + + +@pytest.mark.parametrize("path, bad, p", list(_bad_cases())) +def test_a_bad_row_keeps_the_committed_batches_and_nothing_else( + store, small_chunks, path, bad, p +): + """Documented: rows already committed stay committed, the batch that holds the bad row is rolled back, the error is raised.""" + db = store.db + make_type(db, "Bad", unique=True) + rows = make_rows(14) + sent = list(rows) + sent[p] = BAD_ROWS[bad](rows, p) + with pytest.raises((ArcadeDBError, OverflowError)): + FAIL_CALLS[path](db, "Bad", sent) + assert not db.is_transaction_active() + kept = rows[: (p // COMMIT) * COMMIT] + problems = problems_for_type(db, "Bad", kept, SCHEMA, "k", "after the failed call") + store.reopen() + problems += problems_for_type(store.db, "Bad", kept, SCHEMA, "k", "after reopen") + assert not problems, "\n".join(problems) + + +ARROW_CALLS = { + "insert_arrow": lambda db, name, rows: db.insert_arrow( + name, _arrow(rows), batch_size=COMMIT + ), + "insert_dataframe": lambda db, name, rows: db.insert_dataframe( + name, _frame(rows), batch_size=COMMIT + ), +} + + +@pytest.mark.parametrize("path", list(ARROW_CALLS)) +@pytest.mark.parametrize("p", POSITIONS) +def test_arrow_and_dataframe_batches_before_a_duplicate_stay_committed(store, path, p): + db = store.db + make_type(db, "ArrowDup", unique=True) + rows = make_rows(14) + sent = list(rows) + sent[p] = BAD_ROWS["duplicate_key"](rows, p) + with pytest.raises(ArcadeDBError) as err: + ARROW_CALLS[path](db, "ArrowDup", sent) + committed = (p // COMMIT) * COMMIT + if committed: + assert ( + f"the {committed} row(s) of the batches before this one were committed" + in str(err.value) + ), err.value + else: + assert "batches before this one" not in str(err.value), err.value + kept = rows[:committed] + problems = problems_for_type(db, "ArrowDup", kept, SCHEMA, "k", "after the call") + store.reopen() + problems += problems_for_type( + store.db, "ArrowDup", kept, SCHEMA, "k", "after reopen" + ) + assert not problems, "\n".join(problems) + + +@pytest.mark.parametrize("path", list(ARROW_CALLS)) +def test_a_column_the_declared_type_refuses_stores_nothing(store, path): + """Documented: a TypeError before any row is written.""" + db = store.db + make_type(db, "Refused") + rows = make_rows(12) + text_rows = [{**r, "val": str(r["val"])} for r in rows] + schema = pa.schema( + [ + ("k", pa.int64()), + ("val", pa.string()), + ("flag", pa.bool_()), + ("txt", pa.string()), + ] + ) + if path == "insert_arrow": + with pytest.raises(TypeError): + db.insert_arrow( + "Refused", pa.Table.from_pylist(text_rows, schema=schema), batch_size=3 + ) + else: + frame = _frame(rows) + frame["val"] = pd.array([r["val"] for r in text_rows], dtype="string") + with pytest.raises(TypeError): + db.insert_dataframe("Refused", frame, batch_size=3) + problems = problems_for_type(db, "Refused", [], SCHEMA, "k", "after the call") + store.reopen() + problems += problems_for_type(store.db, "Refused", [], SCHEMA, "k", "after reopen") + assert not problems, "\n".join(problems) + + +@pytest.mark.parametrize("p", POSITIONS) +def test_create_vertices_chunks_before_a_duplicate_stay_stored(store, small_chunks, p): + db = store.db + make_type(db, "BadV", kind="VERTEX", unique=True) + rows = make_rows(14) + sent = [dict(r) for r in rows] + sent[p] = BAD_ROWS["duplicate_key"](rows, p) + batch = db.graph_batch(commit_every=CHUNK, batch_size=CHUNK) + try: + with pytest.raises(ArcadeDBError): + batch.create_vertices("BadV", sent) + finally: + batch.close() + assert not db.is_transaction_active() + kept = rows[: (p // CHUNK) * CHUNK] + problems = problems_for_type(db, "BadV", kept, SCHEMA, "k", "after the call") + store.reopen() + problems += problems_for_type(store.db, "BadV", kept, SCHEMA, "k", "after reopen") + assert not problems, "\n".join(problems) + + +@pytest.mark.parametrize("path", ["insert_many", "insert_columns"]) +def test_a_parallel_duplicate_is_reported_and_nothing_is_stored_twice( + store, small_chunks, path +): + """Documented for parallel=True: the failure is raised after the load, records other than the failed ones may be stored.""" + db = store.db + make_type(db, "ParDup", buckets=4, unique=True) + db.async_executor().set_commit_every(COMMIT) + rows = make_rows(40) + sent = list(rows) + sent[30] = BAD_ROWS["duplicate_key"](rows, 30) + with pytest.raises(ArcadeDBError, match="failed record"): + if path == "insert_many": + db.insert_many("ParDup", sent, parallel=True) + else: + db.insert_columns("ParDup", _numpy_columns(sent), parallel=True) + + def keys(database): + stored = database.query("sql", "SELECT k FROM ParDup").to_list() + return [int(r["k"]) for r in stored] + + before = keys(db) + assert len(before) == len(set(before)), "a key was stored twice" + assert set(before) <= {r["k"] for r in rows}, "a key was stored that was never sent" + assert db.count_type("ParDup") == len(before) + store.reopen() + after = keys(store.db) + assert sorted(after) == sorted(before), "the stored keys changed across a reopen" + + +# -- async: a rejected row and a failing callback must reach the caller ------------------------------------------------------ + + +def test_async_rejected_rows_reach_their_error_callbacks(store): + db = store.db + make_type(db, "AsyncDup", unique=True) + rows = make_rows(12) + sent = list(rows) + sent[5] = {**rows[5], "k": rows[0]["k"]} + sent[9] = {**rows[9], "k": rows[2]["k"]} + rejected, reported = [], [] + ex = db.async_executor().set_parallel_level(1).set_commit_every(1) + ex.on_error(reported.append) + for i, row in enumerate(sent): + ex.create_record( + _doc_of(db, "AsyncDup", row), + error_callback=lambda _e, i=i: rejected.append(i), + ) + ex.wait_completion() + ex.close() + assert sorted(rejected) == [5, 9] + assert len(reported) == 2, reported + kept = [r for i, r in enumerate(sent) if i not in (5, 9)] + problems = problems_for_type(db, "AsyncDup", kept, SCHEMA, "k", "after wait") + store.reopen() + problems += problems_for_type( + store.db, "AsyncDup", kept, SCHEMA, "k", "after reopen" + ) + assert not problems, "\n".join(problems) + + +@pytest.mark.filterwarnings("ignore::pytest.PytestUnraisableExceptionWarning") +def test_a_callback_that_raises_loses_no_record_without_a_report(store): + """A record is stored, or its own error callback was told: nothing is dropped silently when a per-record callback raises.""" + db = store.db + make_type(db, "CbRaise") + rows = make_rows(20) + raised, reported = [], [] + + def callback(i): + def run(_record): + if i % 4 == 1: + raised.append(i) + raise RuntimeError(f"callback {i}") + + return run + + ex = db.async_executor().set_parallel_level(1).set_commit_every(3) + for i, row in enumerate(rows): + ex.create_record( + _doc_of(db, "CbRaise", row), + callback=callback(i), + error_callback=lambda _e, i=i: reported.append(i), + ) + ex.wait_completion(60_000) + ex.close() + assert sorted(raised) == [1, 5, 9, 13, 17], "the callbacks did not all run" + kept = [r for i, r in enumerate(rows) if i not in set(reported)] + problems = problems_for_type( + db, + "CbRaise", + kept, + SCHEMA, + "k", + f"after wait, errors reported for {sorted(reported)}", + ) + store.reopen() + problems += problems_for_type( + store.db, "CbRaise", kept, SCHEMA, "k", "after reopen" + ) + assert not problems, "\n".join(problems) + + +# -- close() straight after the last call ------------------------------------------------------------------------------------- + + +@pytest.mark.parametrize("how", ["executor_close", "database_close"]) +def test_close_straight_after_async_submissions_stores_all_or_raises(store, how): + """Either every record is stored or an error is raised: never a silent partial.""" + db = store.db + make_type(db, "Straight") + n = 200 + rows = make_rows(n) + ex = db.async_executor().set_parallel_level(1).set_commit_every(10) + for row in rows: + ex.create_record(_doc_of(db, "Straight", row)) + try: + if how == "executor_close": + ex.close() + db.close() + except Exception: # noqa: BLE001 - an error was raised: not a silent partial + return + store.db = arcadedb.open_database(store.path) + problems = problems_for_type( + store.db, "Straight", rows, SCHEMA, "k", "after reopen" + ) + assert not problems, "\n".join(problems) + + +def test_close_straight_after_the_last_edges_stores_them_all(store, small_chunks): + db = store.db + rids = _vertex_rids(db) + db.command("sql", "CREATE EDGE TYPE Late") + pairs = _endpoints(40) + weights = [i + 0.25 for i in range(40)] + batch = db.graph_batch(commit_every=CHUNK, batch_size=CHUNK) + batch.new_edges( + [rids[a] for a, _ in pairs], + "Late", + [rids[b] for _, b in pairs], + properties=[{"w": w} for w in weights], + ) + batch.close() + store.reopen() + want = [{"a": a, "b": b, "w": weights[i]} for i, (a, b) in enumerate(pairs)] + problems = problems_for_edges(store.db, "Late", want, "after reopen") + assert not problems, "\n".join(problems) diff --git a/tests/test_insert_columns.py b/tests/test_insert_columns.py index 34ca2ad..482d122 100644 --- a/tests/test_insert_columns.py +++ b/tests/test_insert_columns.py @@ -79,7 +79,7 @@ def test_nulls_in_a_sequence_column_are_nulls(self, temp_db): assert _count(temp_db, "Col", "WHERE label IS NULL") == 2 assert _count(temp_db, "Col", "WHERE label = 'c'") == 1 - def test_a_numpy_string_array_and_an_object_array_with_none_cross_per_element( + def test_a_numpy_string_array_and_an_object_array_with_none_are_stored( self, temp_db ): _make_type(temp_db) diff --git a/tests/test_insert_columns_string_blob.py b/tests/test_insert_columns_string_blob.py new file mode 100644 index 0000000..40e9b30 --- /dev/null +++ b/tests/test_insert_columns_string_blob.py @@ -0,0 +1,183 @@ +"""The JVM wheel sends a column of str and None as one UTF-8 blob plus int32 offsets, not as one JPype conversion per element. + +`_column_to_java` used to convert every element of a string column (`convert_python_to_java`, then an `Object[]` of Java Strings built +element by element). Now a column of nothing but `str` and `None` is one `byte[]`, one `int[]` of row + 1 offsets and, when there is +a null, one `boolean[]`, wrapped in the bridge's `StringColumn`, which `DocumentBatcher.insertColumns` decodes row by row as it builds +each document. The blob crosses through `bytes` into `JArray(JByte)`, a copy; no JPype view of Python memory is relied on. A mixed +column, an empty column, a column with a lone surrogate (no UTF-8 form) and a column too large for an int offset keep the +per-element path. These tests spy on which path ran (they fail on the code before) and compare the stored table with the one the +per-element path stores for the same columns. +""" + +import jpype +import pytest + +np = pytest.importorskip("numpy") + +from arcadedb import core # noqa: E402 + +TEXTS = [ + "plain", + "", + "emoji \U0001f600 and \U00010000 and \U0010ffff", + "embedded\x00nul", + "\x00", + "line
separator
", + "中文 한글 ퟿ é", + "x" * 5_000, +] + + +def _string_column_class(): + return jpype.JClass("com.arcadedb.python.StringColumn") + + +def _make_type(db, name): + db.command("sql", f"CREATE DOCUMENT TYPE {name}") + db.command("sql", f"CREATE PROPERTY {name}.id LONG") + db.command("sql", f"CREATE PROPERTY {name}.label STRING") + + +def _stored(db, name): + rows = db.query("sql", f"SELECT FROM {name} ORDER BY id").to_list() # nosec B608 + return [(r["id"], r.get("label")) for r in rows] + + +@pytest.mark.parametrize( + "values", + [ + ["a", "bb", ""], + ["a", None, "c"], + ("tuple", "of", "str"), + np.array(["x", "yy"]), + np.array(["x", None, "z"], dtype=object), + TEXTS, + ], + ids=["list", "list_with_none", "tuple", "numpy_str", "object_array", "edge_texts"], +) +def test_a_column_of_str_and_none_is_one_string_column(values): + array, n = core._column_to_java("label", values) + assert n == len(values) + assert isinstance(array, _string_column_class()) + + +@pytest.mark.parametrize( + "values", + [ + ["a", 1], + ["a", b"bytes"], + ["a", 1.5, None], + [1, 2, 3], + [None, None], + [], + ], + ids=[ + "str_int", + "str_bytes", + "str_float", + "ints", + "only_none", + "empty", + ], +) +def test_any_other_column_keeps_the_per_element_path(values): + array, n = core._column_to_java("label", values) + assert n == len(values) + assert not isinstance(array, _string_column_class()) + + +def test_a_string_column_makes_no_per_element_conversion(temp_db, monkeypatch): + _make_type(temp_db, "S") + calls = [] + real = core.convert_python_to_java + monkeypatch.setattr( + core, "convert_python_to_java", lambda v: calls.append(v) or real(v) + ) + n = 2_000 + label = [TEXTS[i % len(TEXTS)] if i % 7 else None for i in range(n)] + temp_db.insert_columns("S", {"id": np.arange(n, dtype=np.int64), "label": label}) + assert calls == [], "a string column was converted element by element" + assert _stored(temp_db, "S") == [(i, label[i]) for i in range(n)] + + +@pytest.mark.parametrize("parallel", [False, True], ids=["sync", "parallel"]) +def test_the_blob_stores_the_table_the_per_element_path_stores( + temp_db, monkeypatch, parallel +): + n = 4_000 + label = [ + None if i % 5 == 3 else TEXTS[i % len(TEXTS)] + str(i % 3) for i in range(n) + ] + cols = {"id": np.arange(n, dtype=np.int64), "label": label} + _make_type(temp_db, "Blob") + _make_type(temp_db, "Ref") + temp_db.insert_columns("Blob", cols, parallel=parallel) + with monkeypatch.context() as mp: + mp.setattr(core, "_string_column", lambda values: None) # the path before + array, _n = core._column_to_java("label", label) + assert not isinstance(array, _string_column_class()) + temp_db.insert_columns("Ref", cols, parallel=parallel) + assert _stored(temp_db, "Blob") == _stored(temp_db, "Ref") + assert _stored(temp_db, "Blob") == [(i, label[i]) for i in range(n)] + + +@pytest.mark.parametrize("position", [0, 500, 999], ids=["first", "middle", "last"]) +def test_a_lone_surrogate_keeps_the_column_off_the_blob_and_is_refused_as_before( + temp_db, position +): + # a lone surrogate has no UTF-8 form: the blob builder gives up (it never raises), and the per-element path, which is what ran + # before, refuses the column (JPype cannot encode the str) before anything is written. The refusal is the contract of every bulk + # entry point: a lone surrogate raises instead of becoming a question mark (test_bulk_json_paths.py). + values = [f"row {i} \U0001f600" if i % 3 else f"row {i}" for i in range(1_000)] + values[position] = "lone \ud800 surrogate" + assert core._string_column(values) is None + with pytest.raises(UnicodeEncodeError): + core._column_to_java("label", values) + _make_type(temp_db, "S") + with pytest.raises(UnicodeEncodeError): + temp_db.insert_columns( + "S", {"id": np.arange(1_000, dtype=np.int64), "label": values} + ) + count = temp_db.query("sql", "SELECT count(*) AS c FROM S").first().get("c") + assert count == 0 + + +def test_a_non_bmp_character_still_takes_the_blob(): + # U+1F600 and U+10000 are one code point each and a surrogate PAIR in Java, but valid UTF-8: the blob carries them + values = ["a", "\U0001f600", "\U00010000\U0010ffff", None, "\ud7ff\ue000"] + assert core._string_column(values) is not None + array, n = core._column_to_java("label", values) + assert n == 5 and isinstance(array, _string_column_class()) + + +@pytest.mark.parametrize( + "blob, offsets, nulls", + [ + (b"ab", [1, 2], None), + (b"ab", [0, 2, 1], None), + (b"ab", [0, 3], None), + (b"ab", [0, 1, 2], [False]), + (b"ab", [], None), + ], + ids=[ + "first_not_zero", + "decreasing", + "past_the_end", + "short_null_flags", + "no_offsets", + ], +) +def test_the_java_side_refuses_a_malformed_column(blob, offsets, nulls): + with pytest.raises(Exception): # noqa: B017 - a Java IllegalArgumentException + _string_column_class()( + jpype.JArray(jpype.JByte)(blob), + jpype.JArray(jpype.JInt)(offsets), + None if nulls is None else jpype.JArray(jpype.JBoolean)(nulls), + ) + + +def test_a_column_beyond_an_int_offset_keeps_the_per_element_path(monkeypatch): + monkeypatch.setattr(core, "_JAVA_INT_MAX", 10) + array, n = core._column_to_java("label", ["0123456", "0123456"]) + assert n == 2 + assert not isinstance(array, _string_column_class()) diff --git a/tests/test_insert_columns_strings.py b/tests/test_insert_columns_strings.py new file mode 100644 index 0000000..fbaf6a2 --- /dev/null +++ b/tests/test_insert_columns_strings.py @@ -0,0 +1,87 @@ +"""insert_columns stores a string column exactly, on both wheels, whatever way it crosses. + +The JVM wheel sends a column of nothing but str and None as one UTF-8 blob plus int32 offsets (test_insert_columns_string_blob.py); +the native wheel reads the column in place. A user must not see a difference: a character beyond the BMP, a NUL, a line separator, +an empty string and a null are stored as they were given, in one column and in a mixed column, in the synchronous and the parallel +mode, from a list, a numpy string array and an object array. +""" + +import pytest + +np = pytest.importorskip("numpy") + +TEXTS = [ + "plain", + "", + "emoji \U0001f600 and \U00010000 and \U0010ffff", + "embedded\x00nul", + "\x00", + "line
separator
", + "中文 한글 ퟿ é", + "x" * 5_000, + 'tab\tnewline\nquote"backslash\\', +] + + +def _make_type(db, name): + db.command("sql", f"CREATE DOCUMENT TYPE {name}") + db.command("sql", f"CREATE PROPERTY {name}.id LONG") + db.command("sql", f"CREATE PROPERTY {name}.label STRING") + db.command("sql", f"CREATE PROPERTY {name}.other STRING") + + +def _stored(db, name): + rows = db.query("sql", f"SELECT FROM {name} ORDER BY id").to_list() # nosec B608 + return [(r["id"], r.get("label"), r.get("other")) for r in rows] + + +def _columns(n): + label = [TEXTS[i % len(TEXTS)] for i in range(n)] + other = [None if i % 4 == 1 else f"o{i}" for i in range(n)] + return {"id": np.arange(n, dtype=np.int64), "label": label, "other": other} + + +@pytest.mark.parametrize("parallel", [False, True], ids=["sync", "parallel"]) +def test_strings_are_stored_as_given(temp_db, parallel): + _make_type(temp_db, "S") + cols = _columns(3_000) + assert temp_db.insert_columns("S", cols, parallel=parallel) == 3_000 + assert _stored(temp_db, "S") == [ + (i, cols["label"][i], cols["other"][i]) for i in range(3_000) + ] + + +@pytest.mark.parametrize("kind", ["numpy_str", "object_array"]) +def test_other_containers_of_strings_are_stored_as_given(temp_db, kind): + _make_type(temp_db, "S") + # a numpy "U" array drops a trailing NUL by itself, so its texts have none + texts = [t for t in TEXTS if "\x00" not in t] + label = [texts[i % len(texts)] for i in range(40)] + column = np.array(label) if kind == "numpy_str" else np.array(label, dtype=object) + expected = label + temp_db.insert_columns("S", {"id": np.arange(40, dtype=np.int64), "label": column}) + assert [row[1] for row in _stored(temp_db, "S")] == expected + + +def test_a_mixed_column_keeps_every_value(temp_db): + _make_type(temp_db, "S") + temp_db.command("sql", "CREATE PROPERTY S.n LONG") + labels = ["a", "emoji \U0001f600", None, "", "z"] + temp_db.insert_columns( + "S", + { + "id": np.arange(5, dtype=np.int64), + "label": labels, + "n": [1, None, 3, 4, 5], + }, + ) + assert [row[1] for row in _stored(temp_db, "S")] == labels + assert _stored(temp_db, "S")[0][0] == 0 + + +def test_a_single_row_column(temp_db): + _make_type(temp_db, "S") + temp_db.insert_columns( + "S", {"id": np.array([10], dtype=np.int64), "label": ["one"]} + ) + assert _stored(temp_db, "S") == [(10, "one", None)] diff --git a/tests/test_jvm_cds.py b/tests/test_jvm_cds.py new file mode 100644 index 0000000..033f87e --- /dev/null +++ b/tests/test_jvm_cds.py @@ -0,0 +1,65 @@ +"""The bundled JRE ships a class-data-sharing (CDS) archive that the JVM uses at start (#313). + +A JRE made by jlink has no archive, so every start loaded and verified the JDK's classes again. ``scripts/dump_cds_archive.sh`` dumps the +JVM's default archive as the last step of the JRE build, with ``-XX:+UseCompactObjectHeaders`` (which ``jvm.py`` sets: the JVM ignores an +archive dumped with another setting, and it does so without an error). These tests run on the installed wheel, on every platform and +Python of the JVM matrix: the archive is in the wheel, the JVM maps it with the flags ``start_jvm`` builds (``-Xshare:on`` turns an +unusable archive into a failed start; ``-Xlog:class+load`` shows a JDK class coming from the shared objects file), and an archive does not +serve a JVM with other settings. The archive is the JVM's own default one, so a missing or mismatched file only means a start without it; +the JDK AOT cache (``aot.py``, ``test_aot.py``) stays opt-in. +""" + +import subprocess # nosec B404 - runs the bundled JRE's java with fixed arguments +import sys + +from arcadedb import jvm + + +def _java(): + name = "java.exe" if sys.platform == "win32" else "java" + return str(jvm._extract_runtime_resource("jre") / "bin" / name) + + +def _flags(tmp_path): + """The JVM arguments ``start_jvm`` passes, with a heap that keeps compressed oops on whatever the RAM of this host is.""" + return jvm._build_jvm_args( + heap_size="1g", disable_xml_limits=False, jvm_args=None, log_dir=tmp_path + ) + + +def _run(*args): + return subprocess.run( # nosec B603 - the bundled java, arguments of this file + [_java(), *args], capture_output=True, text=True, timeout=300 + ) + + +def test_the_wheel_ships_the_archive(): + archives = list(jvm._extract_runtime_resource("jre").rglob("*.jsa")) + assert archives, "no .jsa file in the bundled JRE" + assert all(a.stat().st_size > 1_000_000 for a in archives) + + +def test_the_jvm_maps_the_archive_with_the_flags_the_wheel_starts_with(tmp_path): + flags = _flags(tmp_path) + assert "-XX:+UseCompactObjectHeaders" in flags + # -Xshare:on: no usable archive is a failed start, not a quiet start without it + result = _run(*flags, "-Xshare:on", "-version") + assert result.returncode == 0, result.stderr + result = _run(*flags, "-Xshare:auto", "-Xlog:class+load:stdout", "-version") + assert result.returncode == 0, result.stderr + assert "java.lang.Object source: shared objects file" in result.stdout + + +def test_the_default_start_does_not_turn_the_archive_off(tmp_path): + for flag in _flags(tmp_path): + assert not flag.startswith(("-Xshare:", "-XX:-UseSharedSpaces")) + + +def test_an_archive_does_not_serve_a_jvm_with_other_settings(tmp_path): + flags = [f for f in _flags(tmp_path) if f != "-XX:+UseCompactObjectHeaders"] + flags.append("-XX:-UseCompactObjectHeaders") + # the archive was dumped with compact object headers: without them -Xshare:on cannot use it, and a plain start goes on without it + assert _run(*flags, "-Xshare:on", "-version").returncode != 0 + result = _run(*flags, "-Xshare:auto", "-Xlog:class+load:stdout", "-version") + assert result.returncode == 0, result.stderr + assert "java.lang.Object source: shared objects file" not in result.stdout diff --git a/tests/test_jvm_ingest_integrity_faults.py b/tests/test_jvm_ingest_integrity_faults.py new file mode 100644 index 0000000..78586ab --- /dev/null +++ b/tests/test_jvm_ingest_integrity_faults.py @@ -0,0 +1,54 @@ +"""The JVM wheel's column conversion and bulk vertex path with a rows-dropped fault in them (ArcadeData/arcadedb-python#336). + +``arcadedb.core._column_to_java`` converts every column of ``insert_columns`` (and of ``insert_arrow`` and ``insert_dataframe``, +which call it), and ``GraphBatch._create_vertices_json_bulk`` carries the rows of ``create_vertices``. Each test puts a fault into +one of them and runs the same checks as ``test_ingest_integrity.py``: they must report it. A file about the JPype path, so +``NATIVE_NOT_APPLICABLE`` lists it. +""" + +import pytest + +import arcadedb +from arcadedb import core +from tests.test_ingest_integrity import ( + BY_ID, + FOUND, + MUTATION_SIZE, + IntegrityError, + run_cases, +) + + +@pytest.mark.parametrize("family", ["cols", "arrow", "df"]) +def test_a_column_converted_one_row_short_is_reported( + store, small_chunks, monkeypatch, family +): + real = core._column_to_java + seen = [] + + def faulty(name, values): + seen.append(name) + return real(name, values[:-1]) + + monkeypatch.setattr(core, "_column_to_java", faulty) + with pytest.raises(IntegrityError) as found: + run_cases(store, BY_ID[family], [MUTATION_SIZE]) + assert FOUND in str(found.value), found.value + assert seen, "the fault was not in the path" + + +def test_a_bulk_vertex_chunk_one_row_short_is_reported( + store, small_chunks, monkeypatch +): + real = arcadedb.GraphBatch._create_vertices_json_bulk + seen = [] + + def faulty(self, type_name, rows): + seen.append(len(rows)) + return real(self, type_name, rows[:-1]) + + monkeypatch.setattr(arcadedb.GraphBatch, "_create_vertices_json_bulk", faulty) + with pytest.raises(IntegrityError) as found: + run_cases(store, BY_ID["vertices"], [MUTATION_SIZE]) + assert FOUND in str(found.value), found.value + assert seen, "the fault was not in the path" diff --git a/tests/test_native_column_frames_reference.py b/tests/test_native_column_frames_reference.py new file mode 100644 index 0000000..273ba59 --- /dev/null +++ b/tests/test_native_column_frames_reference.py @@ -0,0 +1,528 @@ +"""The native column frames give the table the JSON row path gave (refs #312). + +Before the frames, ``NativeResultSet.to_arrow``, ``to_columns`` and ``to_dataframe`` read every row as JSON text into one Python dict and +built the columns from those dicts. The module below (``REF_SOURCE``) is that assembly, copied from ``arcadedb._native.columns`` as it was +the day the frames came, and kept here as a private reference: each case reads the same query twice, once through the old route (the +rows drained as dicts, then the reference rules) and once through the frames, and compares the two column by column, value by value. + +Equal output is the bar. Two types are different on purpose, because the JVM wheel's frames type them so and both wheels now read frames +with one decoder: a column of ``float[]`` is a ``fixed_size_list`` (the old rules gave ``list``, the same values: a +float32 widened to float64 is exact), and a column of only nulls is a string column (the old rules gave the null type). + +conftest.py collects this file on the native backend only (the old route is the native build's JSON reply). +""" + +import math +import types +from datetime import date, datetime +from decimal import Decimal + +import numpy as np +import pandas as pd +import pyarrow as pa +import pytest + +REF_SOURCE = '''"""Rows to columns for the native backend (the columnar path is Python over the rows; the JPype path decodes a Java buffer).""" + +from datetime import date, datetime, timezone +from decimal import Decimal + + +def _names(rows, columns): + if columns is not None: + return list(columns) + seen = {} + for row in rows: + for k in row: + seen.setdefault(k, None) + return list(seen) + + +def _kind(values): + """'b', 'i', 'f', 'dt', 'dec', 'obj' for the non-null values of a column.""" + kinds = set() + for v in values: + if v is None: + continue + t = type(v) + if t is bool: + kinds.add("b") + elif t is int: + kinds.add("i") + elif t is float: + kinds.add("f") + elif t is datetime or t is date: + kinds.add("dt") + elif t is Decimal: + kinds.add("dec") + else: + kinds.add("obj") + if not kinds: + return "obj" + if len(kinds) == 1: + return kinds.pop() + if kinds == {"i", "f"}: + return "f" + return "obj" + + +def _ms(v): + if isinstance(v, datetime): + if v.tzinfo is not None: + v = v.astimezone(timezone.utc).replace(tzinfo=None) + return v + return datetime(v.year, v.month, v.day) + + +def _float_vectors(values): + """A column of equal-length float lists (None allowed) that float32 holds exactly, as a (count, dim) float32 array with + NaN rows for the nulls; None for any other column. Rows cannot say float[] from List, so only a column whose every + value survives float32 unchanged is read as a vector column (the JPype path decides on the Java element type). + """ + import numpy as np + + dim = 0 + for v in values: + if v is None: + continue + if type(v) is not list or not v or not all(type(x) is float for x in v): + return None + if dim == 0: + dim = len(v) + elif len(v) != dim: + return None + if dim == 0: + return None + arr = np.full((len(values), dim), np.nan, dtype=np.float32) + for i, v in enumerate(values): + if v is not None: + arr[i] = v + if not np.array_equal(arr[i].astype(np.float64), np.array(v)): + return None + return arr + + +def to_columns(rows, columns=None): + import numpy as np + + out = {} + for name in _names(rows, columns): + values = [row.get(name) for row in rows] + if _kind(values) == "obj": + vectors = _float_vectors(values) + if vectors is not None: + out[name] = vectors + continue + has_null = any(v is None for v in values) + kind = _kind(values) + if kind == "i": + out[name] = ( + np.array([np.nan if v is None else v for v in values], dtype=np.float64) + if has_null + else np.array(values, dtype=np.int64) + ) + elif kind == "f": + out[name] = np.array( + [np.nan if v is None else v for v in values], dtype=np.float64 + ) + elif kind == "b": + out[name] = values if has_null else np.array(values, dtype=bool) + elif kind == "dt": + out[name] = np.array( + [ + ( + np.datetime64("NaT", "ms") + if v is None + else np.datetime64(_ms(v), "ms") + ) + for v in values + ], + dtype="datetime64[ms]", + ) + elif kind == "dec": + arr = np.empty(len(values), dtype=object) + arr[:] = values + out[name] = arr + else: + out[name] = values + return out + + +def to_arrow(rows, columns=None): + import pyarrow as pa + + arrays = {} + for name in _names(rows, columns): + values = [row.get(name) for row in rows] + kind = _kind(values) + if kind == "i": + arrays[name] = pa.array(values, type=pa.int64()) + elif kind == "f": + try: + arrays[name] = pa.array(values, type=pa.float64()) + except ( + pa.ArrowInvalid + ): # an int beyond +-2**53 next to floats: the column degrades to exact text + arrays[name] = pa.array( + [None if v is None else str(v) for v in values], type=pa.string() + ) + elif kind == "b": + arrays[name] = pa.array(values, type=pa.bool_()) + elif kind == "dt": + arrays[name] = pa.array( + [None if v is None else _ms(v) for v in values], type=pa.timestamp("ms") + ) + elif kind == "dec": + try: + arrays[name] = pa.array(values) + except ( + pa.ArrowInvalid + ): # wider than decimal256 (76 digits): an exact string + arrays[name] = pa.array( + [None if v is None else str(v) for v in values], type=pa.string() + ) + else: + try: + arrays[name] = pa.array([None if v is None else v for v in values]) + except (pa.ArrowInvalid, pa.ArrowTypeError): + arrays[name] = pa.array( + [None if v is None else str(v) for v in values], type=pa.string() + ) + return pa.table(arrays) +''' + +REF = types.ModuleType("columns_reference") +exec( + compile(REF_SOURCE, "columns_reference", "exec"), REF.__dict__ +) # nosec B102 - the test's own copy of the old rules + + +def _f32(x): + return float(np.float32(x)) + + +def _row(i): + null = i % 7 == 3 + row = { + "n": None if null else i, + "big": (1 << 40) + i, + "sh": None if null else (i % 2000) - 1000, + "bt": None if null else (i % 200) - 100, + "d": i / 3, + "f": None if null else _f32(i / 7), + "s": None if null else f"row {i} \U0001f600 \x00 \u00e9", + "ok": None if null else i % 2 == 0, + "ts": datetime(2026, 1, 2, 3, 4, 5, (i % 1000) * 1000), + "day": date(2000 + i % 30, 1 + i % 12, 1 + i % 28), + "m": Decimal(f"{i}.25"), + "v1": [_f32(i)], + "v3": [_f32(i), _f32(i / 2), 0.5], + "v9": None if null else [_f32(i + k / 10) for k in range(9)], + "v128": [_f32(i * 0.5 + k) for k in range(128)], + "pad": "p" * 400, + "nothing": None, + } + return row + + +TYPES = [ + ("n", "INTEGER"), + ("big", "LONG"), + ("sh", "SHORT"), + ("bt", "BYTE"), + ("d", "DOUBLE"), + ("f", "FLOAT"), + ("s", "STRING"), + ("ok", "BOOLEAN"), + ("ts", "DATETIME"), + ("day", "DATE"), + ("m", "DECIMAL"), + ("v1", "ARRAY_OF_FLOATS"), + ("v3", "ARRAY_OF_FLOATS"), + ("v9", "ARRAY_OF_FLOATS"), + ("v128", "ARRAY_OF_FLOATS"), + ("pad", "STRING"), + ("nothing", "STRING"), +] +NAMES = [n for n, _ in TYPES] +ROWS = 5000 + + +@pytest.fixture(scope="module") +def ref_db(tmp_path_factory): + import arcadedb + + path = str(tmp_path_factory.mktemp("frames_ref") / "db") + db = arcadedb.create_database(path) + db.command("sql", "CREATE DOCUMENT TYPE R") + for name, type_name in TYPES: + db.command("sql", f"CREATE PROPERTY R.{name} {type_name}") + db.insert_many("R", [_row(i) for i in range(ROWS)]) + yield db + db.close() + + +def _eq(have, want): + """Equal values, NaN equal to NaN, None equal only to None: a missing value is not a NaN.""" + if isinstance(have, np.ndarray) or isinstance(want, np.ndarray): + return np.array_equal(np.asarray(have), np.asarray(want), equal_nan=True) + if isinstance(have, (list, tuple)) and isinstance(want, (list, tuple)): + return len(have) == len(want) and all(_eq(x, y) for x, y in zip(have, want)) + if ( + isinstance(have, float) + and isinstance(want, float) + and math.isnan(have) + and math.isnan(want) + ): + return True + return have == want + + +def _old_rows(db, sql, *args): + return db.query("sql", sql, *args)._drain_dicts() + + +def _assert_arrow_equal(new, old): + assert new.column_names == old.column_names + assert new.num_rows == old.num_rows + for name in old.column_names: + have, want = new.column(name), old.column(name) + if pa.types.is_null(want.type): + assert have.type == pa.string() and have.null_count == len(have), name + elif ( + pa.types.is_list(want.type) + and want.type.value_type == pa.float64() + and name.startswith("v") + ): + # a float[] column: the same values as a fixed-size list of float32 (a float32 widened to float64 is exact) + assert have.type == pa.list_( + pa.float32(), + len(want[0].as_py() if want[0].is_valid else want[1].as_py()), + ), name + else: + assert have.type == want.type, (name, have.type, want.type) + assert _eq(have.to_pylist(), want.to_pylist()), name + + +def _assert_columns_equal(new, old): + assert list(new) == list(old) + for name, want in old.items(): + have = new[name] + if isinstance(want, np.ndarray): + assert isinstance(have, np.ndarray), (name, type(have)) + assert have.dtype == want.dtype, (name, have.dtype, want.dtype) + assert have.shape == want.shape, name + np.testing.assert_array_equal(have, want, err_msg=name) + else: + assert isinstance(have, list), (name, type(have)) + assert _eq(have, want), name + + +def _missing_as_none(values): + """pandas decides how a missing value is shown (None, nan or NaT, by dtype and version): all of them are one here.""" + return [ + ( + None + if v is None or v is pd.NaT or (isinstance(v, float) and math.isnan(v)) + else v + ) + for v in values + ] + + +def _assert_frame_equal(new, old_rows, columns=None): + old = pd.DataFrame(old_rows) + if columns is not None: + old = old.reindex(columns=list(columns)) + assert list(new.columns) == list(old.columns) + for name in old.columns: + if name.startswith("v") and name != "vv": + # a vector cell is a numpy float32 array (documented), the values are the stored ones + for cell, want in zip(new[name].tolist(), old[name].tolist()): + if want is None: + # a null vector is a missing value, never a vector of NaN + assert cell is None, (name, cell) + else: + assert isinstance(cell, np.ndarray) and cell.dtype == np.float32, ( + name, + type(cell), + ) + assert _eq(cell.tolist(), want), name + continue + have = _missing_as_none(new[name].tolist()) + want = _missing_as_none(old[name].tolist()) + assert [ + x.to_pydatetime() if isinstance(x, pd.Timestamp) else x for x in have + ] == [ + x.to_pydatetime() if isinstance(x, pd.Timestamp) else x for x in want + ], name + if "ts" in old.columns: + assert str(new["ts"].dtype).startswith("datetime64[us"), new[ + "ts" + ].dtype # microseconds, whatever pandas infers from datetimes + + +QUERIES = [ + ("full", "SELECT FROM R ORDER BY big", None), + ( + "projection", + "SELECT n, big, sh, bt, d, f, s, ok, ts, day, m FROM R ORDER BY big", + None, + ), + ("vectors", "SELECT big, v1, v3, v9, v128 FROM R ORDER BY big", None), + ("limit 1", "SELECT FROM R ORDER BY big LIMIT 1", None), + ("limit 100, one reply", "SELECT FROM R ORDER BY big LIMIT 100", None), + ("limit 300, a reply and a cursor", "SELECT FROM R ORDER BY big LIMIT 300", None), + ("empty", "SELECT FROM R WHERE big < 0", None), + ("subset", "SELECT FROM R ORDER BY big", ["s", "big", "v3", "ts"]), + ("subset with a column nobody has", "SELECT FROM R ORDER BY big", ["n", "absent"]), + ("subset on a short result", "SELECT FROM R ORDER BY big LIMIT 20", ["m", "ok"]), +] + + +@pytest.mark.parametrize("label,sql,columns", QUERIES, ids=[q[0] for q in QUERIES]) +def test_to_arrow_equals_the_old_route(ref_db, label, sql, columns): + old = REF.to_arrow(_old_rows(ref_db, sql), columns) + new = ref_db.query("sql", sql).to_arrow(columns=columns) + _assert_arrow_equal(new, old) + + +@pytest.mark.parametrize("label,sql,columns", QUERIES, ids=[q[0] for q in QUERIES]) +def test_to_columns_equals_the_old_route(ref_db, label, sql, columns): + old = REF.to_columns(_old_rows(ref_db, sql), columns) + new = ref_db.query("sql", sql).to_columns(columns=columns) + _assert_columns_equal(new, old) + + +@pytest.mark.parametrize( + "label,sql,columns", QUERIES[:7], ids=[q[0] for q in QUERIES[:7]] +) +def test_to_dataframe_equals_the_old_route(ref_db, label, sql, columns): + rows = _old_rows(ref_db, sql) + new = ref_db.query("sql", sql).to_dataframe() + _assert_frame_equal(new, rows) + + +@pytest.mark.parametrize("batch_size", [1, 64, 255, 256, 257, 1000, 25000]) +def test_the_frames_do_not_depend_on_the_batch_size(ref_db, batch_size): + sql = "SELECT FROM R ORDER BY big" + old = REF.to_arrow(_old_rows(ref_db, sql)) + _assert_arrow_equal(ref_db.query("sql", sql).to_arrow(batch_size=batch_size), old) + + +def test_a_result_larger_than_one_frame_is_read_in_several(ref_db): + from unittest import mock + + from arcadedb._native import lib + + sizes = [] + real = lib.call_frame + + def spy(*args): + raw = real(*args) + sizes.append(len(raw)) + return raw + + # 5000 rows of about 2.4 KiB of values: several frames of about 1 MiB each, and the table is the same as one read of the rows + with mock.patch.object(lib, "call_frame", spy): + t = ref_db.query("sql", "SELECT FROM R ORDER BY big").to_arrow() + assert t.num_rows == ROWS + assert len(sizes) >= 4, sizes + assert max(sizes) < 4 * (1 << 20), sizes # a frame, not the table + assert t.column("big").to_pylist() == [(1 << 40) + i for i in range(ROWS)] + + +def test_rows_read_by_iteration_first_then_the_rest_as_columns(ref_db): + sql = "SELECT FROM R ORDER BY big" + old = REF.to_arrow(_old_rows(ref_db, sql)).slice(40) + rs = ref_db.query("sql", sql) + taken = [next(rs).get("big") for _ in range(40)] + assert taken == [(1 << 40) + i for i in range(40)] + _assert_arrow_equal(rs.to_arrow(), old) + + +def test_rows_read_past_the_first_reply_then_the_rest_as_columns(ref_db): + sql = "SELECT FROM R ORDER BY big" + old = REF.to_arrow(_old_rows(ref_db, sql)).slice(300) + rs = ref_db.query("sql", sql) + for _ in range(300): + next(rs) + _assert_arrow_equal(rs.to_arrow(), old) + + +def test_a_result_is_read_once(ref_db): + rs = ref_db.query("sql", "SELECT FROM R ORDER BY big LIMIT 500") + assert rs.to_arrow().num_rows == 500 + assert rs.to_arrow().num_rows == 0 + + +def test_to_list_still_reads_the_json_rows(ref_db): + rows = ref_db.query("sql", "SELECT FROM R ORDER BY big LIMIT 300").to_list() + assert len(rows) == 300 and rows[299]["big"] == (1 << 40) + 299 + + +def test_a_frame_stays_near_its_budget_when_every_row_has_other_properties(tmp_path): + """Rows that each carry a property no other row has: the frame holds rows times the union of the names, not the values read.""" + from unittest import mock + + import arcadedb + from arcadedb._native import lib + + db = arcadedb.create_database(str(tmp_path / "sparse")) + try: + db.command("sql", "CREATE DOCUMENT TYPE Sparse") + db.insert_many("Sparse", [{"i": i, f"p{i}": i} for i in range(2000)]) + sizes = [] + real = lib.call_frame + + def spy(*args): + raw = real(*args) + sizes.append(len(raw)) + return raw + + with mock.patch.object(lib, "call_frame", spy): + t = db.query("sql", "SELECT FROM Sparse ORDER BY i").to_arrow() + assert t.num_rows == 2000 and t.num_columns == 2001 + assert t.column("p1999").to_pylist() == [None] * 1999 + [1999] + # about 1 MiB of fixed-width buffers (rows x columns x 9 bytes) is what a frame is allowed; 2000 x 2000 x 9 = 36 MiB in one frame is not + assert len(sizes) > 5 and max(sizes) < 3 * (1 << 20), sizes + finally: + db.close() + + +def test_an_empty_frame_for_a_cursor_with_rows_left_is_an_error_not_a_loop(ref_db): + from unittest import mock + + from arcadedb._native import lib + from arcadedb.exceptions import ArcadeDBError + + header = b'{"count":0,"done":false,"cols":[]}' + empty = len(header).to_bytes(4, "little") + header + + rs = ref_db.query("sql", "SELECT FROM R ORDER BY big") + with mock.patch.object(lib, "call_frame", lambda *args: empty): + with pytest.raises(ArcadeDBError, match="empty column frame"): + rs.to_arrow() + + +def test_a_cursor_reads_a_huge_batch_size_without_wrapping(ref_db): + sql = "SELECT big FROM R ORDER BY big" + for batch_size in (2**31, 2**32, 2**33 + 5): + assert ref_db.query("sql", sql).to_arrow(batch_size=batch_size).num_rows == ROWS + + +def test_a_list_property_is_not_a_vector_column_at_any_result_size(tmp_path): + import arcadedb + + db = arcadedb.create_database(str(tmp_path / "lists")) + try: + db.command("sql", "CREATE DOCUMENT TYPE L") + db.insert_many("L", [{"i": i, "lst": [0.5, 1.0, 2.0]} for i in range(600)]) + for rows in (5, 300, 600): + c = db.query( + "sql", + f"SELECT i, lst FROM L ORDER BY i LIMIT {rows}", # nosec B608 - an integer of the test + ).to_columns() + assert isinstance(c["lst"], list) and c["lst"] == [[0.5, 1.0, 2.0]] * rows + finally: + db.close() diff --git a/tests/test_native_columnar_insert.py b/tests/test_native_columnar_insert.py index 1ec2984..e729696 100644 --- a/tests/test_native_columnar_insert.py +++ b/tests/test_native_columnar_insert.py @@ -315,15 +315,18 @@ def test_rows_with_different_keys_fall_back_to_json(temp_db, spy, monkeypatch): _fallback_case(temp_db, spy, monkeypatch, rows, ROWS) -def test_a_datetime_leaves_the_columnar_entry_but_stays_on_the_json_entry( - temp_db, spy, monkeypatch -): - """columnar.pack refuses a date-time, so the chunk takes the JSON entry, which carries it as a tagged value - (test_native_mixed_chunk.py compares what it stores with INSERT ... CONTENT).""" +def test_a_datetime_column_stays_on_the_columnar_entry(temp_db, spy, monkeypatch): + """A column of date-times is the kind T of the columnar entry (#316): the stored value is the one the JSON entry stores as a tagged + value (tests/test_native_columnar_temporal.py has the full comparison, test_native_mixed_chunk.py the rows that stay on JSON). + """ when = datetime.datetime(2024, 5, 6, 7, 8, 9) rows = [{"id": i, "txt": "t%d" % i, "when": when} for i in range(10)] - _fallback_case(temp_db, spy, monkeypatch, rows, ROWS) - assert spy.count(MANY) == 0 + _types(temp_db, "Ref", "Col") + _json_reference(temp_db, "Ref", rows, monkeypatch) + spy.calls.clear() + temp_db.insert_many("Col", rows) + assert spy.count(COLP) == 1 and spy.count(ROWS) == 0 and spy.count(MANY) == 0 + assert _stored(temp_db, "Col") == _stored(temp_db, "Ref") def test_values_json_cannot_carry_still_go_row_by_row_in_one_crossing( @@ -344,9 +347,7 @@ def test_an_integer_past_64_bits_is_refused_on_both_paths(temp_db, spy): temp_db.insert_columns("Col", {"id": np.array([2**63], dtype=np.uint64)}) -def test_a_dataframe_with_a_column_the_buffers_cannot_carry_falls_back( - temp_db, spy, monkeypatch -): +def test_a_dataframe_with_a_timestamp_column_stays_columnar(temp_db, spy, monkeypatch): import pandas as pd import pyarrow as pa @@ -362,7 +363,7 @@ def test_a_dataframe_with_a_column_the_buffers_cannot_carry_falls_back( } ) temp_db.insert_arrow("Col", table) - assert spy.count(COLP) == 0 + assert spy.count(COLP) == 1 and spy.count(ROWS) == 0 assert pd is not None rows = temp_db.query("sql", "SELECT id, txt FROM Col ORDER BY id").to_list() assert [(r["id"], r["txt"]) for r in rows] == [(1, "a"), (2, "b"), (3, "c")] diff --git a/tests/test_native_columnar_memory.py b/tests/test_native_columnar_memory.py new file mode 100644 index 0000000..b3ed404 --- /dev/null +++ b/tests/test_native_columnar_memory.py @@ -0,0 +1,300 @@ +"""Native: a document or vertex load by ``insert_arrow`` or ``insert_dataframe`` keeps its transient memory to one batch, and a vector column +is never turned into Python floats on its way to the columnar entry (ArcadeData/arcadedb-python #333 and #335, the extra anonymous memory of +the post1 candidate's native wheel). + +Three properties are pinned, none of them with an absolute size: + +* ``_column_values(..., arrow=True)`` returns an Arrow list of numbers (a vector column) as the Arrow array, which ``columnar.pack`` reads in + place; the JVM build (``arrow=False``) still gets Python lists. One ``to_pylist()`` of a 64-dimensional vector column makes 64 Python + floats per row (about 2 KB a row), which a 10,000-row batch held for the whole crossing. +* the Python-side peak of a whole load is a small fraction of what one batch of Python floats costs (``tracemalloc`` sees numpy and Python + objects, not Arrow's memory or the image's heap, which is why the ratio is generous). +* ``columnar.pack`` is called once per batch and never with more than ``batch_size`` rows, and what it laid out is released when the batch + is sent (the arrays of batch k are gone before batch k + 1 is packed). + +Needs the native library, so conftest.py collects this file on the native backend only. +""" + +import ctypes +import gc +import tracemalloc +import weakref + +import numpy as np +import pyarrow as pa +import pytest + +from arcadedb import dataframe +from arcadedb._native import columnar + +DIM = 64 + + +def _vectors(n, start=0, dim=DIM): + r = np.arange(start, start + n, dtype=np.int64) + j = np.arange(dim, dtype=np.int64) + return ( + ((r[:, None] * 131 + j[None, :] * 17) % 1000).astype(np.float32) + ) / np.float32(1000.0) + + +def _fixed(mat): + return pa.FixedSizeListArray.from_arrays( + pa.array(mat.reshape(-1), type=pa.float32()), mat.shape[1] + ) + + +def _bytes(addr_len): + addr, length = addr_len + return ctypes.string_at(addr, length) if length else b"" + + +def _peak(fn): + gc.collect() + tracemalloc.start() + try: + tracemalloc.reset_peak() + base = tracemalloc.get_traced_memory()[0] + fn() + return tracemalloc.get_traced_memory()[1] - base + finally: + tracemalloc.stop() + + +@pytest.mark.parametrize( + "make", + [ + lambda m: _fixed(m), + lambda m: pa.array(m.astype(np.float64).tolist(), type=pa.list_(pa.float64())), + lambda m: pa.array(m.astype(np.int32).tolist(), type=pa.large_list(pa.int32())), + ], + ids=["fixed_size_list_f32", "list_f64", "large_list_i32"], +) +def test_a_vector_column_stays_an_arrow_array_on_the_native_load(make): + chunk = make(_vectors(10)) + kept = dataframe._column_values("embedding", chunk, pa, native=True, arrow=True) + assert isinstance(kept, pa.Array) and kept.type == chunk.type + assert ( + dataframe._column_values("embedding", chunk, pa, native=True, arrow=False) + == chunk.to_pylist() + ) + assert isinstance( + dataframe._column_values("embedding", chunk, pa, native=False, arrow=False), + list, + ) + + +def test_a_list_that_is_not_a_list_of_numbers_still_becomes_python_values(): + for chunk in ( + pa.array([["a", "b"], ["c"]]), + pa.array([[{"x": 1}], [{"x": 2}]]), + pa.array([[[1.0], [2.0]], [[3.0]]]), + ): + got = dataframe._column_values("c", chunk, pa, native=True, arrow=True) + assert isinstance(got, list) and got == chunk.to_pylist() + + +def test_the_vector_layout_from_arrow_is_the_layout_from_python_lists(): + mat = _vectors(300) + table = pa.table({"embedding": _fixed(mat)}).slice( + 7, 211 + ) # a slice: the offset of the array is not zero + declared = columnar.Declared(known={"embedding": "ARRAY_OF_FLOATS"}) + kept = dataframe._column_values( + "embedding", table.column(0).combine_chunks(), pa, True, True + ) + listed = dataframe._column_values( + "embedding", table.column(0).combine_chunks(), pa, True, False + ) + rows_a, (col_a,) = columnar.pack({"embedding": kept}, declared) + rows_b, (col_b,) = columnar.pack({"embedding": listed}, declared) + assert ( + rows_a == rows_b == 211 + and col_a.kind == col_b.kind == "f" + and col_a.dim == col_b.dim == DIM + ) + assert _bytes(col_a.data) == _bytes(col_b.data) == mat[7:218].tobytes() + + +def test_the_python_side_peak_of_a_vector_batch_is_a_fraction_of_the_python_float_copy(): + batch = _fixed(_vectors(10_000)) + declared = columnar.Declared(known={"embedding": "ARRAY_OF_FLOATS"}) + as_python = _peak( + lambda: columnar.pack( + { + "embedding": dataframe._column_values( + "embedding", batch, pa, True, False + ) + }, + declared, + ) + ) + in_place = _peak( + lambda: columnar.pack( + {"embedding": dataframe._column_values("embedding", batch, pa, True, True)}, + declared, + ) + ) + assert ( + as_python > 10 * 1024 * 1024 + ), as_python # 640,000 Python floats: the copy this change removes (about 20 MiB) + assert in_place * 8 < as_python, ( + in_place, + as_python, + ) # measured: about 100 times less + + +class _PackSpy: + """Wraps ``columnar.pack``: the row count of every call and a weak reference to every array it laid out.""" + + def __init__(self, monkeypatch): + self.rows, self.arrays, self.alive_at_next_call = [], [], [] + real = columnar.pack + + def pack(columns, declared): + gc.collect() + self.alive_at_next_call.append( + sum(ref() is not None for refs in self.arrays for ref in refs) + ) + out = real(columns, declared) + self.rows.append(None if out is None else out[0]) + self.arrays.append( + [weakref.ref(c.keep[0]) for c in out[1]] if out is not None else [] + ) + return out + + monkeypatch.setattr(columnar, "pack", pack) + + +def _vector_type(db, kind="VERTEX"): + db.command("sql", f"CREATE {kind} TYPE V") + db.command("sql", "CREATE PROPERTY V.id LONG") + db.command("sql", "CREATE PROPERTY V.embedding ARRAY_OF_FLOATS") + + +@pytest.mark.parametrize("kind", ["VERTEX", "DOCUMENT"]) +def test_every_batch_is_packed_once_and_released_before_the_next( + temp_db, monkeypatch, kind +): + _vector_type(temp_db, kind) + spy = _PackSpy(monkeypatch) + n, batch = 2_505, 1_000 + table = pa.table( + {"id": pa.array(np.arange(n, dtype=np.int64)), "embedding": _fixed(_vectors(n))} + ) + assert temp_db.insert_arrow("V", table, batch_size=batch) == n + assert spy.rows == [ + 1_000, + 1_000, + 505, + ] # one pack per batch, never more than batch_size rows + assert spy.alive_at_next_call == [ + 0, + 0, + 0, + ], spy.alive_at_next_call # batch k is gone when batch k + 1 is packed + gc.collect() + assert not any( + ref() is not None for refs in spy.arrays for ref in refs + ) # and the last one when the call returns + + +def test_a_dataframe_load_is_packed_per_batch_too(temp_db, monkeypatch): + import pandas as pd + + temp_db.command("sql", "CREATE VERTEX TYPE P") + temp_db.command("sql", "CREATE PROPERTY P.id LONG") + temp_db.command("sql", "CREATE PROPERTY P.name STRING") + spy = _PackSpy(monkeypatch) + frame = pd.DataFrame( + {"id": np.arange(250, dtype=np.int64), "name": [f"p{i}" for i in range(250)]} + ) + assert temp_db.insert_dataframe("P", frame, batch_size=100) == 250 + assert spy.rows == [100, 100, 50] and spy.alive_at_next_call == [0, 0, 0] + + +def test_the_python_side_peak_of_a_whole_vector_load_stays_below_one_batch_of_python_floats( + temp_db, +): + _vector_type(temp_db) + n, batch = 40_000, 10_000 + table = pa.table( + {"id": pa.array(np.arange(n, dtype=np.int64)), "embedding": _fixed(_vectors(n))} + ) + one_batch_of_python_floats = _peak( + lambda: table.slice(0, batch).column("embedding").to_pylist() + ) + whole_load = _peak(lambda: temp_db.insert_arrow("V", table, batch_size=batch)) + assert one_batch_of_python_floats > 10 * 1024 * 1024 + assert whole_load * 2 < one_batch_of_python_floats, ( + whole_load, + one_batch_of_python_floats, + ) # measured: about 7 times less + assert temp_db.query("sql", "SELECT count(*) AS n FROM V").first().get("n") == n + + +def _stored(db, key): + return [ + float(x) + for x in db.query("sql", "SELECT embedding FROM V WHERE id = ?", key) + .first() + .get("embedding") + ] + + +@pytest.mark.parametrize("kind", ["VERTEX", "DOCUMENT"]) +def test_the_stored_vectors_are_the_ones_the_python_lists_stored(temp_db, kind): + _vector_type(temp_db, kind) + mat = _vectors(120) + table = pa.table( + {"id": pa.array(np.arange(120, dtype=np.int64)), "embedding": _fixed(mat)} + ) + assert ( + temp_db.insert_arrow("V", table.slice(5, 100), batch_size=30) == 100 + ) # a sliced table, four batches + for key in (5, 6, 34, 35, 64, 104): + assert ( + max(abs(a - b) for a, b in zip(_stored(temp_db, key), mat[key].tolist())) + < 1e-6 + ) + assert temp_db.query("sql", "SELECT count(*) AS n FROM V").first().get("n") == 100 + + +def test_a_vector_column_with_a_null_or_a_list_property_takes_the_row_path_with_the_same_values( + temp_db, +): + _vector_type(temp_db) + temp_db.command("sql", "CREATE VERTEX TYPE L") + temp_db.command("sql", "CREATE PROPERTY L.id LONG") + temp_db.command("sql", "CREATE PROPERTY L.tags LIST") + with_null = pa.table( + { + "id": pa.array([1, 2, 3], type=pa.int64()), + "embedding": pa.array( + [[0.5, 0.25], None, [1.0, 2.0]], type=pa.list_(pa.float32()) + ), + } + ) + assert temp_db.insert_arrow("V", with_null) == 3 + assert [ + float(x) + for x in temp_db.query("sql", "SELECT embedding FROM V WHERE id = 1") + .first() + .get("embedding") + ] == [0.5, 0.25] + assert ( + temp_db.query("sql", "SELECT embedding FROM V WHERE id = 2") + .first() + .get("embedding") + is None + ) + tags = pa.table( + { + "id": pa.array([1, 2], type=pa.int64()), + "tags": pa.array([[1, 2, 3], [4]], type=pa.list_(pa.int64())), + } + ) + assert temp_db.insert_arrow("L", tags) == 2 + assert list( + temp_db.query("sql", "SELECT tags FROM L WHERE id = 1").first().get("tags") + ) == [1, 2, 3] diff --git a/tests/test_native_columnar_pack.py b/tests/test_native_columnar_pack.py index cda0a1f..e892566 100644 --- a/tests/test_native_columnar_pack.py +++ b/tests/test_native_columnar_pack.py @@ -190,7 +190,11 @@ def test_a_bitmap_longer_than_a_byte(): [1, 2**63], # past 64 bits [{"a": 1}, {"a": 2}], # nested [[1], [2]], # a list without a declared ARRAY_OF_FLOATS - [b"ab", b"cd"], + [ + b"ab", + "cd", + ], # bytes and text (a column of bytes alone crosses as kind x since #316) + [b"ab", bytearray(b"cd")], # two bytes-like types [np.int64(1), np.int64(2)], # numpy scalars in a list are not Python numbers ], ) @@ -198,16 +202,107 @@ def test_columns_the_buffer_cannot_carry_unchanged_are_refused(column): assert columnar.pack({"c": column}, NO_TYPES) is None -def test_datetimes_decimals_and_object_arrays_of_them_are_refused(): +def test_numpy_datetime64_arrays_and_mixed_temporal_columns_are_refused(): import datetime import decimal - assert columnar.pack({"d": [datetime.datetime(2020, 1, 1)]}, NO_TYPES) is None - assert columnar.pack({"d": [decimal.Decimal("1.5")]}, NO_TYPES) is None assert ( columnar.pack({"d": np.array(["2020-01-01"], dtype="datetime64[D]")}, NO_TYPES) is None ) + assert ( + columnar.pack( + {"d": [datetime.datetime(2020, 1, 1), datetime.date(2020, 1, 1)]}, NO_TYPES + ) + is None + ) + assert ( + columnar.pack({"d": [decimal.Decimal("1"), decimal.Decimal("NaN")]}, NO_TYPES) + is None + ) + assert columnar.pack({"d": [b"a", "b"]}, NO_TYPES) is None + + +def _ticks(col, dtype): + return np.frombuffer(_bytes(col.data), dtype=dtype).tolist() + + +def test_python_datetimes_are_utc_wall_clock_microseconds(): + import datetime + + utc9 = datetime.timezone(datetime.timedelta(hours=9)) + values = [ + datetime.datetime(1970, 1, 1), + datetime.datetime(1969, 12, 31, 23, 59, 59, 999999), + datetime.datetime(2020, 1, 2, 12, 0, 0, 5, tzinfo=utc9), # 03:00:00.000005 UTC + None, + ] + rows, cols = columnar.pack({"t": values}, NO_TYPES) + col = cols[0] + assert rows == 4 and col.kind == "T" and col.dim == 1_000_000 + assert _ticks(col, np.int64) == [0, -1, 1_577_934_000_000_005, 0] + assert _present(col) == [True, True, True, False] + + +def test_python_dates_are_epoch_days(): + import datetime + + rows, cols = columnar.pack( + {"d": [datetime.date(1970, 1, 2), datetime.date(1969, 12, 31), None]}, NO_TYPES + ) + assert cols[0].kind == "D" and _ticks(cols[0], np.int32) == [1, -1, 0] + assert _present(cols[0]) == [True, True, False] + + +def test_decimals_and_bytes_use_the_string_layout(): + import decimal + + rows, cols = columnar.pack( + { + "a": [decimal.Decimal("1E+3"), None, decimal.Decimal("-0.50")], + "b": [b"\x00\xff", b"", None], + "c": [bytearray(b"xy"), bytearray(b"z"), bytearray(b"")], + }, + NO_TYPES, + ) + a, b, c = cols + assert [x.kind for x in cols] == ["n", "x", "x"] + offsets = np.frombuffer(_bytes(a.data), dtype=np.int32).tolist() + assert offsets == [0, 4, 4, 9] and _bytes(a.aux) == b"1E+3-0.50" + offsets = np.frombuffer(_bytes(b.data), dtype=np.int32).tolist() + assert offsets == [0, 2, 2, 2] and _bytes(b.aux) == b"\x00\xff" + assert _bytes(c.aux) == b"xyz" and _present(a) == [True, False, True] + + +def test_arrow_timestamps_dates_decimals_and_binary(): + import decimal + + import pyarrow as pa + + stamps = pa.array([1, None, -2], pa.int64()).cast( + pa.timestamp("ms", tz="Asia/Seoul") + ) + days = pa.array([3, -4, None], pa.int32()).cast(pa.date32()) + day_ms = pa.array([86_400_000 * 2, -1, None], pa.int64()).cast(pa.date64()) + amounts = pa.array([decimal.Decimal("1.50"), None], pa.decimal128(10, 2)) + blobs = pa.array([b"ab", None, b"c"]) + for name, col, expected in ( + ("t", stamps, ("T", 1_000)), + ("d", days, ("D", 0)), + ("m", day_ms, ("D", 0)), + ): + packed = columnar.pack({name: col}, NO_TYPES)[1][0] + assert (packed.kind, packed.dim) == expected + assert _ticks(columnar.pack({"t": stamps}, NO_TYPES)[1][0], np.int64) == [1, 0, -2] + assert _ticks(columnar.pack({"d": days}, NO_TYPES)[1][0], np.int32) == [3, -4, 0] + # a date64 is floor-divided to days: 2 days, then -1 ms is the day before the epoch + assert _ticks(columnar.pack({"m": day_ms}, NO_TYPES)[1][0], np.int32) == [2, -1, 0] + n = columnar.pack({"n": amounts}, NO_TYPES)[1][0] + assert n.kind == "n" and _bytes(n.aux) == b"1.50" + x = columnar.pack({"x": blobs}, NO_TYPES)[1][0] + assert ( + x.kind == "x" and _bytes(x.aux) == b"abc" and _present(x) == [True, False, True] + ) def test_ragged_columns_and_empty_input_are_left_to_the_row_path(): @@ -233,6 +328,11 @@ def test_the_address_table_and_header_describe_the_columns(): "cols": [["id", "q", 0], ["s", "s", 0]], } assert json.loads(columnar.header_of(rows, 2**40, packed))["commit"] == 2**31 - 1 + # "parallel" and the failure token are in the header only when a token is given (the image hands the records to the async writers then) + parallel = json.loads(columnar.header_of(rows, 10_000, packed, fid=41)) + assert parallel["parallel"] is True and parallel["fid"] == 41 + plain = json.loads(columnar.header_of(rows, 10_000, packed)) + assert "parallel" not in plain and "fid" not in plain def test_rows_become_columns_only_when_every_row_has_the_same_keys(): diff --git a/tests/test_native_columnar_parallel.py b/tests/test_native_columnar_parallel.py new file mode 100644 index 0000000..90d6f79 --- /dev/null +++ b/tests/test_native_columnar_parallel.py @@ -0,0 +1,242 @@ +"""Native ``insert_columns(parallel=True)`` takes the columnar entry and stores what the row path of ``insert_many(parallel=True)`` stores. + +Before, a parallel call skipped the columnar fast path: the columns became one dict per row, then one JSON text per chunk of 10,000 +rows for ``arcadedb_n_async insert_parallel``. Now the same ``arcadedb_n_insert_colp`` crossing that the synchronous call uses carries +``"parallel": true`` and the image hands the records to the async writers, as ``DocumentBatcher.insertColumnsParallel`` does on the +JVM wheel. Needs the native library, so conftest.py collects this file on the native backend only. Each test loads the same data +through the row path (the private reference: ``insert_many(parallel=True)`` with dict rows) and through the columns, compares the +stored records value by value and type by type, and spies on the entry points so a silent fall back cannot pass. +""" + +import numpy as np +import pytest + +from arcadedb._native import bulk, extras, lib # noqa: E402 +from arcadedb.exceptions import ArcadeDBError # noqa: E402 + +COLP, ROWS = "arcadedb_n_insert_colp", "arcadedb_n_insert_rows" + + +class Spy: + """Counts the crossings by name and the calls that hand a chunk of dict rows to the parallel writers.""" + + def __init__(self, monkeypatch): + self.calls, self.row_chunks = [], 0 + real_call, real_rows = lib.call, extras.insert_parallel + + def call(name, *args): + self.calls.append(name) + return real_call(name, *args) + + def insert_parallel(database, type_name, rows): + self.row_chunks += 1 + return real_rows(database, type_name, rows) + + monkeypatch.setattr(lib, "call", call) + monkeypatch.setattr(extras, "insert_parallel", insert_parallel) + + def count(self, name): + return self.calls.count(name) + + +@pytest.fixture +def spy(monkeypatch): + return Spy(monkeypatch) + + +def _count(db, name): + sql = f"SELECT count(*) AS n FROM {name}" # nosec B608 - test-owned type name + return db.query("sql", sql).to_list()[0]["n"] + + +def _types(db, *names, kind="DOCUMENT", buckets=3, unique=False): + for name in names: + db.command("sql", f"CREATE {kind} TYPE {name} BUCKETS {buckets}") + db.command("sql", f"CREATE PROPERTY {name}.id INTEGER") + db.command("sql", f"CREATE PROPERTY {name}.val DOUBLE") + db.command("sql", f"CREATE PROPERTY {name}.flag BOOLEAN") + db.command("sql", f"CREATE PROPERTY {name}.txt STRING") + db.command("sql", f"CREATE PROPERTY {name}.vec ARRAY_OF_FLOATS") + if unique: + db.command("sql", f"CREATE INDEX ON {name} (id) UNIQUE") + + +def _stored(db, name): + """Every record of a type as (value, python type) pairs per property, ordered by id (the writers store in no fixed order).""" + sql = f"SELECT FROM {name} ORDER BY id" # nosec B608 - test-owned type name + return [ + {k: (v, type(v).__name__) for k, v in row.items() if not k.startswith("@")} + for row in db.query("sql", sql).to_list() + ] + + +def _columns(n, dim=4, seed=11): + rng = np.random.default_rng(seed) + ids = np.arange(n, dtype=np.int64) + return { + "id": ids, + "val": (ids * 37 % 10007) / 7.0, + "flag": ids % 3 == 0, + "txt": ["row-%d-%s" % (k, "é" * (k % 5)) for k in range(n)], + "vec": rng.standard_normal((n, dim), dtype=np.float32), + } + + +def _rows(columns): + names = list(columns) + lists = [c.tolist() if hasattr(c, "tolist") else c for c in columns.values()] + return [dict(zip(names, vals)) for vals in zip(*lists)] + + +def test_a_parallel_call_uses_the_columnar_entry_and_stores_the_row_paths_records( + temp_db, spy +): + _types(temp_db, "Ref", "Col") + columns = _columns(3_100) + assert temp_db.insert_many("Ref", _rows(columns), parallel=True) == 3_100 + assert spy.row_chunks == 1 and spy.count(COLP) == 0 # the reference is the row path + + spy.calls.clear() + spy.row_chunks = 0 + assert temp_db.insert_columns("Col", columns, parallel=True) == 3_100 + assert spy.count(COLP) == 1, spy.calls # one crossing for the whole call + assert spy.row_chunks == 0 and spy.count(ROWS) == 0 # no dict rows, no JSON text + + expected = _stored(temp_db, "Ref") + assert len(expected) == 3_100 and _count(temp_db, "Col") == 3_100 + assert expected[7]["id"][1] == "int" and expected[7]["vec"][1] == "list" + assert _stored(temp_db, "Col") == expected + + +def test_nulls_and_awkward_strings_are_stored_as_the_row_path_stores_them(temp_db, spy): + _types(temp_db, "Ref", "Col") + columns = { + "id": list(range(8)), + "txt": ["", "a", None, "snow☃man", "x" * 300, None, "\n\t", "z"], + "val": [1.5, None, 2.5, 1e300, -1e-300, 0.0, 3.0, None], + } + temp_db.insert_many("Ref", _rows(columns), parallel=True) + spy.calls.clear() + temp_db.insert_columns("Col", columns, parallel=True) + assert spy.count(COLP) == 1 + got, want = _stored(temp_db, "Col"), _stored(temp_db, "Ref") + assert got == want and got[2]["txt"][0] is None + + +def test_a_vertex_type_gets_vertices_in_parallel(temp_db, spy): + _types(temp_db, "Ref", "Col", kind="VERTEX") + columns = _columns(1_500) + temp_db.insert_many("Ref", _rows(columns), parallel=True) + spy.calls.clear() + temp_db.insert_columns("Col", columns, parallel=True) + assert spy.count(COLP) == 1 + assert _stored(temp_db, "Col") == _stored(temp_db, "Ref") + kinds = temp_db.query( + "sql", "SELECT FROM schema:types WHERE name = 'Col'" + ).to_list() + assert kinds[0]["type"] == "vertex" + + +def test_a_record_the_writers_reject_is_reported_after_the_wait(temp_db, spy): + _types(temp_db, "Col", unique=True) + ids = np.arange(400, dtype=np.int64) + ids[300] = ids[7] # a duplicate key + with pytest.raises(ArcadeDBError, match="failed record"): + temp_db.insert_columns("Col", {"id": ids, "txt": ["t"] * 400}, parallel=True) + assert spy.count(COLP) == 1 and spy.row_chunks == 0 + stored = _count(temp_db, "Col") + assert 0 < stored <= 400 # the call had waited for the writers: nothing lands later + + +def test_a_value_the_declared_type_refuses_raises_and_the_call_is_final(temp_db, spy): + _types(temp_db, "Col") + values = [str(i) for i in range(1_000)] + values[700] = "not a number" + with pytest.raises( + ArcadeDBError, match="may have been stored|failed record" + ) as err: + temp_db.insert_columns("Col", {"id": values}, parallel=True) + assert "rolled back" not in str(err.value) + assert spy.count(COLP) == 1 + stored = _count(temp_db, "Col") + temp_db.async_executor().wait_completion() + assert _count(temp_db, "Col") == stored <= 1_000 + + +def test_an_edge_type_is_refused_as_before(temp_db): + temp_db.command("sql", "CREATE EDGE TYPE Link") + with pytest.raises(ArcadeDBError, match="edge type"): + temp_db.insert_columns("Link", {"id": [1, 2, 3]}, parallel=True) + + +def test_a_missing_type_is_the_schemas_error(temp_db): + with pytest.raises(ArcadeDBError, match="NoSuchType"): + temp_db.insert_columns("NoSuchType", {"id": [1, 2, 3]}, parallel=True) + + +def test_a_column_the_buffers_cannot_carry_still_goes_through_the_row_path( + temp_db, spy +): + _types(temp_db, "Col") + columns = {"id": [1, 2, 3, 4], "txt": ["a", 2, "c", 4.5]} # a mixed-type column + assert temp_db.insert_columns("Col", columns, parallel=True) == 4 + assert spy.count(COLP) == 0 and spy.row_chunks == 1 + assert _count(temp_db, "Col") == 4 + + +def test_the_switch_that_turns_the_columnar_path_off_applies_to_parallel( + temp_db, spy, monkeypatch +): + _types(temp_db, "Col") + monkeypatch.setattr(bulk, "_ENABLED", False) + assert temp_db.insert_columns("Col", _columns(50), parallel=True) == 50 + assert spy.count(COLP) == 0 and spy.row_chunks == 1 + assert _count(temp_db, "Col") == 50 + + +def test_no_rows_is_zero_and_writes_nothing(temp_db, spy): + _types(temp_db, "Col") + assert temp_db.insert_columns("Col", {"id": []}, parallel=True) == 0 + assert spy.count(COLP) == 0 and _count(temp_db, "Col") == 0 + + +def test_the_synchronous_call_is_unchanged(temp_db, spy): + _types(temp_db, "Col", unique=True) + assert temp_db.insert_columns("Col", _columns(500), commit_every=100) == 500 + assert spy.count(COLP) == 1 and spy.row_chunks == 0 + assert _count(temp_db, "Col") == 500 + + +def test_insert_arrow_parallel_takes_the_columnar_entry(temp_db, spy, monkeypatch): + import pyarrow as pa + + _types(temp_db, "Ref", "ByTable") + columns = _columns(2_000) + table = pa.table( + { + "id": columns["id"], + "val": pa.array( + [None if i % 7 == 0 else v for i, v in enumerate(columns["val"])] + ), + "txt": pa.array(columns["txt"]), + } + ) + with ( + monkeypatch.context() as m + ): # the private reference: the row path of the old code + m.setattr(bulk, "_ENABLED", False) + assert ( + temp_db.insert_arrow("Ref", table, batch_size=700, parallel=True) == 2_000 + ) + assert spy.count(COLP) == 0 and spy.row_chunks >= 1 + + spy.calls.clear() + spy.row_chunks = 0 + assert ( + temp_db.insert_arrow("ByTable", table, batch_size=700, parallel=True) == 2_000 + ) + assert spy.count(COLP) == 3 and spy.row_chunks == 0 # one crossing per batch + + expected = _stored(temp_db, "Ref") + assert len(expected) == 2_000 and expected[0]["val"][0] is None + assert _stored(temp_db, "ByTable") == expected diff --git a/tests/test_native_columnar_parallel_review_fixes.py b/tests/test_native_columnar_parallel_review_fixes.py new file mode 100644 index 0000000..70f17bc --- /dev/null +++ b/tests/test_native_columnar_parallel_review_fixes.py @@ -0,0 +1,203 @@ +"""Review of the native parallel loads (#328): nothing is lost silently, an interrupt leaves no writer running and no token behind. + +``insert_columns(parallel=True)`` hands the records to the async writers in one crossing and waits for them before it returns. These +tests pin what a caller relies on: a record that a writer rejects is counted (``stored + reported failed >= rows``) and no key is stored twice +(a writer that meets a duplicate abandons its whole open batch, and reports it); a load at the writers' batch boundary stores every row and nothing lands after the call returns; +a ^C during the crossing, or between the chunks of the row path, waits for the writers and reads (so removes) the failure token; a wait +that fails still reads the tokens; and the message of a failed parallel batch inside a caller's transaction does not claim that a +rollback removes rows the writers committed. Needs the native library: conftest.py collects this file on the native backend only. +""" + +import re +import time + +import numpy as np +import pytest + +from arcadedb._native import database as native_database +from arcadedb._native import extras +from arcadedb.exceptions import ArcadeDBError + +COLP = "arcadedb_n_insert_colp" + + +def _count(db, name): + sql = f"SELECT count(*) AS n FROM {name}" # nosec B608 - test-owned type name + return db.query("sql", sql).to_list()[0]["n"] + + +def _type(db, name="Col", unique=False): + db.command("sql", f"CREATE DOCUMENT TYPE {name} BUCKETS 3") + db.command("sql", f"CREATE PROPERTY {name}.id INTEGER") + db.command("sql", f"CREATE PROPERTY {name}.txt STRING") + if unique: + db.command("sql", f"CREATE INDEX ON {name} (id) UNIQUE") + + +class TokenSpy: + """The failure tokens that were made and the ones that were read (a read removes the token's entries in the image).""" + + def __init__(self, monkeypatch): + self.made, self.read = [], [] + real_make, real_read = extras.failure_token, extras.failures + real_rows = extras.insert_parallel + + def make(database): + fid = real_make(database) + self.made.append(fid) + return fid + + def failures(database, token): + self.read.append(token) + return real_read(database, token) + + def rows(database, type_name, chunk): + fid = real_rows(database, type_name, chunk) + self.made.append(fid["fid"]) + return fid + + monkeypatch.setattr(extras, "failure_token", make) + monkeypatch.setattr(extras, "failures", failures) + monkeypatch.setattr(extras, "insert_parallel", rows) + + +@pytest.fixture +def tokens(monkeypatch): + return TokenSpy(monkeypatch) + + +def _failed_count(error): + return int(re.search(r"reported (\d+) failed record", str(error)).group(1)) + + +def test_a_duplicate_key_is_reported_and_no_row_is_stored_twice(temp_db): + _type(temp_db, unique=True) + n, dupes = 3_000, 50 + ids = np.arange(n, dtype=np.int64) + ids[n - dupes :] = np.arange(dupes) # the last 50 repeat the first 50 keys + with pytest.raises(ArcadeDBError, match="failed record") as err: + temp_db.insert_columns("Col", {"id": ids, "txt": ["t"] * n}, parallel=True) + stored_ids = [r["id"] for r in temp_db.query("sql", "SELECT id FROM Col").to_list()] + stored = len(stored_ids) + assert ( + stored + _failed_count(err.value) >= n + ) # every row is a stored row or a reported one + # A writer that meets a duplicate key rolls back its WHOLE open batch (up to commit_every rows), so the valid rows of that batch are + # abandoned and reported too: `stored` can be below the number of distinct keys, never above it. + assert stored <= len(set(ids.tolist())) + assert set(stored_ids) <= set(ids.tolist()) # only ids that were sent + assert len(stored_ids) == len(set(stored_ids)) # none stored twice + + +def test_a_load_at_the_writers_batch_boundary_stores_every_row_and_nothing_later( + temp_db, +): + _type(temp_db) + threads = temp_db.async_executor().get_parallel_level() + n = threads * 10_240 + 1 # one row past a full batch for every writer + ids = np.arange(n, dtype=np.int64) + assert ( + temp_db.insert_columns("Col", {"id": ids, "txt": ["t"] * n}, parallel=True) == n + ) + stored = _count(temp_db, "Col") + assert stored == n + time.sleep(0.3) + temp_db.async_executor().wait_completion() + assert _count(temp_db, "Col") == n # nothing landed after the call returned + distinct = temp_db.query("sql", "SELECT count(DISTINCT id) AS c FROM Col") + assert distinct.to_list()[0]["c"] == n + + +def test_an_interrupt_in_the_crossing_waits_for_the_writers_and_reads_the_token( + temp_db, tokens, monkeypatch +): + _type(temp_db) + real_call = native_database._call + + def interrupted(prefix, name, *args): + reply = real_call(prefix, name, *args) + if name == COLP: + raise KeyboardInterrupt # the records are handed over; the reply never reaches the caller + return reply + + monkeypatch.setattr(native_database, "_call", interrupted) + n = 30_000 + with pytest.raises(KeyboardInterrupt): + temp_db.insert_columns( + "Col", {"id": np.arange(n, dtype=np.int64), "txt": ["t"] * n}, parallel=True + ) + assert ( + temp_db.async_executor().is_processing() is False + ) # waited: no writer runs on + assert len(tokens.made) == 1 and tokens.read == tokens.made # and the token is gone + assert ( + _count(temp_db, "Col") == n + ) # the rows handed over were all written, none half + + +def test_an_interrupt_between_the_chunks_of_the_row_path_waits_and_reads_the_tokens( + temp_db, tokens, monkeypatch +): + _type(temp_db) + real_rows = extras.insert_parallel + seen = [] + + def second_chunk_interrupted(database, type_name, chunk): + if seen: + raise KeyboardInterrupt + seen.append(1) + return real_rows(database, type_name, chunk) + + monkeypatch.setattr(extras, "insert_parallel", second_chunk_interrupted) + rows = [{"id": i, "txt": "t"} for i in range(25_000)] + with pytest.raises(KeyboardInterrupt): + temp_db.insert_many("Col", rows, parallel=True) + assert temp_db.async_executor().is_processing() is False + assert _count(temp_db, "Col") == 10_000 # the first chunk, whole + assert len(tokens.made) == 1 and tokens.read == tokens.made + + +def test_a_wait_that_fails_still_reads_the_tokens(temp_db, tokens, monkeypatch): + _type(temp_db) + real_wait = extras.NativeAsyncExecutor.wait_completion + state = {"raised": False} + + def failing_wait(self, timeout_ms=None): + if not state["raised"]: + state["raised"] = True + raise RuntimeError("the wait failed") + return real_wait(self, timeout_ms) + + monkeypatch.setattr(extras.NativeAsyncExecutor, "wait_completion", failing_wait) + n = 2_000 + with pytest.raises(ArcadeDBError, match="the wait failed"): + temp_db.insert_columns( + "Col", {"id": np.arange(n, dtype=np.int64), "txt": ["t"] * n}, parallel=True + ) + assert len(tokens.made) == 1 and tokens.read == tokens.made + temp_db.async_executor().wait_completion() + assert _count(temp_db, "Col") == n + + +def test_a_failed_parallel_batch_in_a_caller_transaction_does_not_promise_a_rollback( + temp_db, +): + import pyarrow as pa + + _type(temp_db, unique=True) + ids = list(range(300)) + ids[250] = 7 # a duplicate key in the third batch of 100 + table = pa.table({"id": pa.array(ids, pa.int64()), "txt": ["t"] * 300}) + temp_db.begin() + try: + with pytest.raises(ArcadeDBError) as err: + temp_db.insert_arrow("Col", table, batch_size=100, parallel=True) + message = str(err.value) + assert "not committed" not in message and "your transaction" in message + assert "async writers" in message and "does not remove them" in message + finally: + temp_db.rollback() + temp_db.async_executor().wait_completion() + assert ( + _count(temp_db, "Col") >= 200 + ) # the writers committed the first two batches: the rollback did not remove them diff --git a/tests/test_native_columnar_temporal.py b/tests/test_native_columnar_temporal.py new file mode 100644 index 0000000..7ae3b54 --- /dev/null +++ b/tests/test_native_columnar_temporal.py @@ -0,0 +1,326 @@ +"""Native: a column of date-times, dates, Decimals or bytes stays on the columnar entry and stores what the row path stores (#316). + +Before, one such column made ``columnar.pack`` return None, so every column of the batch went through one dict per row and a JSON text +with tagged values. Now the kinds T, D, n and x of ``arcadedb_n_insert_colp`` carry them and the image builds the same objects +(``LocalDateTime``, ``LocalDate``, ``BigDecimal``, ``byte[]``) that ``Json.decodeTag`` builds for the tagged values. Each test loads the +same data through the row path (the private reference: the columnar path switched off, so the chunk goes as tagged JSON) and through +the columns, compares every stored value and its type, and spies on the entry points so a silent fall back cannot pass. Needs the +native library, so conftest.py collects this file on the native backend only. +""" + +import datetime as dt +import decimal + +import pytest + +from arcadedb._native import bulk, columnar, lib # noqa: E402 + +COLP, ROWS, MANY = ( + "arcadedb_n_insert_colp", + "arcadedb_n_insert_rows", + "arcadedb_n_exec_many", +) +UTC9 = dt.timezone(dt.timedelta(hours=9)) + + +class Spy: + def __init__(self, monkeypatch): + self.calls = [] + real = lib.call + + def call(name, *args): + self.calls.append(name) + return real(name, *args) + + monkeypatch.setattr(lib, "call", call) + + def count(self, name): + return self.calls.count(name) + + +@pytest.fixture +def spy(monkeypatch): + return Spy(monkeypatch) + + +def _type(db, name, declared=True): + db.command("sql", f"CREATE DOCUMENT TYPE {name}") + db.command("sql", f"CREATE PROPERTY {name}.id INTEGER") + if declared: + for prop, ty in ( + ("ts", "DATETIME_MICROS"), + ("ns", "DATETIME_NANOS"), + ("day", "DATE"), + ("amount", "DECIMAL"), + ("blob", "BINARY"), + ): + db.command("sql", f"CREATE PROPERTY {name}.{prop} {ty}") + + +def _stored(db, name): + """Every record by id as {property: (python type name, repr)}, so a changed type is as visible as a changed value.""" + sql = f"SELECT FROM {name} ORDER BY id" # nosec B608 - test-owned type name + return [ + { + k: (type(v).__name__, repr(v)) + for k, v in sorted(rec.items()) + if not k.startswith("@") and k != "rid" + } + for rec in db.query("sql", sql).to_list() + ] + + +def _rows(columns): + return [dict(zip(columns, vals)) for vals in zip(*columns.values())] + + +def _reference(db, name, rows, monkeypatch): + with monkeypatch.context() as m: + m.setattr(bulk, "_ENABLED", False) + db.insert_many(name, rows) + + +def _columns(n=60): + base = dt.datetime(1969, 12, 31, 23, 59, 59, 999999) + return { + "id": list(range(n)), + "ts": [ + ( + None + if i % 11 == 0 + else (base + dt.timedelta(seconds=i * 86_399, microseconds=i * 7)) + ) + for i in range(n) + ], + "day": [ + None if i % 13 == 0 else dt.date(1960, 1, 1) + dt.timedelta(days=i * 401) + for i in range(n) + ], + "amount": [ + ( + None + if i % 9 == 0 + else decimal.Decimal(f"{i * 1234567}.{i % 100:02d}") + * (-1 if i % 2 else 1) + ) + for i in range(n) + ], + "blob": [ + None if i % 8 == 0 else bytes(range(i % 256)) + b"\x00\xff" + for i in range(n) + ], + } + + +@pytest.mark.parametrize("declared", [True, False], ids=["declared", "schemaless"]) +def test_a_batch_with_temporal_decimal_and_bytes_columns_stays_columnar( + temp_db, spy, monkeypatch, declared +): + for name in ("Ref", "ByRows", "ByColumns"): + _type(temp_db, name, declared) + columns = _columns() + _reference(temp_db, "Ref", _rows(columns), monkeypatch) + assert ( + spy.count(COLP) == 0 and spy.count(ROWS) == 1 + ) # the reference is the JSON path + spy.calls.clear() + assert temp_db.insert_many("ByRows", _rows(columns)) == 60 + assert spy.count(COLP) == 1 and spy.count(ROWS) == 0 and spy.count(MANY) == 0 + spy.calls.clear() + assert temp_db.insert_columns("ByColumns", columns) == 60 + assert spy.count(COLP) == 1 and spy.count(ROWS) == 0 + expected = _stored(temp_db, "Ref") + assert len(expected) == 60 and expected[1]["ts"][0] == "datetime" + assert _stored(temp_db, "ByRows") == expected + assert _stored(temp_db, "ByColumns") == expected + + +def test_time_zone_aware_and_naive_datetimes_follow_the_json_rule( + temp_db, spy, monkeypatch +): + """An aware date-time is its UTC wall clock, a naive one is taken as it is (_tag_datetime): the column does not decide a zone.""" + _type(temp_db, "Ref") + _type(temp_db, "Col") + values = [ + dt.datetime(2020, 1, 2, 12, 0, 0, 5, tzinfo=UTC9), + dt.datetime(1969, 12, 31, 23, 0, 0, tzinfo=dt.timezone.utc), + dt.datetime(2020, 6, 1, 0, 0, 0, tzinfo=dt.timezone(dt.timedelta(hours=-7))), + ] + rows = [{"id": i, "ts": v} for i, v in enumerate(values)] + _reference(temp_db, "Ref", rows, monkeypatch) + spy.calls.clear() + temp_db.insert_many("Col", rows) + assert spy.count(COLP) == 1 + assert _stored(temp_db, "Col") == _stored(temp_db, "Ref") + + +def test_pandas_timestamps_keep_microseconds_as_the_json_path_does( + temp_db, spy, monkeypatch +): + import pandas as pd + + _type(temp_db, "Ref") + _type(temp_db, "Col") + stamps = [ + pd.Timestamp("2026-05-06 07:08:09.123456789"), + pd.Timestamp("1960-05-06 07:08:09.123456", tz="Asia/Seoul"), + ] + rows = [{"id": i, "ts": v} for i, v in enumerate(stamps)] + _reference(temp_db, "Ref", rows, monkeypatch) + spy.calls.clear() + temp_db.insert_many("Col", rows) + assert spy.count(COLP) == 1 and spy.count(ROWS) == 0 + assert _stored(temp_db, "Col") == _stored(temp_db, "Ref") + + +@pytest.mark.parametrize( + "column, colp_runs", + [ + ([dt.datetime(2020, 1, 1), dt.date(2020, 1, 1)], 0), # a date-time and a date + # A non-finite Decimal: the column is refused (BigDecimal("NaN") would throw mid-batch in the image), so the NaN row goes to the + # statement on its own, as before. The row around it is a run of carriable rows: it goes in bulk, and the columnar entry + # carries a finite Decimal now, so that one run crosses on arcadedb_n_insert_colp (before, it was the JSON entry). + ([decimal.Decimal("1.5"), decimal.Decimal("NaN")], 1), + ([b"a", "b"], 0), # bytes and text + ], + ids=["datetime-and-date", "nan-decimal", "bytes-and-text"], +) +def test_a_column_the_buffers_cannot_carry_unchanged_still_falls_back( + temp_db, spy, monkeypatch, column, colp_runs +): + assert columnar.pack({"x": column}, columnar.Declared(known={})) is None + _type(temp_db, "Ref", declared=False) + _type(temp_db, "Col", declared=False) + rows = [{"id": i, "x": v} for i, v in enumerate(column)] + try: + _reference(temp_db, "Ref", rows, monkeypatch) + except ( + Exception + ): # the statement path may refuse it; then the new path must refuse it the same way + with pytest.raises(Exception): + temp_db.insert_many("Col", rows) + assert spy.count(COLP) == colp_runs + return + spy.calls.clear() + temp_db.insert_many("Col", rows) + assert spy.count(COLP) == colp_runs + assert _stored(temp_db, "Col") == _stored(temp_db, "Ref") + + +def _arrow_table(pa): + import numpy as np + + n = 40 + ids = np.arange(n, dtype=np.int64) + epoch_us = (ids - 20) * 1_000_003_000 + 7 # microseconds, before and after 1970 + mask = ids % 6 == 0 + + def stamps(unit, tz=None): + raw = { + "s": epoch_us // 1_000_000, + "ms": epoch_us // 1_000, + "us": epoch_us, + "ns": epoch_us * 1_000, + }[unit] + return pa.array(raw.astype(np.int64), mask=mask).cast(pa.timestamp(unit, tz=tz)) + + return pa.table( + { + "id": ids, + "ts": stamps("us"), + "ns": stamps("ns"), + "sec": stamps("s"), + "zoned": stamps("ms", tz="Asia/Seoul"), + "day": pa.array( + ((ids - 10) * 397).astype(np.int32), mask=ids % 5 == 0 + ).cast(pa.date32()), + "amount": pa.array( + [ + None if i % 7 == 0 else decimal.Decimal(f"{i}.{i % 100:02d}") + for i in range(n) + ], + type=pa.decimal128(20, 2), + ), + "blob": pa.array( + [None if i % 4 == 0 else bytes(range(i)) for i in range(n)] + ), + } + ) + + +def test_arrow_timestamps_dates_decimals_and_binary_stay_columnar( + temp_db, spy, monkeypatch +): + import pyarrow as pa + + for name in ("Ref", "Col"): + _type(temp_db, name) + for prop, ty in (("sec", "DATETIME_SECOND"), ("zoned", "DATETIME")): + temp_db.command("sql", f"CREATE PROPERTY {name}.{prop} {ty}") + table = _arrow_table(pa) + with ( + monkeypatch.context() as m + ): # the reference: tagged values through the JSON path + m.setattr(bulk, "_ENABLED", False) + assert temp_db.insert_arrow("Ref", table) == 40 + assert spy.count(COLP) == 0 + spy.calls.clear() + assert temp_db.insert_arrow("Col", table) == 40 + assert spy.count(COLP) == 1 and spy.count(ROWS) == 0 and spy.count(MANY) == 0 + expected = _stored(temp_db, "Ref") + assert len(expected) == 40 and expected[1]["ns"][0] == "datetime" + assert _stored(temp_db, "Col") == expected + + +def test_an_arrow_batch_with_one_list_column_falls_back_with_the_same_values( + temp_db, spy, monkeypatch +): + """A list column cannot cross as a buffer: the whole batch takes the row path, and the temporal columns reach it as tagged values.""" + import pyarrow as pa + + for name in ("Ref", "Col"): + _type(temp_db, name) + for prop, ty in (("sec", "DATETIME_SECOND"), ("zoned", "DATETIME")): + temp_db.command("sql", f"CREATE PROPERTY {name}.{prop} {ty}") + table = _arrow_table(pa).append_column( + "tags", pa.array([["a", "b"]] * 40, type=pa.list_(pa.string())) + ) + with monkeypatch.context() as m: + m.setattr(bulk, "_ENABLED", False) + temp_db.insert_arrow("Ref", table) + spy.calls.clear() + temp_db.insert_arrow("Col", table) + assert spy.count(COLP) == 0 + assert _stored(temp_db, "Col") == _stored(temp_db, "Ref") + + +def test_a_parallel_load_of_datetimes_is_one_columnar_crossing(temp_db, spy): + _type(temp_db, "Par") + columns = _columns(300) + assert temp_db.insert_columns("Par", columns, parallel=True) == 300 + assert spy.count(COLP) == 1 and spy.count(ROWS) == 0 + assert temp_db.query("sql", "SELECT count(*) AS n FROM Par").first().get("n") == 300 + + +def test_the_image_refuses_a_wrong_ticks_per_second_and_writes_nothing(temp_db): + import json + + _type(temp_db, "Col") + rows, packed = columnar.pack( + {"id": [1, 2], "ts": [dt.datetime(2020, 1, 1)] * 2}, columnar.Declared(known={}) + ) + assert packed[1].kind == "T" and packed[1].dim == 1_000_000 + packed[1].dim = 7 + spec = columnar.spec_of(packed) + reply = json.loads( + lib.call( + COLP, + temp_db._h, + lib.b("Col"), + lib.b(columnar.header_of(rows, 10_000, packed)), + spec, + len(spec), + ) + ) + assert not reply["ok"] and "ticks per second" in reply["error"] + assert temp_db.query("sql", "SELECT count(*) AS n FROM Col").first().get("n") == 0 diff --git a/tests/test_native_columnar_temporal_review_fixes.py b/tests/test_native_columnar_temporal_review_fixes.py new file mode 100644 index 0000000..34898f9 --- /dev/null +++ b/tests/test_native_columnar_temporal_review_fixes.py @@ -0,0 +1,190 @@ +"""Review of the temporal columnar kinds (#333): a value the image cannot take is refused in Python, before the crossing, and never wraps. + +``T`` and ``D`` columns carry integers that the image turns into ``LocalDateTime`` and ``LocalDate``. A ``date64`` past int32 days used +to wrap in the int32 cast and a ``timestamp[s]`` past the years of ``java.time`` threw ``DateTimeException`` in the middle of a batch. Now +``columnar.pack`` returns None for both, so the call takes the row path, which raises what it raised before (nothing stored, no +transaction left open on the synchronous call; ``may have been stored`` for a parallel call, which has handed rows over by then). Also +pinned: a chunk with a date-time column and a float column holding NaN goes columnar now and stores the same NaN that ``INSERT INTO t +CONTENT :row`` stores (both end in ``MutableDocument.putProperty``). Needs the native library: conftest.py collects this file on the +native backend only. +""" + +import datetime as dt +import decimal +import math + +import numpy as np +import pytest + +from arcadedb._native import columnar, lib # noqa: E402 +from arcadedb.exceptions import ArcadeDBError + +pa = pytest.importorskip("pyarrow") + +COLP = "arcadedb_n_insert_colp" +DAY_MS = 86_400_000 +LDT_MIN_SECONDS, LDT_MAX_SECONDS = -31_557_014_167_219_200, 31_556_889_864_403_199 +NO_TYPES = columnar.Declared(known={}) + + +class Spy: + def __init__(self, monkeypatch): + self.calls = [] + real = lib.call + + def call(name, *args): + self.calls.append(name) + return real(name, *args) + + monkeypatch.setattr(lib, "call", call) + + +def _count(db, name): + sql = f"SELECT count(*) AS n FROM {name}" # nosec B608 - test-owned type name + return db.query("sql", sql).first().get("n") + + +def _kinds(columns): + packed = columnar.pack(columns, NO_TYPES) + return None if packed is None else [c.kind for c in packed[1]] + + +def test_a_date64_past_int32_days_is_not_laid_out_and_never_wraps(): + wraps = pa.array([DAY_MS * (2**31 + 5)], pa.int64()).cast(pa.date64()) + assert columnar.pack({"d": wraps}, NO_TYPES) is None + below = pa.array([DAY_MS * (-(2**31) - 1)], pa.int64()).cast(pa.date64()) + assert columnar.pack({"d": below}, NO_TYPES) is None + one_bad = pa.array([0, DAY_MS, DAY_MS * 2**33], pa.int64()).cast(pa.date64()) + assert columnar.pack({"d": one_bad}, NO_TYPES) is None # one value is enough + + +def test_a_date64_at_the_int32_limits_stays_columnar_and_exact(): + edge = pa.array( + [DAY_MS * (2**31 - 1), DAY_MS * -(2**31), None], pa.int64() + ).cast(pa.date64()) + rows, packed = columnar.pack({"d": edge}, NO_TYPES) + assert rows == 3 and packed[0].kind == "D" + import ctypes + + import numpy as np + + addr, length = packed[0].data + days = np.frombuffer(ctypes.string_at(addr, length), dtype=np.int32) + assert days.tolist() == [2**31 - 1, -(2**31), 0] + + +def test_a_timestamp_past_the_years_of_java_time_is_not_laid_out(): + assert ( + columnar.pack({"t": pa.array([0, 2**62], pa.timestamp("s"))}, NO_TYPES) is None + ) + over = pa.array([LDT_MAX_SECONDS + 1], pa.timestamp("s")) + assert columnar.pack({"t": over}, NO_TYPES) is None + under = pa.array([LDT_MIN_SECONDS - 1], pa.timestamp("s")) + assert columnar.pack({"t": under}, NO_TYPES) is None + + +def test_a_timestamp_at_the_limits_stays_columnar(): + edge = pa.array([LDT_MAX_SECONDS, LDT_MIN_SECONDS, None], pa.timestamp("s")) + assert _kinds({"t": edge}) == ["T"] + # an int64 of milliseconds, microseconds or nanoseconds is always inside the years of java.time + for unit in ("ms", "us", "ns"): + big = pa.array([2**63 - 1, -(2**63)], pa.timestamp(unit)) + assert _kinds({"t": big}) == ["T"], unit + + +@pytest.mark.parametrize("text", ["NaN", "-NaN", "sNaN", "Infinity", "-Infinity"]) +def test_a_non_finite_decimal_is_never_laid_out(text): + # new BigDecimal("NaN") throws in the image, in the middle of a batch: the column must stay in Python, in every container + bad = decimal.Decimal(text) + assert columnar.pack({"x": [decimal.Decimal("1.5"), bad]}, NO_TYPES) is None + assert columnar.pack({"x": [None, bad, None]}, NO_TYPES) is None + as_object = np.array([decimal.Decimal("1"), bad], dtype=object) + assert columnar.pack({"x": as_object}, NO_TYPES) is None + finite = columnar.pack({"x": [decimal.Decimal("1.5"), None]}, NO_TYPES) + assert finite is not None and finite[1][0].kind == "n" # the finite column crosses + + +def _type(db, name): + db.command("sql", f"CREATE DOCUMENT TYPE {name}") + db.command("sql", f"CREATE PROPERTY {name}.id INTEGER") + + +def test_a_timestamp_the_image_cannot_take_stores_nothing_and_leaves_no_transaction( + temp_db, +): + _type(temp_db, "Col") + table = pa.table( + { + "id": pa.array([1, 2], pa.int64()), + "t": pa.array([0, 2**62], pa.timestamp("s")), + } + ) + with pytest.raises(ArcadeDBError): + temp_db.insert_arrow("Col", table) + assert not temp_db.is_transaction_active() + assert _count(temp_db, "Col") == 0 + + +def test_a_date_the_image_cannot_take_stores_nothing_and_leaves_no_transaction(temp_db): + _type(temp_db, "Col") + wraps = pa.array([0, DAY_MS * (2**31 + 5)], pa.int64()).cast(pa.date64()) + table = pa.table({"id": pa.array([1, 2], pa.int64()), "d": wraps}) + with pytest.raises(ArcadeDBError): + temp_db.insert_arrow("Col", table) + assert not temp_db.is_transaction_active() + assert _count(temp_db, "Col") == 0 + + +def test_a_parallel_load_of_such_a_timestamp_says_that_rows_may_have_been_stored( + temp_db, +): + _type(temp_db, "Col") + table = pa.table( + { + "id": pa.array([1, 2], pa.int64()), + "t": pa.array([0, 2**62], pa.timestamp("s")), + } + ) + with pytest.raises(ArcadeDBError, match="may have been stored"): + temp_db.insert_arrow("Col", table, parallel=True) + temp_db.async_executor().wait_completion() + assert ( + _count(temp_db, "Col") <= 1 + ) # the row before the bad one may have been handed over + + +def test_a_datetime_chunk_with_nan_stores_the_nan_the_statement_path_stores( + temp_db, monkeypatch +): + for name in ("Col", "Ref"): + _type(temp_db, name) + temp_db.command("sql", f"CREATE PROPERTY {name}.ts DATETIME_MICROS") + temp_db.command("sql", f"CREATE PROPERTY {name}.val DOUBLE") + rows = [ + { + "id": i, + "ts": dt.datetime(2020, 1, 1, 0, 0, i), + "val": float("nan") if i % 2 else 1.5 * i, + } + for i in range(10) + ] + spy = Spy(monkeypatch) + temp_db.insert_many("Col", rows) + assert ( + spy.calls.count(COLP) == 1 + ) # the date-time column no longer forces the JSON path + sql = "INSERT INTO `Ref` CONTENT :row" + temp_db.begin() + temp_db._command_many("sql", sql, [{"row": row} for row in rows]) + temp_db.commit() + + def stored(name): + select = f"SELECT FROM {name} ORDER BY id" # nosec B608 - test-owned type name + return [ + (r["id"], type(r["val"]).__name__, repr(r["val"]), repr(r["ts"])) + for r in temp_db.query("sql", select).to_list() + ] + + got, want = stored("Col"), stored("Ref") + assert got == want + assert all(math.isnan(float(v[2])) for v in got if v[0] % 2) diff --git a/tests/test_native_columns_reference.py b/tests/test_native_columns_reference.py index f1f68ba..14d5612 100644 --- a/tests/test_native_columns_reference.py +++ b/tests/test_native_columns_reference.py @@ -1,372 +1,16 @@ -"""The native columnar readers give the same table as the rules they replaced (refs #312). +"""The drain of a native result set (``NativeResultSet._drain_dicts``) returns the rows row-by-row iteration returns (refs #321). -``arcadedb._native.columns`` infers an Arrow column with ``pa.array(values)`` first and falls back to the per-value -rules (``_kind``) when inference cannot type the column or types it differently; ``to_columns`` calls ``_kind`` once -per column; ``_names`` skips the per-key step for a row whose keys are known; ``NativeResultSet._drain_dicts`` takes -the rows of a reply without one tuple per row. None of that may change a value, a dtype or a column order. +``_drain_dicts`` takes the rows of a reply without building one tuple per row (``_fill(into)``). That may not change a row or +their order, also after some rows were taken one by one, or on an empty or an already drained result. -The reference functions below are the code as it was before: the rules of ``_kind``, ``_names``, ``to_columns`` and -``to_arrow``, copied. ``_ms`` and ``_float_vectors`` were not changed and are shared with the module. Each case builds -rows, runs both and compares the outcome, an exception included (a column that raised before raises the same class now). -Pure Python on the rows: no database is needed except for the drain test at the end. +This file was ``arcadedb._native.columns``'s reference test (#321); that module is gone since the column frames (#331), whose own +reference test is ``test_native_column_frames_reference.py``. Only the tests of the drain remain here. -conftest.py collects this file on the native backend only (the module lives in the native package). +conftest.py collects this file on the native backend only. """ -import random -from datetime import date, datetime, timedelta, timezone -from decimal import Decimal - -import numpy as np -import pyarrow as pa import pytest -from arcadedb._native import columns as new -from arcadedb._native.columns import _float_vectors, _ms - -# --- the reference: the rules before this change -------------------------------------------------------------------------- - - -def _names_ref(rows, columns): - if columns is not None: - return list(columns) - seen = {} - for row in rows: - for k in row: - seen.setdefault(k, None) - return list(seen) - - -def _kind_ref(values): - kinds = set() - for v in values: - if v is None: - continue - t = type(v) - if t is bool: - kinds.add("b") - elif t is int: - kinds.add("i") - elif t is float: - kinds.add("f") - elif t is datetime or t is date: - kinds.add("dt") - elif t is Decimal: - kinds.add("dec") - else: - kinds.add("obj") - if not kinds: - return "obj" - if len(kinds) == 1: - return kinds.pop() - if kinds == {"i", "f"}: - return "f" - return "obj" - - -def _to_columns_ref(rows, columns=None): - out = {} - for name in _names_ref(rows, columns): - values = [row.get(name) for row in rows] - if _kind_ref(values) == "obj": - vectors = _float_vectors(values) - if vectors is not None: - out[name] = vectors - continue - has_null = any(v is None for v in values) - kind = _kind_ref(values) - if kind == "i": - out[name] = ( - np.array([np.nan if v is None else v for v in values], dtype=np.float64) - if has_null - else np.array(values, dtype=np.int64) - ) - elif kind == "f": - out[name] = np.array( - [np.nan if v is None else v for v in values], dtype=np.float64 - ) - elif kind == "b": - out[name] = values if has_null else np.array(values, dtype=bool) - elif kind == "dt": - out[name] = np.array( - [ - ( - np.datetime64("NaT", "ms") - if v is None - else np.datetime64(_ms(v), "ms") - ) - for v in values - ], - dtype="datetime64[ms]", - ) - elif kind == "dec": - arr = np.empty(len(values), dtype=object) - arr[:] = values - out[name] = arr - else: - out[name] = values - return out - - -def _to_arrow_ref(rows, columns=None): - arrays = {} - for name in _names_ref(rows, columns): - values = [row.get(name) for row in rows] - kind = _kind_ref(values) - if kind == "i": - arrays[name] = pa.array(values, type=pa.int64()) - elif kind == "f": - try: - arrays[name] = pa.array(values, type=pa.float64()) - except pa.ArrowInvalid: - arrays[name] = pa.array( - [None if v is None else str(v) for v in values], type=pa.string() - ) - elif kind == "b": - arrays[name] = pa.array(values, type=pa.bool_()) - elif kind == "dt": - arrays[name] = pa.array( - [None if v is None else _ms(v) for v in values], type=pa.timestamp("ms") - ) - elif kind == "dec": - try: - arrays[name] = pa.array(values) - except pa.ArrowInvalid: - arrays[name] = pa.array( - [None if v is None else str(v) for v in values], type=pa.string() - ) - else: - try: - arrays[name] = pa.array([None if v is None else v for v in values]) - except (pa.ArrowInvalid, pa.ArrowTypeError): - arrays[name] = pa.array( - [None if v is None else str(v) for v in values], type=pa.string() - ) - return pa.table(arrays) - - -# --- the inputs ----------------------------------------------------------------------------------------------------------- - -BIG = 2**53 + 1 -STAMP = datetime(2026, 1, 2, 3, 4, 5, 678901) -AWARE = datetime(2026, 1, 2, 12, 0, 0, tzinfo=timezone(timedelta(hours=9))) -WIDE40 = Decimal("1234567890123456789012345678901234567890.5") # decimal256 -WIDE80 = Decimal("1" * 80) # wider than decimal256: exact text - -# one column per case, each a list of values that go in rows as {"c": value} -COLUMNS = { - "ints": [1, 2, 3], - "ints-null": [1, None, 3], - "ints-int64-edges": [-(2**63), 2**63 - 1, 0], - "int-above-int64": [2**63, 1], - "int-above-uint64": [2**64, 1], - "int-below-int64": [-(2**63) - 1, 1], - "floats": [0.5, 1.5, float("nan")], - "floats-null": [0.5, None, float("inf")], - "ints-and-floats": [1, 2.5, None, 3], - "int-beyond-2-53-and-floats": [BIG, 0.5], - "big-int-beyond-2-53-and-floats": [2**60, 1.5, None], - "int-above-int64-and-floats": [2**63, 0.5], - "int-above-uint64-and-floats": [2**64, 1.5, None], - "int-below-int64-and-floats": [-(2**63) - 1, 0.5], - "int-above-int64-and-string": [2**63, "x"], - "bools": [True, False, True], - "bools-null": [True, None, False], - "bool-and-int": [True, 1, 0], - "bool-and-float": [True, 0.5], - "strings": ["a", "bb", ""], - "strings-null": ["a", None, "c"], - "string-and-int": ["seven", 1], - "all-null": [None, None, None], - "single-null": [None], - "datetimes": [STAMP, datetime(1969, 12, 31, 23, 59, 59, 999000)], - "datetimes-null": [STAMP, None], - "dates": [date(2026, 1, 2), date(1969, 12, 31)], - "date-and-datetime": [date(2026, 1, 2), STAMP, None], - "aware-datetime": [AWARE, STAMP], - "datetime-and-string": [STAMP, "x"], - "decimals": [Decimal("1.50"), Decimal("-0.25")], - "decimals-null": [Decimal("2"), None, Decimal("30")], - "decimal-40-digits": [WIDE40, Decimal("1")], - "decimal-80-digits": [WIDE80, Decimal("1")], - "decimal-and-int": [Decimal("1.5"), 2], - "vectors": [[0.5, 1.5], [0.25, 2.0]], - "vectors-null": [[0.5, 1.5], None, [0.25, 2.0]], - "vector-not-float32": [[0.1, 0.2], [0.3, 0.4]], - "vector-ragged": [[0.5], [0.5, 1.5]], - "lists-of-ints": [[1, 2], [3]], - "lists-mixed": [[1, "a", 2.5], ["x"]], - "empty-lists": [[], []], - "dicts": [{"a": 1}, {"b": [1.5, None]}], - "nested-float-lists": [[[0.5, 1.5]], [[2.5]]], - "list-of-datetimes": [[STAMP], [None]], - "bytes-like": [b"ab", b"c"], -} - - -def _rows(values, name="c"): - return [{name: v} for v in values] - - -def _wide_null_heavy(seed=7, count=1000): - """Many rows, ragged keys, about nine values in ten missing or None, three vectors, strings, ints and floats.""" - rng = random.Random(seed) # nosec B311 - fixed test data, not a secret - rows = [] - for i in range(count): - row = {} - if rng.random() < 0.9: - row["i"] = rng.randrange(-1000, 1000) if rng.random() < 0.9 else None - if rng.random() < 0.1: - row["f"] = rng.random() * 10 - if rng.random() < 0.1: - row["s"] = f"s{i}" if rng.random() < 0.8 else None - if rng.random() < 0.1: - row["late-" + str(i % 3)] = i - if rng.random() < 0.1: - row["v"] = [float(np.float32(rng.random())) for _ in range(4)] - if rng.random() < 0.05: - row["m"] = rng.choice([1, 2.5, "x", None, True]) - rows.append(row) - return rows - - -# --- the comparison ------------------------------------------------------------------------------------------------------- - - -def _outcome(fn, *args): - try: - return ("ok", fn(*args)) - except ( - Exception - ) as exc: # noqa: BLE001 - the class of the failure is part of the result - return ("raises", type(exc)) - - -def _same_table(a, b): - assert a.schema == b.schema, (a.schema, b.schema) - assert a.column_names == b.column_names - # not Table.equals: it is False for a NaN next to the same NaN. repr counts NaN, -0.0 and the type of every value - assert repr(a.to_pydict()) == repr(b.to_pydict()) - - -def _same_column(a, b, name): - assert type(a) is type(b), (name, type(a), type(b)) - if isinstance(a, np.ndarray): - assert a.dtype == b.dtype and a.shape == b.shape, (name, a.dtype, b.dtype) - assert repr(a.tolist()) == repr(b.tolist()), name - else: - assert repr(a) == repr(b), name - - -def _check_arrow(rows, columns=None): - want = _outcome(_to_arrow_ref, rows, columns) - have = _outcome(new.to_arrow, rows, columns) - assert have[0] == want[0], (have, want) - if want[0] == "ok": - _same_table(have[1], want[1]) - else: - assert have[1] is want[1] - - -def _check_columns(rows, columns=None): - want = _outcome(_to_columns_ref, rows, columns) - have = _outcome(new.to_columns, rows, columns) - assert have[0] == want[0], (have, want) - if want[0] == "ok": - assert list(have[1]) == list(want[1]) - for name in want[1]: - _same_column(have[1][name], want[1][name], name) - else: - assert have[1] is want[1] - - -@pytest.mark.parametrize("case", sorted(COLUMNS)) -def test_a_single_column_gives_the_same_arrow_table(case): - _check_arrow(_rows(COLUMNS[case])) - - -@pytest.mark.parametrize("case", sorted(COLUMNS)) -def test_a_single_column_gives_the_same_columns(case): - _check_columns(_rows(COLUMNS[case])) - - -def test_a_table_of_all_the_cases_gives_the_same_arrow_table_and_columns(): - names = sorted(COLUMNS) - count = max(len(COLUMNS[n]) for n in names) - rows = [ - {n: COLUMNS[n][i] for n in names if i < len(COLUMNS[n])} for i in range(count) - ] - # a case that raises on its own raises for the table: compare the table without the columns that raise - ok = [n for n in names if _outcome(_to_arrow_ref, _rows(COLUMNS[n]))[0] == "ok"] - rows = [{n: r[n] for n in ok if n in r} for r in rows] - _check_arrow(rows) - ok_columns = [ - n for n in ok if _outcome(_to_columns_ref, _rows(COLUMNS[n]))[0] == "ok" - ] - _check_columns([{n: r[n] for n in ok_columns if n in r} for r in rows]) - - -@pytest.mark.parametrize("seed", [1, 7, 42]) -def test_a_null_heavy_ragged_table_gives_the_same_tables(seed): - rows = _wide_null_heavy(seed) - _check_arrow(rows) - _check_columns(rows) - - -def test_a_chosen_column_list_gives_the_same_tables(): - rows = _wide_null_heavy() - for columns in (["i", "s"], ["v", "nope", "i"], []): - _check_arrow(rows, columns) - _check_columns(rows, columns) - - -@pytest.mark.parametrize("columns", [None, ["a"]]) -def test_no_rows_gives_the_same_tables(columns): - _check_arrow([], columns) - _check_columns([], columns) - - -def test_rows_of_empty_dicts_give_the_same_tables(): - _check_arrow([{}, {}]) - _check_columns([{}, {}]) - - -def test_names_are_the_keys_in_the_order_they_are_first_seen(): - rows = [{"b": 1}, {"a": 1, "b": 2}, {"b": 3}, {"c": 1, "a": 2}, {"d": 1, "b": 1}] - assert new._names(rows, None) == _names_ref(rows, None) == ["b", "a", "c", "d"] - rows = _wide_null_heavy() - assert new._names(rows, None) == _names_ref(rows, None) - assert new._names(rows, ["z", "y"]) == ["z", "y"] - - -def test_the_inference_only_takes_a_column_the_rules_type_the_same(): - """A temporal column and an int above int64 are left to the rules; the rest is the inferred array.""" - assert new._inferred([STAMP, None]) is None - assert new._inferred([date(2026, 1, 2)]) is None - assert new._inferred([2**63]) is None - assert new._inferred([1, None, 3]).type == pa.int64() - assert new._inferred([0.5, 1]).type == pa.float64() - assert new._inferred([True, None]).type == pa.bool_() - assert new._inferred(["a", None]).type == pa.string() - assert new._inferred([None, None]).type == pa.null() - - -def test_to_columns_asks_for_the_kind_of_a_column_once(monkeypatch): - calls = [] - real = new._kind - - def counting(values): - calls.append(len(values)) - return real(values) - - monkeypatch.setattr(new, "_kind", counting) - new.to_columns(_rows([1, None, 3]), None) - assert calls == [3] - calls.clear() - new.to_columns(_rows([[0.5, 1.5], None]), None) - assert calls == [2] - - # --- the drain ------------------------------------------------------------------------------------------------------------ DRAIN_ROWS = 5000 # more than the first reply (256) and two further replies (2048 each) @@ -414,15 +58,3 @@ def test_the_drain_of_an_empty_result_and_a_second_drain_are_empty(drain_db): rs = drain_db.query("sql", SELECT_DRAIN) assert len(rs._drain_dicts()) == DRAIN_ROWS assert rs._drain_dicts() == [] - - -def test_to_arrow_and_to_columns_of_a_result_equal_the_reference_over_its_rows( - drain_db, -): - rows = drain_db.query("sql", SELECT_DRAIN)._drain_dicts() - table = drain_db.query("sql", SELECT_DRAIN).to_arrow() - _same_table(table, _to_arrow_ref(rows)) - columns = drain_db.query("sql", SELECT_DRAIN).to_columns() - want = _to_columns_ref(rows) - for name in want: - _same_column(columns[name], want[name], name) diff --git a/tests/test_native_graph_batch_binary_vectors.py b/tests/test_native_graph_batch_binary_vectors.py index 429adf4..c554310 100644 --- a/tests/test_native_graph_batch_binary_vectors.py +++ b/tests/test_native_graph_batch_binary_vectors.py @@ -1,7 +1,8 @@ """GraphBatch float32 vectors cross the native library in a binary side buffer, not as number text. ``create_vertices`` rows and ``new_edges`` props that hold a float32 ``ndarray`` go through ``arcadedb_n_gb_bin`` (the float32 buffer is -read in Java through the ``["B", offset, count]`` tag, as ``arcadedb_n_exec_bin`` reads query parameters); everything else keeps the text +read in Java through the ``["B", offset, count]`` tag, as ``arcadedb_n_exec_bin`` reads query parameters; a ``create_vertices`` column of +float32 arrays of one length crosses as one matrix described by ``"vecs"`` instead, see test_native_vertex_vector_crossings.py); everything else keeps the text of ``arcadedb_n_gb``: a vector given as a list, a float64 array, a row with no vector. The stored values do not change: the same float32 as the ``"F"``-tagged text gave, bit for bit. A library without the entry (``lib.GB_BIN`` false) keeps the text path. @@ -95,9 +96,11 @@ def test_float32_vectors_take_the_binary_entry_and_are_stored_bit_equal(db, call assert calls.entries("create_vertices") == ["arcadedb_n_gb_bin"] _, _, text, count = next(c for c in calls.calls if c[1] == "create_vertices") assert count == N * dim - assert '"B"' in text and '"F"' not in text + assert ( + '"vecs"' in text and '"B"' not in text and '"F"' not in text + ) # one matrix, no tag per row (#337) assert str(float(x[2, 0])) not in text # the numbers are not in the text - assert len(text) < N * 80 # a vector is a tag, not dim numbers + assert len(text) < N * 80 # a row is its other properties, not dim numbers _assert_bit_equal(_stored(db, "Vx"), x) diff --git a/tests/test_native_ingest_integrity_faults.py b/tests/test_native_ingest_integrity_faults.py new file mode 100644 index 0000000..088d461 --- /dev/null +++ b/tests/test_native_ingest_integrity_faults.py @@ -0,0 +1,39 @@ +"""The native columnar fast path with a rows-dropped fault in it (ArcadeData/arcadedb-python#336). + +``arcadedb._native.bulk.insert_columns_fast`` is the one entry that ``insert_columns``, ``insert_arrow`` and ``insert_dataframe`` +share on the native build. Each test puts a fault into it and runs the same checks as ``test_ingest_integrity.py``: they must +report it. Needs the native library, so conftest.py collects this file on the native backend only. +""" + +import pytest + +from arcadedb._native import bulk +from tests.test_ingest_integrity import ( + BY_ID, + FOUND, + MUTATION_SIZE, + IntegrityError, + run_cases, +) + + +@pytest.mark.parametrize("fault", ["drop_the_last_row", "skip_the_call"]) +@pytest.mark.parametrize("family", ["cols", "arrow", "df"]) +def test_a_fault_in_the_columnar_fast_path_is_reported( + store, small_chunks, monkeypatch, family, fault +): + real = bulk.insert_columns_fast + seen = [] + + def faulty(db, type_name, columns, *args, **kwargs): + seen.append(type_name) + if fault == "skip_the_call": + return len(next(iter(columns.values()))) + short = {name: column[:-1] for name, column in columns.items()} + return real(db, type_name, short, *args, **kwargs) + + monkeypatch.setattr(bulk, "insert_columns_fast", faulty) + with pytest.raises(IntegrityError) as found: + run_cases(store, BY_ID[family], [MUTATION_SIZE]) + assert FOUND in str(found.value), found.value + assert seen, "the fault was not in the path" diff --git a/tests/test_native_mixed_chunk.py b/tests/test_native_mixed_chunk.py index db4162c..cf409cb 100644 --- a/tests/test_native_mixed_chunk.py +++ b/tests/test_native_mixed_chunk.py @@ -169,7 +169,8 @@ def test_a_pandas_timestamp_is_stored_as_the_statement_path_stores_it( _statement_path(db, "Ref", rows) spy.clear() db.insert_many("Bulk", rows) - assert spy.count(MANY) == 0 and spy.count(ROWS) == 1 + # one column of Timestamps is the kind T of the columnar entry (#316): microseconds, as the tagged path keeps them + assert spy.count(MANY) == 0 and spy.count(ROWS) == 0 and spy.count(COLP) == 1 assert _stored(db, "Bulk") == _stored(db, "Ref") @@ -249,8 +250,10 @@ def test_an_uncarriable_value_sends_its_own_row_to_the_statement_not_the_chunk( spy.clear() db.insert_many("Bulk", rows) assert spy.count(MANY) == 1, "one statement run for the one uncarriable row" + # the rows before and after it stay on the bulk entry; its columnar form carries the date-time column now, so each run + # crosses on arcadedb_n_insert_colp (before it was the JSON entry, arcadedb_n_insert_rows) assert ( - spy.count(ROWS) == 2 + spy.count(COLP) == 2 and spy.count(ROWS) == 0 ), "the rows before and after it stay on the bulk entry" assert _stored(db, "Bulk") == _stored(db, "Ref") diff --git a/tests/test_native_one_row_crossings.py b/tests/test_native_one_row_crossings.py new file mode 100644 index 0000000..73f32a9 --- /dev/null +++ b/tests/test_native_one_row_crossings.py @@ -0,0 +1,368 @@ +"""A created or updated record costs the fewest native crossings (conftest.py collects this file on the native backend only). + +``save()`` of a new record and ``Vertex.new_edge`` read the row the statement returns, so they used the lazy form of the command +(``arcadedb_n_cmd``: a cursor id, then a second crossing, ``arcadedb_n_next_bin``, for the row they always read). They now use the +form whose reply carries the first rows (``arcadedb_n_exec``): one crossing. ``save()`` of a saved record spent four crossings (the +transaction check, the SELECT of the stale-record check, and two for the UPDATE) and now spends three. The stale-record check still +reads the stored record first: the engine's reply to an UPDATE comes after the write, and the check must refuse before writing +anything inside the caller's transaction, so it cannot be read from the update's own result. + +The counts are crossings (the ctypes doors ``cmd``, ``call``, ``call_bin``, ``exec_query`` and the batched cursor close), not seconds: +deterministic. The oracle for the values and the errors is the lazy form itself (``db.command(...).first()``), kept for every other caller. +""" + +import contextlib +import datetime as dt +from decimal import Decimal + +import numpy as np +import pytest + +from arcadedb import ArcadeDBError +from arcadedb._native import extras, lib, records + + +class _Crossings: + """Every call into the native library made while installed, by entry point.""" + + def __init__(self, monkeypatch): + self.names = [] + for attr in ("cmd", "call", "call_bin", "exec_query"): + monkeypatch.setattr(lib, attr, self._spy(attr, getattr(lib, attr))) + real_close = lib._lib.arcadedb_n_rs_close_many + + def close_many(*args): + self.names.append("arcadedb_n_rs_close_many") + return real_close(*args) + + monkeypatch.setattr(lib._lib, "arcadedb_n_rs_close_many", close_many) + + def _spy(self, attr, real): + def spy(*args, **kwargs): + self.names.append(args[0] if attr in ("call", "call_bin") else attr) + return real(*args, **kwargs) + + return spy + + def reset(self): + lib.flush_closes() # a cursor closed earlier is not part of the call measured + del self.names[:] + + +@pytest.fixture +def db(temp_db): + temp_db.command("sql", "CREATE DOCUMENT TYPE D") + temp_db.command("sql", "CREATE VERTEX TYPE V") + temp_db.command("sql", "CREATE EDGE TYPE E") + return temp_db + + +@pytest.fixture +def crossings(db, monkeypatch): + return _Crossings(monkeypatch) + + +def _plain(values): + return {k: v for k, v in values.items() if not k.startswith("@")} + + +# -- create + + +def test_saving_a_new_document_is_one_crossing(db, crossings): + with db.transaction(): + doc = db.new_document("D") + doc.set("a", 1) + crossings.reset() + doc.save() + assert crossings.names == [ + "exec_query" + ] # was: arcadedb_n_cmd, then arcadedb_n_next_bin + assert doc.get_identity() is not None + + +def test_saving_a_new_vertex_with_a_float_array_is_one_crossing(db, crossings): + with db.transaction(): + vertex = db.new_vertex("V") + vertex.set("v", np.arange(16, dtype=np.float32)) + crossings.reset() + vertex.save() + assert crossings.names == [ + "arcadedb_n_exec_bin" + ] # the float side buffer: the long path, still one crossing + + +def test_new_edge_is_one_crossing_with_and_without_properties(db, crossings): + with db.transaction(): + a, b = db.new_vertex("V"), db.new_vertex("V") + a.save() + b.save() + crossings.reset() + plain = a.new_edge("E", b) + assert crossings.names == ["exec_query"] + crossings.reset() + weighted = a.new_edge("E", b, w=1.5, tag="x") + assert crossings.names == ["exec_query"] + assert plain.get_identity() != weighted.get_identity() + assert weighted.get("w") == 1.5 and weighted.get("tag") == "x" + + +CREATE_CASES = { + "ints and text": {"a": 1, "b": "text", "c": None, "d": True, "e": 2.5}, + "non-ascii": {"s": "café 日本語 \U0001f600"}, + "date and time": { + "d": dt.date(2026, 11, 9), + "t": dt.datetime(2026, 11, 9, 12, 30, 15, 123000), + }, + "decimal": {"x": Decimal("1.50")}, + "list and map": {"l": [1, 2, 3], "m": {"k": [1.5, "v"]}}, + "float array": {"v": np.arange(16, dtype=np.float32)}, + "nothing": {}, +} + + +@pytest.mark.parametrize("case", list(CREATE_CASES)) +def test_a_created_record_has_the_values_the_lazy_form_gives(db, case): + values = CREATE_CASES[case] + with db.transaction(): + doc = db.new_document("D") + for name, value in values.items(): + doc.set(name, value) + doc.save() + # the oracle: the same statement through the lazy command form every other caller still uses + want = db.command("sql", "INSERT INTO D CONTENT :row", {"row": values}).first() + got = db.lookup_by_rid(str(doc.get_identity())) + assert doc.get_type_name() == want.get_element().get_type_name() == "D" + assert _plain(doc.to_dict()) == _plain(want.to_dict()) == _plain(got.to_dict()) + + +def test_a_create_that_the_engine_refuses_fails_with_the_lazy_forms_message(db): + db.command("sql", "CREATE PROPERTY D.k STRING") + db.command("sql", "CREATE INDEX ON D (k) UNIQUE") + with db.transaction(): + db.command("sql", "INSERT INTO D SET k = 'dup'").close() + with pytest.raises(ArcadeDBError) as lazy: + with db.transaction(): + db.command( + "sql", "INSERT INTO D CONTENT :row", {"row": {"k": "dup"}} + ).first() + with pytest.raises(ArcadeDBError) as eager: + with db.transaction(): + doc = db.new_document("D") + doc.set("k", "dup") + doc.save() + assert str(eager.value) == str(lazy.value) + assert type(eager.value) is type(lazy.value) + + +# -- update + + +def _saved(db, **values): + with db.transaction(): + doc = db.new_document("D") + for name, value in values.items(): + doc.set(name, value) + doc.save() + return str(doc.get_identity()) + + +def test_updating_a_saved_record_is_three_crossings(db, crossings): + rid = _saved(db, a=1, name="x") + with db.transaction(): + record = db.lookup_by_rid(rid).modify() + record.set("a", 2) + crossings.reset() + record.save() + # the transaction check, the SELECT of the stale-record check, the UPDATE (was four: the UPDATE took two) + assert crossings.names == ["arcadedb_n_tx", "exec_query", "exec_query"] + assert db.lookup_by_rid(rid).get("a") == 2 + + +def test_an_update_that_reads_its_row_back_is_three_crossings(db, crossings): + rid = _saved(db, a=1) + with db.transaction(): + record = db.lookup_by_rid(rid).modify() + record.set("brand_new", "value") # a new property is read back (RETURN AFTER) + crossings.reset() + record.save() + assert crossings.names == ["arcadedb_n_tx", "exec_query", "exec_query"] + assert db.lookup_by_rid(rid).get("brand_new") == "value" + + +def test_a_stale_record_is_still_refused_before_anything_is_written(db): + rid = _saved(db, a=1, b=1) + mine = db.lookup_by_rid(rid).modify() + with db.transaction(): + bump_sql = f"UPDATE {rid} SET b = 9" # nosec B608 - a record id this test read + db.command("sql", bump_sql).close() + mine.set("a", 2) + with pytest.raises(ArcadeDBError, match="ConcurrentModificationException"): + with db.transaction(): + mine.save() + stored = db.lookup_by_rid(rid) + assert (stored.get("a"), stored.get("b")) == (1, 9) # nothing of mine was written + + +def test_an_update_outside_a_transaction_still_raises_before_reading(db, crossings): + rid = _saved(db, a=1) + record = db.lookup_by_rid(rid).modify() + record.set("a", 2) + crossings.reset() + with pytest.raises(ArcadeDBError, match="Transaction not begun"): + record.save() + assert crossings.names == ["arcadedb_n_tx"] + + +# -- vetoes and errors of the first row, through the form whose reply carries the row + + +def test_a_before_create_veto_on_a_new_record_writes_nothing_and_raises_nothing( + db, crossings +): + veto = db.on("before_create", lambda r: r.get("name") != "blocked", "D") + vetoes = extras.veto_count() + with db.transaction(): + doc = db.new_document("D") + doc.set("name", "blocked") + crossings.reset() + assert doc.save() is doc # no error: a veto is not a failure + assert crossings.names == ["exec_query"] # the veto came in the one crossing + assert extras.veto_count() == vetoes + 1 + assert doc.get_rid() == records._UNSAVED # the object was not given a record id + assert db.count_type("D") == 0 # nothing stored + veto.remove() + with db.transaction(): + doc.save() # the same object saves once the veto is lifted + assert db.count_type("D") == 1 + assert db.lookup_by_rid(str(doc.get_identity())).get("name") == "blocked" + + +def test_a_before_update_veto_writes_nothing_and_the_next_save_is_no_conflict( + db, crossings +): + rid = _saved(db, name="a") + veto = db.on("before_update", lambda r: r.get("name") != "frozen", "D") + vetoes = extras.veto_count() + mutable = db.lookup_by_rid(rid).modify() + mutable.set("name", "frozen") # a changed value of the kind stored as sent + with db.transaction(): + crossings.reset() + assert mutable.save() is mutable + assert crossings.names == ["arcadedb_n_tx", "exec_query", "exec_query"] + assert extras.veto_count() == vetoes + 1 + assert db.lookup_by_rid(rid).get("name") == "a" # nothing written + veto.remove() + mutable.set("name", "thawed") + with db.transaction(): + mutable.save() # the copy is as it was: the earlier veto left no conflict behind + assert db.lookup_by_rid(rid).get("name") == "thawed" + + +def test_a_before_update_veto_on_an_update_that_reads_its_row_back(db, crossings): + rid = _saved(db, name="a") + veto = db.on("before_update", lambda r: r.get("extra") != "x", "D") + vetoes = extras.veto_count() + mutable = db.lookup_by_rid(rid).modify() + mutable.set("extra", "x") # a new property: the statement is the RETURN AFTER one + with db.transaction(): + crossings.reset() + assert mutable.save() is mutable + assert crossings.names == ["arcadedb_n_tx", "exec_query", "exec_query"] + assert extras.veto_count() == vetoes + 1 + stored = db.lookup_by_rid(rid) + assert stored.get("extra") is None and stored.get("name") == "a" + veto.remove() + with db.transaction(): + mutable.save() + stored = db.lookup_by_rid(rid) + assert stored.get("extra") == "x" and stored.get("name") == "a" + + +@contextlib.contextmanager +def _rolled_back(db): + """A transaction that is always rolled back: a statement that failed leaves nothing to commit.""" + db.begin() + try: + yield + finally: + db.rollback() + + +def _raises_at(call): + """``("call", error)`` when the statement call raises, ``("read", error)`` when reading its first row does, ``(None, None)``.""" + try: + result = call() + except ArcadeDBError as error: + return "call", error + try: + result.first() + except ArcadeDBError as error: + return "read", error + return None, None + + +STATEMENT_ERRORS = { + "insert": ("INSERT INTO D SET v = 1 / (1 - 1)", False), + "update": ("UPDATE :rid SET v = 1 / (1 - 1)", True), +} + + +@pytest.mark.parametrize("case", list(STATEMENT_ERRORS)) +def test_a_statement_error_has_the_timing_class_and_message_of_the_lazy_form(db, case): + """An INSERT or UPDATE is run by the engine while the statement is called, so both forms raise from the call (a CI run showed + it: the first version of this test expected the error when the row is read, and the call raised). What a caller of save() sees + is the same either way: one ArcadeDBError with one message. What is pinned is that the two forms agree on all three. + """ + sql, with_rid = STATEMENT_ERRORS[case] + rid = _saved(db, v=1) + params = ({"rid": records.RID(rid)},) if with_rid else () + with _rolled_back(db): + lazy_phase, lazy = _raises_at(lambda: db.command("sql", sql, *params)) + with _rolled_back(db): + eager_phase, eager = _raises_at( + lambda: db._command_first_row("sql", sql, *params) + ) + assert lazy_phase is not None and eager_phase == lazy_phase + assert str(eager) == str(lazy) and type(eager) is type(lazy) + assert eager.native.get("cls") == lazy.native.get("cls") + + +def test_a_row_error_is_raised_when_the_row_is_read_by_both_forms(db): + """A statement whose rows the engine computes lazily (a SELECT run as a command) fails when its first row is read: the public + ``command`` as before (the cursor's read), and the eager form (the error travels in the reply as ``rerr`` and is raised by the + read of the row it replaces). The call itself returns.""" + _saved(db, i=1) + sql = "SELECT 1 / (i - i) AS f FROM D" + with _rolled_back(db): + lazy = db.command("sql", sql) + with pytest.raises(ArcadeDBError) as lazy_error: + lazy.first() + eager = db._command_first_row("sql", sql) + with pytest.raises(ArcadeDBError) as eager_error: + eager.first() + assert str(lazy_error.value).startswith("Reading the result set failed: ") + assert str(eager_error.value) == str(lazy_error.value) + assert type(eager_error.value) is type(lazy_error.value) + + +def test_a_before_create_callback_that_raises_fails_save_as_the_lazy_form_does(db): + def explode(record): + raise RuntimeError("no creating today") + + handle = db.on("before_create", explode, "D") + with _rolled_back(db): + lazy_phase, lazy = _raises_at( + lambda: db.command("sql", "INSERT INTO D CONTENT :row", {"row": {"a": 1}}) + ) + with pytest.raises(ArcadeDBError) as eager: + with db.transaction(): + doc = db.new_document("D") + doc.set("a", 1) + doc.save() + handle.remove() + assert lazy_phase is not None + assert str(eager.value) == str(lazy) + assert "before_create" in str(eager.value) + assert "RuntimeError: no creating today" in str(eager.value) + assert db.count_type("D") == 0 diff --git a/tests/test_native_reply_reader.py b/tests/test_native_reply_reader.py new file mode 100644 index 0000000..0b78909 --- /dev/null +++ b/tests/test_native_reply_reader.py @@ -0,0 +1,195 @@ +"""How the native build reads a reply out of the library (``lib._take``; ``call``, ``call_bin`` and ``exec_query`` all end there). + +A reply is a malloc'd UTF-8 text, optionally followed by a float32 tail, and it must be freed exactly once. It used to be copied as +``bytes`` (``ctypes.string_at``), cut for the tail marker, decoded to ``str``, and only then parsed. It is now decoded straight from its +address in one copy, and only the last 40 bytes (the marker) and the float tail are ever copied as ``bytes``. + +The oracle is a private copy of the former reader (``_former``): every case is read by both, from two identical replies, and the text, +the floats and the number of frees must agree. The copy counts are read from a spy on ``ctypes.string_at``: the former reader called it +once with the whole reply, the new one never does for a text and asks only for the marker window and the tail. The free counts are read +from a spy on ``arcadedb_free``: once, after the use, also when the decode or the tail fails. conftest.py collects this file on the +native backend only: it needs the native library. +""" + +import ctypes +import json + +import numpy as np +import pytest + +from arcadedb._native import lib +from arcadedb.exceptions import ArcadeDBError + +_string_at = ctypes.string_at + + +def _reply(raw: bytes, floats=None): + """A reply in the library's own memory, as the library hands one out: the text, its NUL, then (4-byte aligned) the floats.""" + tail = b"" + if floats is not None: + raw = raw[:-1] + b',"bin":%d}' % len(floats) + pad = -(len(raw) + 1) % 4 + tail = b"\0" * pad + np.asarray(floats, dtype=np.float32).tobytes() + data = raw + b"\0" + tail + ptr = lib.alloc(len(data)) + ctypes.memmove(ptr, data, len(data)) + return ptr, len(raw) + + +def _former(th, ptr): + """The reader this file replaces (``call_bin`` before the change), for the oracle.""" + try: + raw = _string_at(ptr) + floats = None + if raw.endswith(b"}"): + i = raw.rfind(b',"bin":', max(0, len(raw) - 40)) + if i >= 0: + n = int(raw[i + 7 : -1]) + off = (len(raw) + 1 + 3) & ~3 + floats = np.frombuffer(_string_at(ptr + off, 4 * n), dtype=np.float32) + raw = raw[:i] + b"}" + finally: + lib._lib.arcadedb_free(th, ptr) + return raw.decode("utf-8"), floats + + +@pytest.fixture +def frees(temp_db, monkeypatch): + """Every pointer given to ``arcadedb_free``, in order.""" + seen = [] + real = lib._lib.arcadedb_free + + def spy(th, ptr): + seen.append(ptr) + return real(th, ptr) + + monkeypatch.setattr(lib._lib, "arcadedb_free", spy) + return seen + + +@pytest.fixture +def string_at_sizes(monkeypatch): + """The size asked of each ``ctypes.string_at`` call (None: the whole NUL-terminated string).""" + seen = [] + + def spy(ptr, size=-1): + seen.append(None if size == -1 else size) + return _string_at(ptr, size) + + monkeypatch.setattr(ctypes, "string_at", spy) + return seen + + +LONG = json.dumps({"ok": True, "rows": [{"t": "x" * 5000}], "meta": []}).encode() + +CASES = { + "empty object": (b"{}", None), + "ascii": (b'{"ok":true,"rows":[{"a":1}],"drained":true}', None), + "non-ascii": ('{"ok":true,"t":"café 日本語 \U0001f600"}'.encode(), None), + "long text": (LONG, None), + "tail": ( + b'{"ok":true,"rows":[{"v":["B",0,4]}],"meta":[],"drained":true}', + [1, 2, 3, 4], + ), + "tail, short text": (b'{"v":1}', [0.5, 1.5]), + "tail, non-ascii text": ( + '{"ok":true,"rows":[{"t":"日本語 café \U0001f600","v":["B",0,3]}],"drained":true}'.encode(), + [1.5, -2.5, 3.25], + ), + "tail, long text": (LONG, np.arange(16, dtype=np.float32)), + "tail of no floats": (b'{"ok":true}', []), + **{ + f"tail, text of {k} digits": (b'{"a":' + b"1" * k + b"}", [7.0, 8.0]) + for k in range(1, 5) # the four paddings that align the floats to 4 bytes + }, + "marker deep in the text": ( + b'{"a":"' + b"x" * 100 + b'","b":1,"bin":2,"c":' + b"y" * 60 + b"}", + None, + ), +} + + +@pytest.mark.parametrize("name", list(CASES)) +@pytest.mark.parametrize("tail", [True, False]) +def test_every_case_reads_as_the_former_reader_does(temp_db, frees, name, tail): + raw, floats = CASES[name] + th = lib._thread_handle() + a, _ = _reply(raw, floats) + b, _ = _reply(raw, floats) + want_text, want_floats = _former(th, a) + if not tail: # call(): the text as it stands, tail marker included + want_text = _string_at(b).decode("utf-8") + del frees[:] + got_text, got_floats = lib._take(lib._lib, th, b, tail) + assert got_text == want_text + assert frees == [b] # once, and the reply that was read + if tail and floats is not None: + assert got_floats.dtype == np.float32 + assert np.array_equal(got_floats, want_floats) + assert ( + json.loads(got_text) is not None + ) # the text is a whole JSON value without the marker + else: + assert got_floats is None + + +def test_a_text_reply_is_not_copied_as_bytes(frees, string_at_sizes): + text = lib.call("arcadedb_n_diag") + assert json.loads(text) + assert ( + string_at_sizes == [] + ) # the former reader called string_at once, for the whole reply + assert len(frees) == 1 + + +def test_a_reply_with_a_tail_copies_only_the_marker_window_and_the_floats( + temp_db, frees, string_at_sizes +): + ptr, _ = _reply(LONG, np.arange(16, dtype=np.float32)) + th = lib._thread_handle() + text, floats = lib._take(lib._lib, th, ptr, True) + assert len(text) > 5000 and floats.size == 16 + assert sorted(string_at_sizes) == [ + 40, + 64, + ] # the last 40 bytes, 16 floats: never the 5 kB of text + assert frees == [ptr] + + +def test_a_reply_that_fails_to_decode_is_still_freed_once(temp_db, frees): + th = lib._thread_handle() + for raw in (b'{"a":"\xff"}', b'{"a":"\xc3"}', b'{"a":"\xed\xa0\x80"}'): + ptr, _ = _reply(raw) + del frees[:] + with pytest.raises(UnicodeDecodeError): + lib._take(lib._lib, th, ptr, False) + assert frees == [ptr] + ptr, _ = _reply(raw) + del frees[:] + with pytest.raises(UnicodeDecodeError): + lib._take(lib._lib, th, ptr, True) + assert frees == [ptr] + + +def test_a_reply_whose_tail_fails_is_still_freed_once(temp_db, frees): + th = lib._thread_handle() + ptr, _ = _reply(b'{"ok":true,"bin":x}') + with pytest.raises(ValueError): + lib._take(lib._lib, th, ptr, True) + assert frees == [ptr] + + +def test_no_reply_raises_and_frees_nothing(temp_db, frees): + with pytest.raises(ArcadeDBError): + lib._take(lib._lib, lib._thread_handle(), None, False) + assert frees == [] + + +def test_the_public_readers_free_each_reply_once(temp_db, frees): + temp_db.command("sql", "CREATE DOCUMENT TYPE R") + del frees[:] + text, floats = lib.call_bin("arcadedb_n_diag") + assert json.loads(text) and floats is None + assert len(frees) == 1 + lib.call("arcadedb_n_diag") + assert len(frees) == 2 diff --git a/tests/test_native_vertex_columnar.py b/tests/test_native_vertex_columnar.py new file mode 100644 index 0000000..154a1fe --- /dev/null +++ b/tests/test_native_vertex_columnar.py @@ -0,0 +1,55 @@ +"""Native: a vertex type loaded by ``insert_dataframe`` or ``insert_arrow`` crosses as columns (``arcadedb_n_insert_colp``), one crossing +per batch, with no GraphBatch crossing and no JSON text (ArcadeData/arcadedb-python #319). The stored vertices are compared with the old +route in ``tests/test_vertex_load_columnar.py``, which runs on both wheels. Needs the native library, so conftest.py collects this file on +the native backend only. +""" + +import numpy as np +import pandas as pd +import pyarrow as pa + +from arcadedb._native import lib # noqa: E402 + +COLP, ROWS, MANY = ( + "arcadedb_n_insert_colp", + "arcadedb_n_insert_rows", + "arcadedb_n_exec_many", +) + + +def _spy(monkeypatch): + calls = [] + real = lib.call + + def call(name, *args): + calls.append(name) + return real(name, *args) + + monkeypatch.setattr(lib, "call", call) + return calls + + +def _vertex_type(db, name): + db.command("sql", f"CREATE VERTEX TYPE {name}") + db.command("sql", f"CREATE PROPERTY {name}.id LONG") + + +def test_every_batch_of_a_vertex_load_is_one_columnar_crossing(temp_db, monkeypatch): + _vertex_type(temp_db, "V") + calls = _spy(monkeypatch) + frame = pd.DataFrame( + { + "id": np.arange(25, dtype=np.int64), + "name": [f"p{i}" for i in range(25)], + "score": [None if i % 5 == 0 else i * 0.5 for i in range(25)], + "born": pd.to_datetime(["2000-01-01"] * 25), + } + ) + assert temp_db.insert_dataframe("V", frame, batch_size=10) == 25 + assert calls.count(COLP) == 3 # batches of 10, 10 and 5 + assert calls.count(ROWS) == 0 and calls.count(MANY) == 0 + assert not [c for c in calls if "gb" in c or "graph" in c], calls + assert temp_db.query("sql", "SELECT count(*) AS n FROM V").first().get("n") == 25 + calls.clear() + assert temp_db.insert_arrow("V", pa.Table.from_pandas(frame)) == 25 + assert calls.count(COLP) == 1 diff --git a/tests/test_native_vertex_vector_crossings.py b/tests/test_native_vertex_vector_crossings.py new file mode 100644 index 0000000..e684845 --- /dev/null +++ b/tests/test_native_vertex_vector_crossings.py @@ -0,0 +1,256 @@ +"""Native GraphBatch.create_vertices: float32 vectors are split and encoded as columns, not row by row (#337). + +Before, every float32 array of a row was found by ``json.dumps`` through a Python hook (``codec._gb_default``, then ``_default``: about a +dozen ``isinstance`` tests and one array copy per row), the rows were walked by ``codec._prep`` first (one Python call per value), a tag +``{"\\u0001":["B",offset,count]}`` was written and parsed back in Java per row, and ``np.concatenate`` joined one small array per row +(the 29 to 31 percent prep share of the e2 and l3d loads in the untimed ingest profile of 26.11.1). Now ``take_vector_columns`` stacks each +float32 column once, ``codec.encode_vertex_args`` writes the other properties with one ``json.dumps`` (no hook when they are plain scalars) +and the matrices cross as the front of the side buffer, described by one ``"vecs"`` entry, which the library +(``NativeExtras.withVectors``) turns into one ``float[]`` per row. One ``arcadedb_n_gb_bin`` crossing per chunk, as before. + +Needs the native library, so conftest.py collects this file on the native backend only. The stored values are in test_vertex_vector_loads.py +(both wheels) and test_native_graph_batch_binary_vectors.py. +""" + +import ctypes +import json +from datetime import datetime + +import numpy as np +import pytest + +from arcadedb._native import codec, lib + +CHUNK, ROWS, DIM = 100, 250, 16 # three chunks: 100, 100, 50 + + +class _Spy: + """The create_vertices crossings (entry, JSON text, side buffer, float count) and the per-row Python calls of the codec.""" + + def __init__(self, monkeypatch): + self.crossings, self.hooks = [], {"gb_default": 0, "default": 0, "prep": 0} + real_call, real_gb_default = lib.call, codec._gb_default + real_default, real_prep = codec._default, codec._prep + + def call(name, *args): + if ( + name in ("arcadedb_n_gb", "arcadedb_n_gb_bin") + and args[1] == b"create_vertices" + ): + buf = args[3] if name == "arcadedb_n_gb_bin" else None + count = args[4] if name == "arcadedb_n_gb_bin" else 0 + self.crossings.append((name, args[2].decode(), buf, count)) + return real_call(name, *args) + + def counting(key, real): + def wrapper(*args, **kwargs): + self.hooks[key] += 1 + return real(*args, **kwargs) + + return wrapper + + monkeypatch.setattr(lib, "call", call) + monkeypatch.setattr( + codec, "_gb_default", counting("gb_default", real_gb_default) + ) + monkeypatch.setattr(codec, "_default", counting("default", real_default)) + monkeypatch.setattr(codec, "_prep", counting("prep", real_prep)) + + def entries(self): + return [c[0] for c in self.crossings] + + def around(self, call): + """``(result, hooks, crossings)`` of one ``call()``: the codec hook calls and the library crossings made inside it only. + + ``codec._prep`` is counted by name, so it also counts the calls the batch itself makes outside ``create_vertices``: creating + the batch (``_enc`` of its options, whose ``_prep`` recurses into the patched name once per option) and ``close()`` + (``encode_gb_args({})``). Those are a few calls and do not depend on the rows, so the counts here are taken around the + ``create_vertices`` call alone. + """ + before, seen = dict(self.hooks), len(self.crossings) + result = call() + used = {key: self.hooks[key] - before[key] for key in before} + return result, used, len(self.crossings) - seen + + +@pytest.fixture +def spy(monkeypatch): + return _Spy(monkeypatch) + + +@pytest.fixture +def db(temp_db): + for name in ("Declared", "Free"): + temp_db.command("sql", f"CREATE VERTEX TYPE {name}") + temp_db.command("sql", f"CREATE PROPERTY {name}.id LONG") + temp_db.command("sql", "CREATE PROPERTY Declared.embedding ARRAY_OF_FLOATS") + temp_db.command("sql", "CREATE PROPERTY Free.born DATETIME") + return temp_db + + +def _x(n=ROWS, dim=DIM, seed=7): + return np.random.default_rng(seed).standard_normal((n, dim)).astype(np.float32) + + +def _stored(db, type_name, prop="embedding"): + sql = f"SELECT id, {prop} AS v FROM {type_name}" # nosec B608 - test-owned names + return {r["id"]: r["v"] for r in db.query("sql", sql).to_list()} + + +def _assert_bit_equal(stored, x): + assert sorted(stored) == list(range(len(x))) + for i in range(len(x)): + assert np.array(stored[i], dtype=np.float32).tobytes() == x[i].tobytes() + + +@pytest.mark.parametrize("type_name", ["Declared", "Free"]) +def test_a_float32_column_is_one_matrix_with_no_hook_call_per_row(db, spy, type_name): + x = _x() + rows = [{"id": i, "embedding": x[i]} for i in range(ROWS)] + with db.graph_batch(commit_every=CHUNK) as gb: + rids, used, crossed = spy.around(lambda: gb.create_vertices(type_name, rows)) + assert len(rids) == ROWS + assert spy.entries() == ["arcadedb_n_gb_bin"] * 3 # one crossing per chunk of 100 + # no per-row hook: the per-row hooks of the old path are never called, and nothing is called more than once per crossing + assert used["gb_default"] == 0 and used["default"] == 0, used + assert used["prep"] <= crossed, used # the old path called it for every row + assert [c[3] for c in spy.crossings] == [CHUNK * DIM, CHUNK * DIM, 50 * DIM] + for _, text, buf, count in spy.crossings: + args = json.loads(text) + assert args["vecs"] == {"names": ["embedding"], "dims": [DIM], "offs": [0]} + assert '"B"' not in text and '"F"' not in text # no tag and no number text + assert all(set(row) == {"id"} for row in args["rows"]) + assert isinstance( + buf, ctypes.Array + ) # a view of the matrix, not a copy of its bytes + _assert_bit_equal(_stored(db, type_name), x) + + +def test_the_python_work_does_not_grow_with_the_row_count(db, spy): + # 250 rows in three chunks of 100, then 2,500 rows in three chunks of 1,000: ten times the rows, the same number of crossings, and + # the same bound on the codec hooks (the old path called the hooks once or more for every row, so it would show 250 and 2,500) + for rows_n, chunk in ((250, 100), (2500, 1000)): + x = _x(rows_n) + rows = [{"id": i, "embedding": x[i]} for i in range(rows_n)] + with db.graph_batch(commit_every=chunk) as gb: + rids, used, crossed = spy.around(lambda: gb.create_vertices("Free", rows)) + assert len(rids) == rows_n and crossed == 3 + assert used["gb_default"] == 0 and used["default"] == 0, used + assert used["prep"] <= crossed, used + + +def test_two_columns_and_a_date_time_keep_the_buffer_layout(db, spy): + x, y = _x(), _x(dim=8, seed=8) + when = datetime(2020, 1, 2, 3, 4, 5) + rows = [ + {"id": i, "embedding": x[i], "second": y[i], "born": when} for i in range(ROWS) + ] + with db.graph_batch(commit_every=CHUNK) as gb: + gb.create_vertices("Free", rows) + assert spy.entries() == ["arcadedb_n_gb_bin"] * 3 + first = json.loads(spy.crossings[0][1]) + assert first["vecs"] == { + "names": ["embedding", "second"], + "dims": [DIM, 8], + "offs": [0, CHUNK * DIM], + } + assert spy.crossings[0][3] == CHUNK * DIM + CHUNK * 8 + _assert_bit_equal(_stored(db, "Free"), x) + _assert_bit_equal(_stored(db, "Free", "second"), y) + sql = "SELECT born FROM Free WHERE id = 7" + assert db.query("sql", sql).first().get("born") == when + + +def test_a_float32_array_that_does_not_qualify_follows_the_matrices_in_the_buffer( + db, spy +): + x = _x(60) + rows = [{"id": i, "embedding": x[i], "short": x[i][: 2 + i % 2]} for i in range(60)] + with db.graph_batch(commit_every=CHUNK) as gb: + gb.create_vertices("Free", rows) + assert spy.entries() == ["arcadedb_n_gb_bin"] + text, count = spy.crossings[0][1], spy.crossings[0][3] + assert json.loads(text)["vecs"]["offs"] == [0] + assert json.loads(text)["rows"][0]["short"] == {"\u0001": ["B", 60 * DIM, 2]} + assert count == 60 * DIM + sum(2 + i % 2 for i in range(60)) + _assert_bit_equal(_stored(db, "Free"), x) + short = _stored(db, "Free", "short") + for i in range(60): + assert ( + np.array(short[i], dtype=np.float32).tobytes() + == x[i][: 2 + i % 2].tobytes() + ) + + +def test_a_float64_or_list_column_keeps_the_text_entry(db, spy): + x = _x(30) + rows = [{"id": i, "embedding": x[i].astype(np.float64)} for i in range(30)] + with db.graph_batch(commit_every=CHUNK) as gb: + gb.create_vertices("Free", rows) + gb.create_vertices( + "Declared", [{"id": i, "embedding": x[i].tolist()} for i in range(30)] + ) + assert spy.entries() == ["arcadedb_n_gb"] * 2 + assert all('"vecs"' not in c[1] for c in spy.crossings) + _assert_bit_equal(_stored(db, "Declared"), x) + + +def test_a_library_without_the_entry_keeps_the_text_path(db, spy, monkeypatch): + monkeypatch.setattr(lib, "GB_BIN", False) + x = _x(30) + with db.graph_batch(commit_every=CHUNK) as gb: + gb.create_vertices("Free", [{"id": i, "embedding": x[i]} for i in range(30)]) + assert spy.entries() == ["arcadedb_n_gb"] + assert '"F"' in spy.crossings[0][1] and '"vecs"' not in spy.crossings[0][1] + _assert_bit_equal(_stored(db, "Free"), x) + + +def test_a_vector_column_outside_the_side_buffer_is_refused(db): + x = _x(4) + with db.graph_batch() as gb: + text = json.dumps( + { + "type": "Free", + "rows": [{"id": i} for i in range(4)], + "vecs": {"names": ["embedding"], "dims": [DIM], "offs": [8]}, + } + ) + buf = x.tobytes() # 4 vectors of 16 floats: offset 8 leaves room for 56, not 64 + reply = json.loads( + lib.call( + "arcadedb_n_gb_bin", + gb._gb, + b"create_vertices", + text.encode(), + buf, + x.size, + ) + ) + assert not reply.get("ok") + assert "outside the side buffer" in json.dumps(reply) + assert db.query("sql", "SELECT count(*) AS c FROM Free").first().get("c") == 0 + + +def test_the_encoder_in_isolation(): + x = _x(5, 4) + rows = [{"id": i} for i in range(5)] + text, buf, count = codec.encode_vertex_args("T", rows, ["v"], [x]) + assert json.loads(text) == { + "type": "T", + "rows": rows, + "vecs": {"names": ["v"], "dims": [4], "offs": [0]}, + } + assert count == 20 and bytes(buf) == x.tobytes() + frozen = x.copy() + frozen.setflags( + write=False + ) # a read-only matrix cannot be viewed writable: its bytes cross instead + assert codec.encode_vertex_args("T", rows, ["v"], [frozen])[1] == x.tobytes() + assert ( + codec._plain_rows(rows) + and codec._plain_rows([{}, {}]) + and codec._plain_rows([]) + ) + assert not codec._plain_rows([{"a": np.float64(1)}]) + assert not codec._plain_rows([{"a": [1]}]) + assert not codec._plain_rows([{1: 2}]) diff --git a/tests/test_parity_crash_retry.py b/tests/test_parity_crash_retry.py new file mode 100644 index 0000000..1aab86b --- /dev/null +++ b/tests/test_parity_crash_retry.py @@ -0,0 +1,1051 @@ +"""The crash handling of the native parity harness (native/wheel/parity_wheel.py): record, retry once, report, stay strict. + +The Windows legs of the native release run lost one test process each (ArcadeData/arcadedb-python#295), with no exit code and no trace. +The harness now records the exit code of a process that died without a result (`CRASH(rc=0xC0000005,3221225477)`), runs such a file once +more when --retry-crash is given (the workflow gives it on Windows legs only) and marks a retry that produced a result `FLAKY-CRASH`. A +failing test, a timeout, a second crash and every crash without the flag stay failures. These tests run the decision with a fake runner, +with real child processes that exit with a high code or die from a signal, and the harness itself against a small fixture repository +(real pytest children, the wheel check stubbed), so the code that ships is the code that is tested. Nothing here needs Windows: the exit +code of a crash differs by platform and the tests read it from the platform they run on. +""" + +from __future__ import annotations + +import importlib.util +import json +import os +import re +import signal +import subprocess # nosec B404 - test-controlled child processes +import sys +import textwrap +from pathlib import Path + +import pytest + +ROOT = Path(__file__).resolve().parents[1] +WHEEL_DIR = ROOT / "native" / "wheel" +WORKFLOWS = ROOT / ".github" / "workflows" + +# the harness imports parity_crash as a sibling of the script +if str(WHEEL_DIR) not in sys.path: + sys.path.insert(0, str(WHEEL_DIR)) + + +def _load(name: str): + spec = importlib.util.spec_from_file_location(name, WHEEL_DIR / f"{name}.py") + module = importlib.util.module_from_spec(spec) + sys.modules[name] = module # a dataclass looks its module up here + spec.loader.exec_module(module) + return module + + +pc = _load("parity_crash") +parity_wheel = _load("parity_wheel") +report = _load("report_flaky_crash") + +WIN = sys.platform == "win32" +# An exit code a test process can die with: the NTSTATUS of an access violation on Windows, 139 (128 + SIGSEGV) elsewhere (POSIX keeps 8 bits). +HIGH = 0xC0000005 if WIN else 139 +HIGH_TEXT = "0xC0000005,3221225477" if WIN else "0x0000008B,139" +# os._exit takes a C int: the same NTSTATUS as a signed 32-bit number on Windows (the exit code stays 0xC0000005 to the parent) +HIGH_ARG = HIGH - (1 << 32) if WIN else HIGH +OTHER = 0xC0000374 if WIN else 134 +OTHER_TEXT = "0xC0000374,3221226356" if WIN else "0x00000086,134" +EM_DASH = "\u2014" # written as an escape so that this file holds none itself + + +def attempts(*items): + """A fake runner: attempt n returns items[n - 1]; `calls` records which attempts ran.""" + calls = [] + + def run(n): + calls.append(n) + return items[n - 1] + + return run, calls + + +RESULT = pc.Attempt(rc=0, result=True) +FAILED = pc.Attempt(rc=1, result=True) # pytest wrote its XML, with failures in it +DIED = pc.Attempt(rc=HIGH, result=False) +DIED_AGAIN = pc.Attempt(rc=OTHER, result=False) +ON = lambda budget=3: pc.RetryPolicy(enabled=True, budget=budget) # noqa: E731 + + +# --------------------------------------------------------------------------------------------------------------------------------- +# the decision, with a fake runner +# --------------------------------------------------------------------------------------------------------------------------------- + + +def test_a_file_with_a_result_runs_once_even_when_the_retry_is_on(): + run, calls = attempts(RESULT, RESULT) + out = pc.run_file(run, ON()) + assert calls == [1] + assert out.status == "" + assert (out.retried, out.flaky, out.hard_crash) == (False, False, False) + + +def test_a_file_with_failing_tests_is_never_retried(): + # exit code 1 with an XML is a test failure: the count of failures comes from the XML and stays strict + run, calls = attempts(FAILED, RESULT) + out = pc.run_file(run, ON()) + assert calls == [1] + assert out.status == "" + assert out.final is FAILED + + +def test_a_crash_without_the_retry_flag_is_a_crash_with_its_exit_code(): + # the default policy is what Linux and macOS use: strict, one run, the exit code recorded + run, calls = attempts(DIED, RESULT) + out = pc.run_file(run, pc.RetryPolicy()) + assert calls == [1] + assert out.status == f"CRASH(rc={HIGH_TEXT})" + assert out.hard_crash and not out.flaky and not out.retried + + +def test_a_crash_that_produces_a_result_when_run_again_is_flaky_not_clean(): + run, calls = attempts(DIED, RESULT) + out = pc.run_file(run, ON()) + assert calls == [1, 2] + assert out.status == f"FLAKY-CRASH(rc={HIGH_TEXT})" + assert out.flaky and out.retried and not out.hard_crash + assert out.final is RESULT + + +def test_a_second_crash_is_a_crash_with_both_exit_codes(): + run, calls = attempts(DIED, DIED_AGAIN) + out = pc.run_file(run, ON()) + assert calls == [1, 2] + assert out.status == f"CRASH(rc={HIGH_TEXT};retry rc={OTHER_TEXT})" + assert out.hard_crash and not out.flaky + + +def test_a_timeout_and_an_unreadable_xml_are_not_crashes_and_are_not_retried(): + run, calls = attempts(pc.Attempt(rc=None, result=False, timed_out=True), RESULT) + out = pc.run_file(run, ON()) + assert (calls, out.status, out.timed_out) == ([1], "TIMEOUT", True) + run, calls = attempts(pc.Attempt(rc=1, result=False, bad_xml=True), RESULT) + out = pc.run_file(run, ON()) + assert (calls, out.status) == ([1], "BADXML") + + +def test_a_retry_that_times_out_or_writes_a_bad_xml_fails_the_leg(): + run, _ = attempts(DIED, pc.Attempt(rc=None, result=False, timed_out=True)) + out = pc.run_file(run, ON()) + assert out.status == f"CRASH(rc={HIGH_TEXT};retry TIMEOUT)" + assert out.timed_out + run, _ = attempts(DIED, pc.Attempt(rc=1, result=False, bad_xml=True)) + assert pc.run_file(run, ON()).status == f"CRASH(rc={HIGH_TEXT};retry BADXML)" + + +def test_a_retry_that_runs_and_has_failures_is_flaky_and_the_counts_decide(): + # the status says the file crashed once; the failures of the second run are counted by the harness and fail the leg + run, _ = attempts(DIED, FAILED) + out = pc.run_file(run, ON()) + assert out.flaky + assert pc.event_of("test_x.py", out, final_failed=2)["retry"] == "ran, 2 failed" + + +def test_the_retry_budget_of_a_leg_is_spent_by_the_files_not_the_crashes(): + policy = ON(budget=2) + statuses = [] + for _ in range(3): + run, calls = attempts(DIED, RESULT) + out = pc.run_file(run, policy) + statuses.append((out.status.split("(")[0], calls)) + # two files retried and flaky, the third crashes where the budget is spent: a wheel that crashes everywhere cannot pass on retries + assert statuses == [ + ("FLAKY-CRASH", [1, 2]), + ("FLAKY-CRASH", [1, 2]), + ("CRASH", [1]), + ] + assert policy.used == 2 + run, _ = attempts(DIED) + assert ( + pc.run_file(run, policy).status == f"CRASH(rc={HIGH_TEXT};retry budget spent)" + ) + + +def test_exit_codes_are_written_as_ntstatus_and_decimal_or_as_a_signal(): + assert pc.format_rc(3221225477, windows=True) == "0xC0000005,3221225477" + # the same NTSTATUS read as a signed number folds back to 32 bits on Windows + assert pc.format_rc(-1073741819, windows=True) == "0xC0000005,3221225477" + assert pc.format_rc(139, windows=False) == "0x0000008B,139" + assert pc.format_rc(0, windows=False) == "0x00000000,0" + # the number of SIGABRT is the platform's (6 on POSIX, 22 on Windows) + assert pc.format_rc(-signal.SIGABRT, windows=False) == f"{-signal.SIGABRT},SIGABRT" + assert pc.format_rc(-250, windows=False) == "-250,signal 250" + assert pc.format_rc(None) == "none" + + +def test_a_crash_event_names_the_kind_and_what_the_retry_did(): + run, _ = attempts(DIED, RESULT) + ev = pc.event_of("test_free_threading.py", pc.run_file(run, ON()), final_failed=0) + assert ev == { + "file": "test_free_threading.py", + "kind": "FLAKY-CRASH", + "first_rc": HIGH_TEXT, + "retry_rc": "0x00000000,0", + "retry": "passed", + "status": f"FLAKY-CRASH(rc={HIGH_TEXT})", + } + run, _ = attempts(DIED, DIED_AGAIN) + ev = pc.event_of("a.py", pc.run_file(run, ON()), final_failed=0) + assert (ev["kind"], ev["retry"], ev["retry_rc"]) == ( + "CRASH", + "crashed again", + OTHER_TEXT, + ) + run, _ = attempts(DIED) + ev = pc.event_of("a.py", pc.run_file(run, pc.RetryPolicy()), final_failed=0) + assert (ev["kind"], ev["retry"], ev["retry_rc"]) == ("CRASH", "not retried", "") + run, _ = attempts(RESULT) + assert pc.event_of("a.py", pc.run_file(run, ON()), final_failed=0) is None + + +def test_a_result_followed_by_an_odd_exit_code_is_noticed_but_not_a_crash(): + # the junit XML is written at the end of the session; a process that dies in shutdown after it leaves a result + assert pc.dirty_exit(pc.Attempt(rc=HIGH, result=True)) is True + assert pc.dirty_exit(pc.Attempt(rc=1, result=True)) is False + assert pc.dirty_exit(pc.Attempt(rc=0, result=True)) is False + assert pc.dirty_exit(DIED) is False + assert pc.is_crash(pc.Attempt(rc=HIGH, result=True)) is False + + +# --------------------------------------------------------------------------------------------------------------------------------- +# real child processes: a high exit code, a signal, and a fresh process for the retry +# --------------------------------------------------------------------------------------------------------------------------------- + +EXIT_ONCE = """ +import os, pathlib, sys +d, n, high = pathlib.Path(sys.argv[1]), sys.argv[2], int(sys.argv[3]) +with open(d / "pids", "a") as fh: + fh.write(f"{os.getpid()}\\n") +if not (d / "died").exists(): + (d / "died").write_text("1") + os._exit(high) +(d / f"result{n}").write_text("ok") +""" + +EXIT_ALWAYS = """ +import os, sys +os._exit(int(sys.argv[3])) +""" + +SEGV = """ +import os, sys +if sys.platform == "win32": + # a real access violation (0xC0000005); ctypes would turn one into an OSError, and raise(SIGSEGV) exits with 3 + import ctypes, faulthandler + ctypes.windll.kernel32.SetErrorMode(0x8003) # no crash dialog and no Windows Error Reporting for this child + faulthandler._read_null() +else: + import resource, signal + resource.setrlimit(resource.RLIMIT_CORE, (0, 0)) # no core file for a test child + os.kill(os.getpid(), signal.SIGSEGV) +""" + + +def real_runner(code, d): + """attempt(n) runs `code` in a new python process; the process is a crash unless it wrote result in directory d.""" + + def run(n): + r = subprocess.run( # nosec B603 - fixed argument list, no shell + [sys.executable, "-c", code, str(d), str(n), str(HIGH_ARG)], + capture_output=True, + text=True, + timeout=120, + ) + return pc.Attempt(rc=r.returncode, result=(d / f"result{n}").exists()) + + return run + + +def test_a_real_process_that_exits_with_a_high_code_is_recorded_with_that_code( + tmp_path, +): + out = pc.run_file(real_runner(EXIT_ALWAYS, tmp_path), pc.RetryPolicy()) + assert out.final.rc == HIGH + assert out.status == f"CRASH(rc={HIGH_TEXT})" + + +def test_a_real_process_killed_by_a_fault_is_recorded_as_one(tmp_path): + out = pc.run_file(real_runner(SEGV, tmp_path), pc.RetryPolicy()) + if WIN: + assert out.final.rc == 0xC0000005 + assert out.status == "CRASH(rc=0xC0000005,3221225477)" + else: + assert out.final.rc == -signal.SIGSEGV + assert out.status == f"CRASH(rc={-signal.SIGSEGV},SIGSEGV)" + + +def test_the_retry_of_a_real_crash_is_a_new_process_and_a_pass_is_flaky(tmp_path): + out = pc.run_file(real_runner(EXIT_ONCE, tmp_path), ON()) + pids = (tmp_path / "pids").read_text().split() + assert len(pids) == 2 and pids[0] != pids[1] + assert (tmp_path / "result1").exists() is False and (tmp_path / "result2").exists() + assert out.status == f"FLAKY-CRASH(rc={HIGH_TEXT})" + assert out.final.rc == 0 + + +def test_a_real_file_that_crashes_every_time_fails_after_one_retry(tmp_path): + out = pc.run_file(real_runner(EXIT_ALWAYS, tmp_path), ON()) + assert out.status == f"CRASH(rc={HIGH_TEXT};retry rc={HIGH_TEXT})" + assert len(out.attempts) == 2 + + +# --------------------------------------------------------------------------------------------------------------------------------- +# the harness against a small repository: real pytest children, the wheel check stubbed +# --------------------------------------------------------------------------------------------------------------------------------- + +FIXTURE_FILES = { + "test_ok.py": "def test_a():\n pass\n\n\ndef test_b():\n pass\n", + "test_fails.py": "def test_a():\n assert False, 'a real failure'\n", + "test_dies_once.py": textwrap.dedent(f""" + import os, pathlib + + + def test_a(): + marker = pathlib.Path(os.environ["DIES_ONCE_MARKER"]) + if not marker.exists(): + marker.write_text("1") + os._exit({HIGH_ARG}) + """), + "test_dies_always.py": f"import os\n\n\ndef test_a():\n os._exit({HIGH_ARG})\n", +} + +# a process that dies in shutdown, after pytest wrote its XML (pytest_sessionfinish runs before pytest_unconfigure) +DIRTY_CONFTEST = f""" +import os + + +def pytest_unconfigure(config): + if any("test_dirty" in str(a) for a in config.args): + os._exit({HIGH_ARG}) +""" + +IDENT = ( + "/x/site-packages/arcadedb/__init__.py 26.11.1.post2 native 26.11.1 abc1234\n" + 'META {"python": "3.14.0", "wheel_version": "26.11.1.post2", "engine_version": "26.11.1", "source_commit": "abc1234"}\n' +) + + +@pytest.fixture +def harness(tmp_path, monkeypatch): + """run(*flags) -> (rows by file name, the TOTAL row, meta.json, the xml directory); the fixture repo is rebuilt for each call.""" + real_run = subprocess.run + + def run(cmd, *args, **kwargs): + if "-I" in cmd and any("import json, sys, arcadedb" in str(c) for c in cmd): + return subprocess.CompletedProcess(cmd, 0, stdout=IDENT, stderr="") + return real_run(cmd, *args, **kwargs) + + monkeypatch.setattr(parity_wheel.subprocess, "run", run) + # no taskset and no nice: the cores of this machine are not the harness's to choose + monkeypatch.setattr(parity_wheel.shutil, "which", lambda name: None) + monkeypatch.delenv("PYTHONPATH", raising=False) + counter = iter(range(100)) + + def go(*flags, dirty=False, seed=None): + """`dirty` adds test_dirty.py and a conftest whose process dies after the XML; `seed` puts files into the XML directory first + ({name: text} or {name: (text, age in seconds)}).""" + repo = tmp_path / f"repo{next(counter)}" + (repo / "tests").mkdir(parents=True) + files = dict(FIXTURE_FILES) + if dirty: + files["test_dirty.py"] = "def test_a():\n pass\n" + (repo / "tests" / "conftest.py").write_text( + DIRTY_CONFTEST if dirty else "# fixture\n" + ) + for name, text in files.items(): + (repo / "tests" / name).write_text(text) + for name, item in (seed or {}).items(): + text, age = item if isinstance(item, tuple) else (item, 0) + path = repo / "out" / "parity-wheel-xml" / name + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(text) + if age: + os.utime(path, (path.stat().st_mtime - age, path.stat().st_mtime - age)) + monkeypatch.setenv("DIES_ONCE_MARKER", str(repo / "marker")) + monkeypatch.setenv("KEY", "windows-x64") + out = repo / "out" / "parity-wheel.tsv" + parity_wheel.main( + [ + "--repo", + str(repo), + "--python", + sys.executable, + "--out", + str(out), + "--timeout", + "120", + *flags, + ] + ) + lines = [ln.split("\t") for ln in out.read_text().splitlines()] + header = lines[0] + rows = {ln[0]: dict(zip(header, ln)) for ln in lines[1:]} + meta = json.loads((repo / "out" / "parity-wheel.meta.json").read_text()) + return rows, meta, repo / "out" / "parity-wheel-xml" + + return go + + +def test_the_harness_is_strict_without_the_flag(harness): + rows, meta, xmldir = harness() + assert rows["test_ok.py"]["status"] == "" and rows["test_ok.py"]["passed"] == "2" + assert ( + rows["test_fails.py"]["failed"] == "1" and rows["test_fails.py"]["status"] == "" + ) + # both crashing files are CRASH rows with the exit code, one attempt each, no retry + for name in ("test_dies_once.py", "test_dies_always.py"): + assert rows[name]["status"] == f"CRASH(rc={HIGH_TEXT})", name + assert rows[name]["attempts"] == "1", name + assert rows[name]["exit"] == HIGH_TEXT, name + assert rows["TOTAL"]["status"] == "timeouts=0 flaky_crash=0 crash=2" + assert meta["retry_crash"] is False + assert [e["kind"] for e in meta["crash_events"]] == ["CRASH", "CRASH"] + assert not (xmldir / "test_dies_once.py.attempt1.log").exists() + + +def test_the_harness_retries_only_the_crashes_with_the_flag(harness): + rows, meta, xmldir = harness("--retry-crash") + # a clean file and a file with a failing test run once, whatever the flag says + assert (rows["test_ok.py"]["attempts"], rows["test_ok.py"]["status"]) == ("1", "") + assert (rows["test_fails.py"]["attempts"], rows["test_fails.py"]["status"]) == ( + "1", + "", + ) + assert rows["test_fails.py"]["failed"] == "1" + # the file that died once passes on the retry: FLAKY-CRASH with the first exit code, counted from the second run + once = rows["test_dies_once.py"] + assert once["status"] == f"FLAKY-CRASH(rc={HIGH_TEXT})" + assert (once["attempts"], once["passed"], once["failed"]) == ("2", "1", "0") + assert once["exit"] == "0x00000000,0" + # the file that always dies is retried once and then fails the leg + always = rows["test_dies_always.py"] + assert always["status"] == f"CRASH(rc={HIGH_TEXT};retry rc={HIGH_TEXT})" + assert always["attempts"] == "2" + assert rows["TOTAL"]["status"] == "timeouts=0 flaky_crash=1 crash=1" + # the log of the first attempt is kept beside the log of the retry + assert (xmldir / "test_dies_once.py.attempt1.log").exists() + assert (xmldir / "test_dies_once.py.log").exists() + assert (xmldir / "test_dies_once.py.xml").exists() + # the metadata names the wheel and the Python once and every crash event + assert meta["wheel_version"] == "26.11.1.post2" and meta["python"] == "3.14.0" + assert meta["retry_crash"] is True + events = {e["file"]: e for e in meta["crash_events"]} + assert set(events) == {"test_dies_once.py", "test_dies_always.py"} + assert events["test_dies_once.py"]["kind"] == "FLAKY-CRASH" + assert events["test_dies_once.py"]["retry"] == "passed" + assert events["test_dies_always.py"]["retry"] == "crashed again" + + +def test_the_harness_retry_is_off_unless_asked_for(): + assert parity_wheel.parse_args(["--repo", "r", "--out", "o"]).retry_crash is False + assert ( + parity_wheel.parse_args( + ["--repo", "r", "--out", "o", "--retry-crash"] + ).retry_crash + is True + ) + assert parity_wheel.parse_args(["--repo", "r", "--out", "o"]).max_crash_retries == 3 + + +# --------------------------------------------------------------------------------------------------------------------------------- +# the faulthandler trace of a native run on Windows (tests/conftest.py) +# --------------------------------------------------------------------------------------------------------------------------------- + +FAULT_CHILD = """ +import faulthandler, os, sys + +if os.name == "posix": + import resource + + resource.setrlimit(resource.RLIMIT_CORE, (0, 0)) # no core file for a test child +else: + import ctypes + + ctypes.windll.kernel32.SetErrorMode(0x8003) # no crash dialog and no Windows Error Reporting for this child + +sys.path.insert(0, sys.argv[1]) +from tests import conftest + +trace = sys.argv[2] +os.environ[conftest.FAULT_FILE_ENV] = trace + +# the JVM build ignores the variable: faulthandler stays off on Windows, as before +conftest.BACKEND = "jvm" +sys.platform = "win32" +faulthandler.enable() +conftest.pytest_configure(None) +assert not faulthandler.is_enabled(), "jvm on win32" +assert not os.path.exists(trace), "jvm on win32 opened the trace file" + +# another platform keeps what it has, whatever the variable says +conftest.BACKEND = "native" +sys.platform = "linux" +faulthandler.enable() +conftest.pytest_configure(None) +assert faulthandler.is_enabled(), "native on linux" +assert not os.path.exists(trace), "native on linux opened the trace file" + +# the native build on Windows without the variable: off, as before +sys.platform = "win32" +del os.environ[conftest.FAULT_FILE_ENV] +conftest.pytest_configure(None) +assert not faulthandler.is_enabled(), "native on win32 without the variable" + +# the native build on Windows with the variable: on, and a fault leaves its trace in the file +os.environ[conftest.FAULT_FILE_ENV] = trace +conftest.pytest_configure(None) +assert faulthandler.is_enabled(), "native on win32 with the variable" +print("configured", flush=True) +faulthandler._read_null() # a read of address 0: an access violation on Windows, SIGSEGV elsewhere +""" + + +def test_a_native_run_on_windows_keeps_a_faulthandler_trace_and_nothing_else_does( + tmp_path, +): + trace = tmp_path / "fault.txt" + r = subprocess.run( # nosec B603 - fixed argument list, no shell + [sys.executable, "-c", FAULT_CHILD, str(ROOT), str(trace)], + capture_output=True, + text=True, + timeout=120, + ) + # every assert of the child passed (a failed one would end the child before it printed) and it then died of the fault + assert "AssertionError" not in r.stderr, r.stderr + assert "configured" in r.stdout, r.stderr + assert r.returncode == (0xC0000005 if WIN else -signal.SIGSEGV), ( + r.returncode, + r.stderr, + ) + text = trace.read_text(encoding="utf-8") + assert "Segmentation fault" in text or "access violation" in text + assert "most recent call first" in text + + +# --------------------------------------------------------------------------------------------------------------------------------- +# the report: the job summary and one comment on issue 295 +# --------------------------------------------------------------------------------------------------------------------------------- + + +def write_meta(root, artifact, key, python, wheel, events, retry=True): + d = root / artifact + d.mkdir(parents=True) + (d / "parity-wheel.meta.json").write_text( + json.dumps( + { + "key": key, + "platform": "win32", + "python": python, + "wheel_version": wheel, + "retry_crash": retry, + "crash_events": events, + } + ) + ) + + +FLAKY_EVENT = { + "file": "test_free_threading.py", + "kind": "FLAKY-CRASH", + "first_rc": "0xC0000005,3221225477", + "retry_rc": "0x00000000,0", + "retry": "passed", + "status": "FLAKY-CRASH(rc=0xC0000005,3221225477)", + "trace1": "test_free_threading.py.attempt1.fault.txt", +} +CRASH_EVENT = { + "file": "test_vector.py", + "kind": "CRASH", + "first_rc": "0xC0000374,3221226356", + "retry_rc": "0xC0000374,3221226356", + "retry": "crashed again", + "status": "CRASH(rc=0xC0000374,3221226356;retry rc=0xC0000374,3221226356)", +} +RELEASE_ENV = { + "GITHUB_REPOSITORY": "ArcadeData/arcadedb-python", + "GITHUB_RUN_ID": "42744219731", + "GITHUB_RUN_ATTEMPT": "2", + "GITHUB_EVENT_NAME": "push", + "GITHUB_REF_TYPE": "tag", + "GITHUB_REF_NAME": "26.11.1.post2", + "GITHUB_SERVER_URL": "https://github.com", + "GH_TOKEN": "token", # nosec B105 - a made-up token in a test environment, it reaches no service +} + + +@pytest.fixture +def artifacts(tmp_path): + root = tmp_path / "artifacts" + write_meta( + root, + "native-parity-windows-x64-py3.11", + "windows-x64", + "3.11.9", + "26.11.1.post2", + [FLAKY_EVENT], + ) + write_meta( + root, + "native-parity-windows-x64-py3.12", + "windows-x64", + "3.12.7", + "26.11.1.post2", + [], + ) + write_meta( + root, + "native-parity-windows-x64-py3.14", + "windows-x64", + "3.14.0", + "26.11.1.post2", + [CRASH_EVENT], + ) + return root + + +class FakeGh: + def __init__(self, returncode=0, stderr=""): + self.calls, self.returncode, self.stderr = [], returncode, stderr + + def __call__(self, cmd, **kwargs): + self.calls.append((cmd, json.loads(kwargs["input"])["body"])) + return subprocess.CompletedProcess( + cmd, self.returncode, stdout="", stderr=self.stderr + ) + + +def test_the_events_of_every_leg_are_collected_with_the_identity_of_their_leg( + artifacts, +): + events = report.load_events(str(artifacts)) + assert [(e["file"], e["python"], e["wheel"]) for e in events] == [ + ("test_free_threading.py", "3.11.9", "26.11.1.post2"), + ("test_vector.py", "3.14.0", "26.11.1.post2"), + ] + assert report.load_events(str(artifacts / "none")) == [] + + +def test_a_release_posts_one_comment_listing_every_event_and_the_summary_has_the_table( + artifacts, tmp_path, capsys +): + gh, summary = FakeGh(), tmp_path / "summary.md" + code = report.report( + report.load_events(str(artifacts)), RELEASE_ENV, str(summary), run=gh + ) + assert code == 0 + assert len(gh.calls) == 1 + cmd, body = gh.calls[0] + assert cmd[:5] == [ + "gh", + "api", + "--method", + "POST", + "repos/ArcadeData/arcadedb-python/issues/295/comments", + ] + for text in ( + "https://github.com/ArcadeData/arcadedb-python/actions/runs/42744219731", + "attempt 2", + "26.11.1.post2", + "test_free_threading.py", + "test_vector.py", + "FLAKY-CRASH 0xC0000005,3221225477", + "3.11.9", + "3.14.0", + "native-parity-windows-x64-py3.11", + ): + assert text in body, text + assert body.startswith("CRASH and FLAKY-CRASH in a native parity run.") + assert EM_DASH not in body + md = summary.read_text(encoding="utf-8") + assert md.startswith("### Parity crashes") + assert "**FLAKY-CRASH: 1 file(s)" in md and "**CRASH: 1 file(s)" in md + assert ( + "| windows-x64 | 3.11.9 | 26.11.1.post2 | `test_free_threading.py` | FLAKY-CRASH 0xC0000005,3221225477 | passed (0x00000000,0) |" + in md + ) + assert EM_DASH not in md + + +def test_a_dispatch_posts_and_a_pull_request_a_branch_push_and_a_fork_do_not(artifacts): + events = report.load_events(str(artifacts)) + posted = {} + for name, env in { + "dispatch": { + **RELEASE_ENV, + "GITHUB_EVENT_NAME": "workflow_dispatch", + "GITHUB_REF_TYPE": "branch", + }, + "pull_request": { + **RELEASE_ENV, + "GITHUB_EVENT_NAME": "pull_request", + "GITHUB_REF_TYPE": "branch", + }, + "branch push": { + **RELEASE_ENV, + "GITHUB_EVENT_NAME": "push", + "GITHUB_REF_TYPE": "branch", + }, + "schedule": { + **RELEASE_ENV, + "GITHUB_EVENT_NAME": "schedule", + "GITHUB_REF_TYPE": "branch", + }, + "fork": {**RELEASE_ENV, "GITHUB_REPOSITORY": "humemai/arcadedb-python"}, + }.items(): + gh = FakeGh() + assert report.report(events, env, run=gh) == 0 + posted[name] = len(gh.calls) + assert posted == { + "dispatch": 1, + "pull_request": 0, + "branch push": 0, + "schedule": 0, + "fork": 0, + } + + +def test_a_token_that_cannot_write_issues_is_a_visible_warning_and_never_a_failure( + artifacts, capsys +): + gh = FakeGh( + returncode=1, stderr="gh: Resource not accessible by integration (HTTP 403)" + ) + assert report.report(report.load_events(str(artifacts)), RELEASE_ENV, run=gh) == 0 + out = capsys.readouterr().out + assert len(gh.calls) == 1 + assert "::warning::the FLAKY-CRASH comment on #295 was skipped" in out + assert "HTTP 403" in out + # no token at all: gh is not even called + gh = FakeGh() + no_token = {k: v for k, v in RELEASE_ENV.items() if k != "GH_TOKEN"} + assert report.report(report.load_events(str(artifacts)), no_token, run=gh) == 0 + assert gh.calls == [] + assert ( + "::warning::the FLAKY-CRASH comment on #295 was skipped: no GH_TOKEN" + in capsys.readouterr().out + ) + + +def test_a_run_without_a_crash_writes_nothing_and_posts_nothing(tmp_path, capsys): + root = tmp_path / "a" + write_meta( + root, + "native-parity-windows-x64-py3.12", + "windows-x64", + "3.12.7", + "26.11.1.post2", + [], + ) + gh, summary = FakeGh(), tmp_path / "summary.md" + assert ( + report.report(report.load_events(str(root)), RELEASE_ENV, str(summary), run=gh) + == 0 + ) + assert gh.calls == [] and not summary.exists() + + +def test_a_broken_artifact_directory_does_not_fail_the_report(tmp_path, capsys): + (tmp_path / "a" / "x").mkdir(parents=True) + (tmp_path / "a" / "x" / "parity-wheel.meta.json").write_text("{not json") + assert report.main(["--artifacts", str(tmp_path / "a"), "--summary", ""]) == 0 + assert "could not be read" in capsys.readouterr().out + + +# --------------------------------------------------------------------------------------------------------------------------------- +# the workflows: the flag on Windows legs only, and issues: write on one job only +# --------------------------------------------------------------------------------------------------------------------------------- + + +def jobs_of(text): + """{job name: its text} for the jobs: block of a workflow file (a job starts at two spaces of indentation).""" + body = text.split("\njobs:\n", 1)[1] + parts = re.split(r"^ ([A-Za-z0-9_-]+):\n", body, flags=re.M) + return dict(zip(parts[1::2], parts[2::2])) + + +def test_the_workflow_passes_the_retry_flag_on_windows_legs_only(): + text = (WORKFLOWS / "test-native.yml").read_text(encoding="utf-8") + assert ( + text.count("--retry-crash") == 2 + ) # once in the comment of the header, once in the Windows branch + assert re.search( + r'if \[ "\$RUNNER_OS" = Windows \]; then retry=\(--retry-crash\); fi', text + ) + jobs = jobs_of(text) + assert "retry=(--retry-crash)" in jobs["test"] + assert '${retry[@]+"${retry[@]}"}' in jobs["test"] + # the other jobs that run the harness never pass it + for name, job in jobs.items(): + if name != "test": + assert "--retry-crash" not in job, name + # the examples, pyver and build jobs call no harness with it; the Windows-only condition is the only place it is set + assert jobs["pyver"].count("parity_wheel.py") == 1 + + +def test_issues_write_is_held_by_the_crash_report_job_and_the_release_call_only(): + jobs = jobs_of((WORKFLOWS / "test-native.yml").read_text(encoding="utf-8")) + for name, job in jobs.items(): + assert ("issues: write" in job) == (name == "crash-report"), name + report_job = jobs["crash-report"] + assert "continue-on-error: true" in report_job + assert "contents: read" in report_job + assert "native/wheel/report_flaky_crash.py" in report_job + release = jobs_of((WORKFLOWS / "release-native.yml").read_text(encoding="utf-8")) + for name, job in release.items(): + assert ("issues: write" in job) == (name == "test"), name + assert "uses: ./.github/workflows/test-native.yml" in release["test"] + + +def test_the_flaky_comment_text_has_no_em_dash_in_any_source_that_writes_it(): + for path in ( + WHEEL_DIR / "report_flaky_crash.py", + WHEEL_DIR / "parity_crash.py", + WHEEL_DIR / "parity_wheel.py", + ): + assert EM_DASH not in path.read_text(encoding="utf-8"), path.name + + +# --------------------------------------------------------------------------------------------------------------------------------- +# a process that dies after its result, a stale XML, a locked log: the harness records, stays strict where it must and never fails on housekeeping +# --------------------------------------------------------------------------------------------------------------------------------- + +STALE_XML = '' + + +def lock(monkeypatch, name, suffix, times): + """os. raises PermissionError for a path ending in `suffix`: `times` times (None: always). Returns the list of the calls it saw.""" + real, left, seen = getattr(os, name), [times], [] + + def wrapper(*args, **kwargs): + target = str(args[1] if name == "replace" else args[0]) + if target.endswith(suffix): + seen.append(target) + if left[0] is None or left[0] > 0: + if left[0] is not None: + left[0] -= 1 + raise PermissionError(13, "locked by a virus scanner", target) + return real(*args, **kwargs) + + monkeypatch.setattr(os, name, wrapper) + return seen + + +def test_a_process_that_dies_after_its_result_is_a_dirty_exit_and_fails_nothing( + harness, tmp_path +): + rows, meta, xmldir = harness( + "--files", "test_dirty.py", "--retry-crash", dirty=True + ) + row = rows["test_dirty.py"] + # the XML says one passed test: the status stays clean, the file is not retried (even with the retry on), the exit code is kept + assert ( + row["status"], + row["passed"], + row["failed"], + row["attempts"], + row["exit"], + ) == ("", "1", "0", "1", HIGH_TEXT) + assert rows["TOTAL"]["status"] == "timeouts=0 flaky_crash=0 crash=0" + assert [ + (e["kind"], e["file"], e["first_rc"], e["attempt"], e["log"]) + for e in meta["crash_events"] + ] == [("DIRTY-EXIT", "test_dirty.py", HIGH_TEXT, 1, "test_dirty.py.log")] + assert not (xmldir / "test_dirty.py.attempt1.log").exists() + # the report lists it in the job summary and in the comment on issue 295 + root = tmp_path / "artifacts" + (root / "native-parity-windows-x64-py3.14").mkdir(parents=True) + (root / "native-parity-windows-x64-py3.14" / "parity-wheel.meta.json").write_text( + json.dumps(meta) + ) + gh, summary = FakeGh(), tmp_path / "summary.md" + assert ( + report.report(report.load_events(str(root)), RELEASE_ENV, str(summary), run=gh) + == 0 + ) + md = summary.read_text(encoding="utf-8") + assert md.startswith("### Parity crashes") + assert ( + "**DIRTY-EXIT: 1 file(s) wrote their result and then the process died in shutdown.**" + in md + ) + assert ( + f"| windows-x64 | 3.14.0 | 26.11.1.post2 | `test_dirty.py` | DIRTY-EXIT {HIGH_TEXT} | not retried, result kept |" + in md + ) + assert "`test_dirty.py.log`" in md and "attempt1" not in md + assert len(gh.calls) == 1 + body = gh.calls[0][1] + assert body.startswith("DIRTY-EXIT in a native parity run.") + assert ( + f"DIRTY-EXIT {HIGH_TEXT}" in body + and "not retried, result kept" in body + and "did not fail the leg" in body + ) + assert EM_DASH not in body and EM_DASH not in md + + +def test_a_stale_xml_cannot_pass_for_the_result_of_a_file_that_crashes(harness): + rows, meta, xmldir = harness( + "--files", "test_dies_always.py", seed={"test_dies_always.py.xml": STALE_XML} + ) + row = rows["test_dies_always.py"] + assert row["status"] == f"CRASH(rc={HIGH_TEXT})" + assert (row["passed"], row["attempts"]) == ("0", "1") + assert not (xmldir / "test_dies_always.py.xml").exists() + + +def test_a_stale_xml_that_cannot_be_removed_is_still_not_the_result_of_this_run( + harness, monkeypatch, capsys +): + seen = lock(monkeypatch, "remove", ".xml", None) + monkeypatch.setattr(parity_wheel, "HOUSEKEEPING_PAUSE", 0) + rows, meta, xmldir = harness( + "--files", + "test_dies_always.py", + seed={"test_dies_always.py.xml": (STALE_XML, 600)}, + ) + row = rows["test_dies_always.py"] + assert row["status"] == f"CRASH(rc={HIGH_TEXT})" and row["passed"] == "0" + assert len(seen) == 2 # one try and one more after the pause + assert ( + "::warning::removing a stale test_dies_always.py.xml was skipped: PermissionError" + in capsys.readouterr().out + ) + + +def test_housekeeping_tries_twice_waits_once_and_never_raises(monkeypatch, capsys): + sleeps = [] + monkeypatch.setattr(parity_wheel, "HOUSEKEEPING_PAUSE", 0.25) + monkeypatch.setattr(parity_wheel.time, "sleep", sleeps.append) + calls = [] + + def once(): + calls.append(1) + if len(calls) < 2: + raise PermissionError("locked") + + assert parity_wheel.housekeeping("a log", once) is True + assert (len(calls), sleeps, capsys.readouterr().out) == (2, [0.25], "") + calls.clear() + sleeps.clear() + + def always(): + calls.append(1) + raise PermissionError("locked") + + assert parity_wheel.housekeeping("keeping a log", always) is False + assert (len(calls), sleeps) == (2, [0.25]) + assert capsys.readouterr().out.startswith( + "::warning::keeping a log was skipped: PermissionError" + ) + + +def test_a_log_that_is_locked_once_is_kept_after_a_short_wait( + harness, monkeypatch, capsys +): + monkeypatch.setattr(parity_wheel, "HOUSEKEEPING_PAUSE", 0) + seen = lock(monkeypatch, "replace", ".attempt1.log", 1) + rows, meta, xmldir = harness("--files", "test_dies_once.py", "--retry-crash") + assert rows["test_dies_once.py"]["status"] == f"FLAKY-CRASH(rc={HIGH_TEXT})" + assert len(seen) == 2 + assert (xmldir / "test_dies_once.py.attempt1.log").exists() + assert "::warning::" not in capsys.readouterr().out + + +def test_a_log_that_stays_locked_is_a_warning_and_the_result_still_counts( + harness, monkeypatch, capsys +): + monkeypatch.setattr(parity_wheel, "HOUSEKEEPING_PAUSE", 0) + seen = lock(monkeypatch, "replace", ".attempt1.log", None) + rows, meta, xmldir = harness("--files", "test_dies_once.py", "--retry-crash") + once = rows["test_dies_once.py"] + assert (once["status"], once["passed"], once["failed"], once["attempts"]) == ( + f"FLAKY-CRASH(rc={HIGH_TEXT})", + "1", + "0", + "2", + ) + assert len(seen) == 2 + assert not (xmldir / "test_dies_once.py.attempt1.log").exists() + assert ( + "::warning::keeping the log of the first attempt of test_dies_once.py was skipped: PermissionError" + in capsys.readouterr().out + ) + + +def test_an_empty_trace_that_cannot_be_deleted_is_a_warning_and_not_a_failure( + harness, monkeypatch, capsys +): + monkeypatch.setattr(parity_wheel, "HOUSEKEEPING_PAUSE", 0) + seen = lock(monkeypatch, "remove", ".fault.txt", None) + rows, meta, xmldir = harness( + "--files", "test_ok.py", seed={"test_ok.py.attempt1.fault.txt": ""} + ) + assert (rows["test_ok.py"]["status"], rows["test_ok.py"]["passed"]) == ("", "2") + assert len(seen) == 2 and (xmldir / "test_ok.py.attempt1.fault.txt").exists() + assert ( + "::warning::removing the empty trace of test_ok.py was skipped: PermissionError" + in capsys.readouterr().out + ) + + +NOT_RETRIED_EVENT = { + "file": "test_find_nearest_query_forms.py", + "kind": "CRASH", + "first_rc": "-11,SIGSEGV", + "retry_rc": "", + "retry": "not retried", + "status": "CRASH(rc=-11,SIGSEGV)", +} + + +def test_the_report_names_the_log_that_exists(tmp_path): + root = tmp_path / "a" + write_meta( + root, + "native-parity-linux-x86_64-py3.12", + "linux-x86_64", + "3.12.7", + "26.11.1.post2", + [NOT_RETRIED_EVENT], + retry=False, + ) + write_meta( + root, + "native-parity-windows-x64-py3.11", + "windows-x64", + "3.11.9", + "26.11.1.post2", + [FLAKY_EVENT], + ) + write_meta( + root, + "native-parity-windows-x64-py3.14", + "windows-x64", + "3.14.0", + "26.11.1.post2", + [CRASH_EVENT], + ) + events = report.load_events(str(root)) + where = {line.split("`")[1]: line for line in report._where(events)} + # a crash that was not retried (Linux, macOS, a spent budget) has name.log alone; attempt1.log exists only after a retry + assert ( + "`test_find_nearest_query_forms.py.log`" + in where["test_find_nearest_query_forms.py"] + ) + assert "attempt1.log" not in where["test_find_nearest_query_forms.py"] + assert "`test_free_threading.py.attempt1.log`" in where["test_free_threading.py"] + assert "`test_vector.py.attempt1.log`" in where["test_vector.py"] + body = report.comment_md( + events, "https://example.invalid/run", "1", "push", "26.11.1.post2" + ) + assert ( + "`test_find_nearest_query_forms.py.log`" in body + and "test_find_nearest_query_forms.py.attempt1.log" not in body + ) diff --git a/tests/test_parity_gate_awk.py b/tests/test_parity_gate_awk.py new file mode 100644 index 0000000..a2e8069 --- /dev/null +++ b/tests/test_parity_gate_awk.py @@ -0,0 +1,170 @@ +"""The gate of the native parity legs lets a FLAKY-CRASH row pass, says so, and fails every other status. + +test-native.yml reads parity-wheel.tsv with an awk program in the step "Gate: no failure, error, crash, timeout or uncounted skip" (the +`test` job and the `pyver` job carry the same program). These tests cut that program out of the workflow file and run it with the awk +of the machine on small tables, so the code that ships is the code that is tested. A retried crash (FLAKY-CRASH) must not block a +release, and it must be impossible to miss; a crash that stayed a crash, a timeout, a failing test and a skip must still fail the leg. +The file is not collected on Windows (tests/conftest.py WINDOWS_NOT_APPLICABLE): the gate runs in the bash of a runner, as in +test_release_preflight.py. +""" + +from __future__ import annotations + +import os +import re +import subprocess # nosec B404 - test-controlled child processes +from pathlib import Path + +import pytest + +WORKFLOW = ( + Path(__file__).resolve().parents[1] / ".github" / "workflows" / "test-native.yml" +) +HEADER = "file\tpassed\tfailed\terror\tnotyet\tna_jvm\tskipped\tstatus\tfailing_tests\tattempts\texit\n" + + +def gate_programs(): + """The awk program of every gate step in the workflow, by job.""" + text = WORKFLOW.read_text(encoding="utf-8") + found = re.findall(r"awk -F'\\t' '\n(.*?)' \"\$tsv\"", text, flags=re.S) + assert len(found) == 2, "the test job and the pyver job each carry one gate" + return found + + +def row(name, passed=3, failed=0, error=0, status="", attempts=1, exit_="0x00000000,0"): + return f"{name}\t{passed}\t{failed}\t{error}\t0\t0\t0\t{status}\t\t{attempts}\t{exit_}\n" + + +def total( + passed=10, failed=0, error=0, skipped=0, status="timeouts=0 flaky_crash=0 crash=0" +): + return f"TOTAL\t{passed}\t{failed}\t{error}\t0\t0\t{skipped}\t{status}\t\t\t\n" + + +def run_gate(program, tmp_path, *rows): + tsv = tmp_path / "parity-wheel.tsv" + tsv.write_text(HEADER + "".join(rows)) + return subprocess.run( # nosec B603 B607 - fixed argument list, no shell + [ + "awk", + "-F\\t", + program, + str(tsv), + ], # -F'\t' of the workflow: awk reads the backslash-t itself + capture_output=True, + text=True, + env={**os.environ, "KEY": "windows-x64", "PY": "3.14"}, + timeout=60, + ) + + +@pytest.fixture(params=[0, 1], ids=["test-job", "pyver-job"]) +def program(request): + return gate_programs()[request.param] + + +def test_a_clean_table_passes_and_prints_no_flaky_line(program, tmp_path): + r = run_gate(program, tmp_path, row("test_a.py"), total()) + assert r.returncode == 0, r.stderr + assert r.stdout.startswith("parity windows-x64 py3.14: passed=10 failed=0 error=0") + assert "FLAKY-CRASH" not in r.stdout + + +def test_a_file_that_crashed_and_passed_on_its_retry_passes_the_gate_loudly( + program, tmp_path +): + status = "FLAKY-CRASH(rc=0xC0000005,3221225477)" + r = run_gate( + program, + tmp_path, + row("test_a.py"), + row("test_free_threading.py", status=status, attempts=2), + total(), + ) + assert r.returncode == 0, r.stderr + assert ( + f"FLAKY-CRASH (died without a result, passed on the one retry; first exit code in brackets): test_free_threading.py({status})" + in r.stdout + ) + + +def test_a_crash_that_stayed_a_crash_fails_the_gate(program, tmp_path): + for status in ( + "CRASH(rc=0xC0000005,3221225477)", + "CRASH(rc=0xC0000005,3221225477;retry rc=0xC0000005,3221225477)", + "CRASH(rc=0xC0000005,3221225477;retry budget spent)", + ): + r = run_gate( + program, tmp_path, row("test_a.py", passed=0, status=status), total() + ) + assert r.returncode == 1, status + assert f"test_a.py({status})" in r.stderr, status + + +def test_a_flaky_file_does_not_hide_another_files_crash(program, tmp_path): + r = run_gate( + program, + tmp_path, + row( + "test_flaky.py", status="FLAKY-CRASH(rc=0xC0000005,3221225477)", attempts=2 + ), + row( + "test_dead.py", + passed=0, + status="CRASH(rc=0xC0000374,3221226356;retry rc=0xC0000374,3221226356)", + attempts=2, + ), + total(), + ) + assert r.returncode == 1 + assert "test_dead.py(CRASH" in r.stderr and "test_flaky.py" not in r.stderr + + +def test_a_flaky_file_whose_retry_had_a_failing_test_fails_the_gate(program, tmp_path): + # the status passes, the counts of the second run decide + r = run_gate( + program, + tmp_path, + row( + "test_flaky.py", + failed=1, + status="FLAKY-CRASH(rc=0xC0000005,3221225477)", + attempts=2, + ), + total(failed=1), + ) + assert r.returncode == 1 + + +def test_a_timeout_an_unreadable_xml_a_failure_and_a_skip_still_fail_the_gate( + program, tmp_path +): + assert ( + run_gate( + program, tmp_path, row("test_a.py", status="TIMEOUT"), total() + ).returncode + == 1 + ) + assert ( + run_gate( + program, tmp_path, row("test_a.py", status="BADXML"), total() + ).returncode + == 1 + ) + assert ( + run_gate( + program, tmp_path, row("test_a.py", failed=1), total(failed=1) + ).returncode + == 1 + ) + assert ( + run_gate(program, tmp_path, row("test_a.py"), total(skipped=1)).returncode == 1 + ) + assert run_gate(program, tmp_path, row("test_a.py")).returncode == 1 # no TOTAL row + + +def test_a_file_that_died_after_its_result_does_not_fail_the_gate(program, tmp_path): + # DIRTY-EXIT is a record in the metadata, not a status: the row is clean and only its exit column differs + r = run_gate(program, tmp_path, row("test_a.py", exit_="0x0000008B,139"), total()) + assert r.returncode == 0, r.stderr + assert "FLAKY-CRASH" not in r.stdout diff --git a/tests/test_vertex_load_columnar.py b/tests/test_vertex_load_columnar.py new file mode 100644 index 0000000..e5a8d87 --- /dev/null +++ b/tests/test_vertex_load_columnar.py @@ -0,0 +1,178 @@ +"""``insert_dataframe`` and ``insert_arrow`` load a vertex type through the columnar insert, and the stored vertices, their indexes and +their buckets are those of the old route, ``GraphBatch.create_vertices`` (ArcadeData/arcadedb-python #319). + +Both routes are ``newVertex``, ``set`` per property and ``save`` on the caller's thread inside a transaction, so the records, the +index entries and the bucket each vertex lands in come out the same; the one difference the engine code shows is that a GraphBatch +turns the write-ahead log off for its load (its ``useWAL`` default) while the columnar insert uses the database's own settings, as the +document loads of the same calls always did. The old route is the private reference here: a type loaded through +``graph_batch().create_vertices`` and a type loaded through ``insert_dataframe`` are compared value by value and type by type, with the +index lookups and the per-bucket record counts. A spy on ``create_vertices`` proves which route ran (the old code calls it, the new code +must not). Runs on both wheels. +""" + +import datetime as dt + +import numpy as np +import pandas as pd +import pyarrow as pa +import pytest + +from arcadedb.exceptions import ArcadeDBError + +N = 37 # not a multiple of the batch size or of the bucket count + + +def _type(db, name, buckets=3): + db.command("sql", f"CREATE VERTEX TYPE {name} BUCKETS {buckets}") + db.command("sql", f"CREATE PROPERTY {name}.id LONG") + db.command("sql", f"CREATE PROPERTY {name}.grp INTEGER") + db.command("sql", f"CREATE PROPERTY {name}.emb ARRAY_OF_FLOATS") + db.command("sql", f"CREATE INDEX ON {name} (id) UNIQUE") + db.command("sql", f"CREATE INDEX ON {name} (grp) NOTUNIQUE") + + +def _frame(n=N): + ids = np.arange(n, dtype=np.int64) + return pd.DataFrame( + { + "id": ids, + "grp": (ids % 5).astype(np.int32), + "name": [None if i % 9 == 0 else f"n{i}-é" for i in range(n)], + "score": [None if i % 4 == 0 else i * 0.25 for i in range(n)], + "born": pd.to_datetime( + [ + dt.datetime(1969, 12, 31, 23, 59, 59, 999999) + dt.timedelta(days=i) + for i in range(n) + ] + ), + "emb": [np.array([i, i + 0.5, -i], dtype=np.float32) for i in range(n)], + } + ) + + +def _rows_of(frame): + out = [] + for rec in frame.to_dict("records"): + row = {} + for k, v in rec.items(): + if v is None or (isinstance(v, float) and v != v): + v = None # an Arrow null, as the old route of insert_dataframe handed it over + elif isinstance(v, pd.Timestamp): + v = v.to_pydatetime() + elif isinstance(v, np.ndarray): + v = v.tolist() + elif isinstance(v, np.generic): + v = v.item() + row[k] = v + out.append(row) + return out + + +def _old_route(db, name, frame): + with db.graph_batch() as batch: + batch.create_vertices(name, _rows_of(frame)) + + +def _stored(db, name): + sql = f"SELECT FROM {name} ORDER BY id" # nosec B608 - test-owned type name + return [ + {k: (type(v).__name__, v) for k, v in rec.items() if not k.startswith("@")} + for rec in db.query("sql", sql).to_list() + ] + + +def _bucket_counts(db, name): + counts = [] + for i in range(3): + sql = f"SELECT count(*) AS n FROM bucket:{name}_{i}" # nosec B608 - test-owned names + counts.append(db.query("sql", sql).first().get("n")) + return sorted(counts) + + +def _index_view(db, name): + by_id, by_grp = {}, {} + for i in (0, 7, N - 1): + sql = f"SELECT id, name FROM {name} WHERE id = {i}" # nosec B608 - test-owned names + by_id[i] = [ + (r.get("id"), r.get("name")) for r in db.query("sql", sql).to_list() + ] + for g in range(5): + sql = f"SELECT id FROM {name} WHERE grp = {g}" # nosec B608 - test-owned names + by_grp[g] = sorted(r.get("id") for r in db.query("sql", sql).to_list()) + return by_id, by_grp + + +@pytest.fixture +def old_and_new(temp_db): + _type(temp_db, "Old") + _type(temp_db, "New") + _old_route(temp_db, "Old", _frame()) + return temp_db + + +def test_the_vertices_indexes_and_buckets_equal_those_of_graph_batch( + old_and_new, monkeypatch +): + db = old_and_new + with db.graph_batch() as batch: + graph_batch_class = type(batch) + + def refused(*args, **kwargs): + raise AssertionError( + "a vertex load through insert_dataframe went through GraphBatch.create_vertices" + ) + + monkeypatch.setattr(graph_batch_class, "create_vertices", refused) + assert db.insert_dataframe("New", _frame(), batch_size=10) == N + expected = _stored(db, "Old") + assert len(expected) == N and expected[7]["emb"][0] == "list" + assert expected[0].get("name", ("NoneType", None))[1] is None + assert sum(_bucket_counts(db, "Old")) == N and max(_bucket_counts(db, "Old")) < N + assert _stored(db, "New") == expected + assert _bucket_counts(db, "New") == _bucket_counts(db, "Old") + assert _index_view(db, "New") == _index_view(db, "Old") + assert db.lookup_by_key("New", ["id"], [7]).get("name") == "n7-é" + assert ( + db.query("sql", "SELECT outE().size() AS e FROM New WHERE id = 7") + .first() + .get("e") + == 0 + ) # a vertex + + +def test_insert_arrow_loads_a_vertex_type_the_same_way(old_and_new): + db = old_and_new + assert db.insert_arrow("New", pa.Table.from_pandas(_frame()), batch_size=16) == N + assert _stored(db, "New") == _stored(db, "Old") + + +def test_a_duplicate_key_fails_and_rolls_the_batch_back(temp_db): + _type(temp_db, "Dup") + frame = _frame(20) + frame.loc[15, "id"] = 3 # the unique index refuses it + with pytest.raises(ArcadeDBError, match="Duplicat"): + temp_db.insert_dataframe("Dup", frame, batch_size=50) + assert temp_db.is_transaction_active() is False + assert temp_db.query("sql", "SELECT count(*) AS n FROM Dup").first().get("n") == 0 + + +def test_a_vertex_load_refuses_to_run_inside_a_transaction_the_caller_opened(temp_db): + _type(temp_db, "Mine") + with temp_db.transaction(): + with pytest.raises( + ArcadeDBError, match="cannot run inside a transaction the caller opened" + ): + temp_db.insert_dataframe("Mine", _frame(5)) + assert temp_db.query("sql", "SELECT count(*) AS n FROM Mine").first().get("n") == 0 + assert temp_db.insert_dataframe("Mine", _frame(5)) == 5 # outside one it loads + + +def test_a_document_load_still_joins_the_callers_transaction(temp_db): + temp_db.command("sql", "CREATE DOCUMENT TYPE Doc") + with pytest.raises(RuntimeError, match="stop"): + with temp_db.transaction(): + temp_db.insert_dataframe( + "Doc", pd.DataFrame({"id": [1, 2, 3]}), batch_size=2 + ) + raise RuntimeError("stop") + assert temp_db.query("sql", "SELECT count(*) AS n FROM Doc").first().get("n") == 0 diff --git a/tests/test_vertex_load_review_fixes.py b/tests/test_vertex_load_review_fixes.py new file mode 100644 index 0000000..a0eea0c --- /dev/null +++ b/tests/test_vertex_load_review_fixes.py @@ -0,0 +1,92 @@ +"""Review of the vertex loads through the columnar insert (#335): a commit conflict is retried, a failed later batch is exact. + +The old route, ``GraphBatch.create_vertices``, rolled a vertex chunk back and wrote it again (with a back-off, up to ``commitRetries``) +when its commit raised a ``NeedRetryException``: an MVCC conflict with another writer, a lock timeout. The columnar insert is one plain +transaction per chunk, so the same conflict would have surfaced as an error with earlier chunks already committed. The chunk loop of +``BulkOps.insertColumns`` (native wheel) and ``DocumentBatcher.insertColumns`` (JVM wheel) now does the retry, only in a transaction it +opened itself. Runs on both wheels. + +* several threads load one vertex type with a NOTUNIQUE index on a handful of key values, in many small batches: every row is stored; +* a duplicate key in a later batch: the earlier batches stay committed, the failing batch leaves no row and no index entry, the message + states the committed count, and the fixed tail loads afterwards. +""" + +import threading + +import numpy as np +import pyarrow as pa +import pytest + +from arcadedb.exceptions import ArcadeDBError + + +def _count(db, name): + sql = f"SELECT count(*) AS n FROM {name}" # nosec B608 - test-owned type name + return db.query("sql", sql).first().get("n") + + +def _vertex_type(db, name, unique=False): + db.command("sql", f"CREATE VERTEX TYPE {name} BUCKETS 3") + db.command("sql", f"CREATE PROPERTY {name}.id LONG") + db.command("sql", f"CREATE PROPERTY {name}.grp INTEGER") + db.command("sql", f"CREATE INDEX ON {name} (grp) NOTUNIQUE") + if unique: + db.command("sql", f"CREATE INDEX ON {name} (id) UNIQUE") + + +def _table(first, n, grps=5): + ids = np.arange(first, first + n, dtype=np.int64) + return pa.table({"id": ids, "grp": pa.array(ids % grps, pa.int32())}) + + +@pytest.mark.parametrize("threads", [2, 4]) +def test_threads_loading_one_vertex_type_in_small_batches_lose_no_row(temp_db, threads): + _vertex_type(temp_db, "V") + batches, batch_size = 20, 25 + errors = [] + + def load(worker): + try: + table = _table(worker * 1_000_000, batches * batch_size) + n = temp_db.insert_arrow("V", table, batch_size=batch_size) + assert n == batches * batch_size + except BaseException as e: # noqa: BLE001 - reported below, in the test thread + errors.append(repr(e)) + + pool = [threading.Thread(target=load, args=(w,)) for w in range(threads)] + for t in pool: + t.start() + for t in pool: + t.join(timeout=300) + assert not errors, errors + total = threads * batches * batch_size + assert _count(temp_db, "V") == total + distinct = temp_db.query("sql", "SELECT count(DISTINCT id) AS n FROM V") + assert distinct.first().get("n") == total # no row written twice by a retry + by_group = temp_db.query("sql", "SELECT grp, count(*) AS n FROM V GROUP BY grp") + assert sum(r.get("n") for r in by_group.to_list()) == total # the index agrees + + +def test_a_duplicate_in_a_later_batch_keeps_the_earlier_batches_and_nothing_of_its_own( + temp_db, +): + _vertex_type(temp_db, "Dup", unique=True) + ids = np.arange(500, dtype=np.int64) + ids[250] = 7 # the third batch of five repeats a key that the first batch committed + table = pa.table({"id": ids, "grp": pa.array(ids % 5, pa.int32())}) + with pytest.raises(ArcadeDBError, match="Duplicat") as err: + temp_db.insert_arrow("Dup", table, batch_size=100) + assert "the 200 row(s) of the batches before this one were committed" in str( + err.value + ) + assert temp_db.is_transaction_active() is False + assert _count(temp_db, "Dup") == 200 # batches 1 and 2, whole + for key in (200, 249, 251, 299): # the failing batch left no index entry behind + assert temp_db.lookup_by_key("Dup", ["id"], [key]) is None + assert temp_db.lookup_by_key("Dup", ["id"], [7]).get("id") == 7 # the committed one + + ids[250] = 250 # the fixed tail loads + tail = pa.table({"id": ids[200:], "grp": pa.array(ids[200:] % 5, pa.int32())}) + assert temp_db.insert_arrow("Dup", tail, batch_size=100) == 300 + assert _count(temp_db, "Dup") == 500 + assert temp_db.lookup_by_key("Dup", ["id"], [250]).get("id") == 250 diff --git a/tests/test_vertex_vector_loads.py b/tests/test_vertex_vector_loads.py new file mode 100644 index 0000000..60718b1 --- /dev/null +++ b/tests/test_vertex_vector_loads.py @@ -0,0 +1,344 @@ +"""GraphBatch.create_vertices: float32 vectors of any property cross as one matrix, on both wheels (#337). + +A row value that is a Java ``float[]`` (what ``to_java_float_array`` returns on the JVM wheel) or a numpy float32 array (what it returns on +the native wheel) is a vector candidate for every property, declared ``ARRAY_OF_FLOATS`` or not. Before, the JVM wheel refused a Java +``float[]`` as a candidate and sent the whole load through the property-matrix path (a JPype conversion of every value of every row), and +both wheels wrote one tag per row for a float32 array. Now one float32 matrix per property and chunk crosses (``take_vector_columns``). + +This file pins what is stored (bit for bit, for both wheels, with the one call both wheels share) and the shared split itself. The crossing +counts are in test_graph_batch_vector_crossings_jvm.py (JPype) and test_native_vertex_vector_crossings.py (ctypes). +""" + +import numpy as np +import pytest + +import arcadedb +from arcadedb import type_conversion as tc + +N = 40 +TYPES = ["Declared", "Free"] # embedding declared ARRAY_OF_FLOATS, and not declared +SHAPES = ["to_java_float_array", "float32_ndarray"] + + +def _vectors(dim, n=N, seed=5): + """float32 values a decimal text of 6 digits would not hold: -0.0, the largest, the smallest normal and the smallest subnormal.""" + x = np.random.default_rng(seed).standard_normal((n, dim)).astype(np.float32) + x[0, 0] = np.float32(-0.0) + x[1, 0] = np.finfo(np.float32).max + x[1, 1] = np.finfo(np.float32).tiny + x[1, 2] = np.finfo(np.float32).smallest_subnormal + return x + + +def _make(shape): + if shape == "to_java_float_array": + return ( + arcadedb.to_java_float_array + ) # a float[] on the JVM wheel, a float32 ndarray on the native wheel + return lambda vector: vector + + +def _bits(stored): + return ( + np.array(stored, dtype=np.float64).astype(np.float32).view(np.uint32).tolist() + ) + + +@pytest.fixture +def db(temp_db): + for name in ("Declared", "Bulk", "One"): + temp_db.command("sql", f"CREATE VERTEX TYPE {name}") + temp_db.command("sql", f"CREATE PROPERTY {name}.id LONG") + temp_db.command("sql", "CREATE PROPERTY Declared.embedding ARRAY_OF_FLOATS") + temp_db.command("sql", "CREATE PROPERTY Declared.born DATETIME") + temp_db.command("sql", "CREATE VERTEX TYPE Free") + temp_db.command("sql", "CREATE PROPERTY Free.id LONG") + temp_db.command("sql", "CREATE PROPERTY Free.born DATETIME") + temp_db.command("sql", "CREATE PROPERTY Bulk.embedding ARRAY_OF_FLOATS") + temp_db.command("sql", "CREATE PROPERTY One.embedding ARRAY_OF_FLOATS") + return temp_db + + +def _stored(db, type_name, prop="embedding"): + sql = f"SELECT id, {prop} AS v FROM {type_name}" # nosec B608 - test-owned names + return {r["id"]: r["v"] for r in db.query("sql", sql).to_list()} + + +def _count(db, type_name): + sql = f"SELECT count(*) AS c FROM {type_name}" # nosec B608 - test-owned name + return db.query("sql", sql).first().get("c") + + +@pytest.mark.parametrize("type_name", TYPES) +@pytest.mark.parametrize("dim", [64, 96]) +@pytest.mark.parametrize("shape", SHAPES) +def test_float32_vectors_are_stored_bit_equal(db, type_name, dim, shape): + x = _vectors(dim) + make = _make(shape) + with db.graph_batch() as gb: + rids = gb.create_vertices( + type_name, [{"id": i, "embedding": make(x[i])} for i in range(N)] + ) + assert len(rids) == N and len(set(rids)) == N + stored = _stored(db, type_name) + assert sorted(stored) == list(range(N)) + for i in range(N): + assert _bits(stored[i]) == x[i].view(np.uint32).tolist() + for i in (0, 17, N - 1): # the RIDs come back in row order + sql = f"SELECT id FROM {rids[i]}" # nosec B608 - a RID from this test + assert db.query("sql", sql).first().get("id") == i + + +@pytest.mark.parametrize("shape", SHAPES) +def test_the_matrix_stores_what_the_per_vertex_path_stores(db, shape): + x = _vectors(32, 12) + rows = [{"id": i, "embedding": _make(shape)(x[i])} for i in range(12)] + with db.graph_batch() as gb: + gb.create_vertices("Bulk", rows) + with db.graph_batch() as gb: + for row in rows: + gb.create_vertex("One", **row) + assert _stored(db, "Bulk") == _stored(db, "One") + assert _count(db, "Bulk") == 12 + + +def test_float64_numpy_rows_keep_their_documented_values(db): + x64 = np.random.default_rng(3).random((N, 64)) * 1000.0 + 1e-3 + rows = [{"id": i, "embedding": x64[i]} for i in range(N)] + with db.graph_batch() as gb: + gb.create_vertices("Declared", rows) + gb.create_vertices("Free", rows) + declared, free = _stored(db, "Declared"), _stored(db, "Free") + for i in range(N): + # declared ARRAY_OF_FLOATS: the engine stores floats, so the doubles round to float32 + assert _bits(declared[i]) == x64[i].astype(np.float32).view(np.uint32).tolist() + # undeclared: the list keeps its doubles, exactly + assert np.array(free[i], dtype=np.float64).tolist() == x64[i].tolist() + + +def test_lists_of_floats_keep_their_documented_values(db): + x64 = np.random.default_rng(4).random((N, 40)) * 1000.0 + 1e-3 + rows = [{"id": i, "embedding": x64[i].tolist()} for i in range(N)] + with db.graph_batch() as gb: + gb.create_vertices("Declared", rows) + gb.create_vertices("Free", rows) + declared, free = _stored(db, "Declared"), _stored(db, "Free") + for i in range(N): + assert _bits(declared[i]) == x64[i].astype(np.float32).view(np.uint32).tolist() + assert np.array(free[i], dtype=np.float64).tolist() == x64[i].tolist() + + +@pytest.mark.parametrize("type_name", TYPES) +@pytest.mark.parametrize("shape", SHAPES) +def test_ragged_null_and_missing_vectors_take_the_fallback(db, type_name, shape): + x = _vectors(16, 6) + make = _make(shape) + rows = [ + {"id": 0, "embedding": make(x[0])}, + {"id": 1, "embedding": make(x[1][:9])}, # another length + {"id": 2, "embedding": None}, + {"id": 3}, # no vector + {"id": 4, "embedding": make(x[4])}, + None, # a vertex without properties + ] + with db.graph_batch() as gb: + assert len(gb.create_vertices(type_name, rows)) == 6 + stored = _stored(db, type_name) + assert ( + _count(db, type_name) == 6 and None in stored + ) # the vertex of the None row has no id + assert _bits(stored[0]) == x[0].view(np.uint32).tolist() + assert _bits(stored[1]) == x[1][:9].view(np.uint32).tolist() + assert stored[2] is None and stored[3] is None + assert _bits(stored[4]) == x[4].view(np.uint32).tolist() + + +@pytest.mark.parametrize("type_name", TYPES) +@pytest.mark.parametrize("shape", SHAPES) +def test_a_float64_row_among_float32_rows_loses_nothing(db, type_name, shape): + x = _vectors(16, 8) + make = _make(shape) + rows = [{"id": i, "embedding": make(x[i]), "tag": f"t{i}"} for i in range(8)] + rows[5] = {"id": 5, "tag": "t5", "embedding": make(x[5].astype(np.float64))} + with db.graph_batch() as gb: + gb.create_vertices(type_name, rows) + stored, tags = _stored(db, type_name), _stored(db, type_name, "tag") + assert [tags[i] for i in range(8)] == [f"t{i}" for i in range(8)] + for i in range(8): + if type_name == "Free" and i == 5 and shape == "float32_ndarray": + # a float64 array on an undeclared property keeps its doubles (it is no float32 vector) + assert ( + np.array(stored[i], dtype=np.float64).tolist() + == x[5].astype(np.float64).tolist() + ) + else: + assert _bits(stored[i]) == x[i].view(np.uint32).tolist() + + +@pytest.mark.parametrize("type_name", TYPES) +@pytest.mark.parametrize("shape", SHAPES) +def test_the_other_properties_stay_with_their_rows(db, type_name, shape): + x = _vectors(16, 12) + make = _make(shape) + rows = [] + for i in range(12): + row = { + "id": i, + "name": f"n{i}", + "score": i * 0.5, + "flag": i % 2 == 0, + "note": None, + "embedding": make(x[i]), + } + if i % 4 == 3: # another key order: the rows no longer share one + row = dict(reversed(list(row.items()))) + rows.append(row) + with db.graph_batch() as gb: + gb.create_vertices(type_name, rows) + sql = f"SELECT id, name, score, flag, embedding FROM {type_name} ORDER BY id" # nosec B608 - test-owned name + got = db.query("sql", sql).to_list() + assert [r["id"] for r in got] == list(range(12)) + for i, r in enumerate(got): + assert ( + r["name"] == f"n{i}" and r["score"] == i * 0.5 and r["flag"] is (i % 2 == 0) + ) + assert _bits(r["embedding"]) == x[i].view(np.uint32).tolist() + + +@pytest.mark.parametrize("type_name", TYPES) +@pytest.mark.parametrize("shape", SHAPES) +def test_two_vector_properties_and_a_date_time_in_one_load(db, type_name, shape): + from datetime import datetime + + x, y = _vectors(8, 20), _vectors(8, 20, seed=9) + make = _make(shape) + rows = [ + { + "id": i, + "embedding": make(x[i]), + "second": make(y[i]), + "born": datetime(2020, 1, 2, 3, 4, 5), + } + for i in range(20) + ] + with db.graph_batch() as gb: + gb.create_vertices(type_name, rows) + first, second = _stored(db, type_name), _stored(db, type_name, "second") + for i in range(20): + assert _bits(first[i]) == x[i].view(np.uint32).tolist() + assert _bits(second[i]) == y[i].view(np.uint32).tolist() + sql = f"SELECT born FROM {type_name} WHERE id = 7" # nosec B608 - test-owned name + assert db.query("sql", sql).first().get("born") == datetime(2020, 1, 2, 3, 4, 5) + + +@pytest.mark.parametrize("type_name", TYPES) +@pytest.mark.parametrize("shape", SHAPES) +def test_a_load_over_the_chunk_boundary(db, type_name, shape): + make = _make(shape) + with db.graph_batch() as gb: + # the rows per crossing: the JVM wheel's 100,000, the native wheel's commit_every (1000 unless graph_batch() got another value) + chunk = getattr(gb, "_vertex_chunk", None) or gb._BULK_CHUNK + n = chunk + 3 + x = _vectors(4, n) + rids = gb.create_vertices( + type_name, [{"id": i, "embedding": make(x[i])} for i in range(n)] + ) + assert len(rids) == n and len(set(rids)) == n + assert _count(db, type_name) == n + stored = _stored(db, type_name) + got = np.array([stored[i] for i in range(n)], dtype=np.float64).astype(np.float32) + assert (got.view(np.uint32) == x.view(np.uint32)).all() + for i in ( + 0, + chunk - 1, + chunk, + n - 1, + ): # the RIDs come back in row order across the boundary + sql = f"SELECT id FROM {rids[i]}" # nosec B608 - a RID from this test + assert db.query("sql", sql).first().get("id") == i + + +@pytest.mark.parametrize("shape", SHAPES) +def test_insert_many_takes_the_same_vectors(db, shape): + db.command("sql", "CREATE DOCUMENT TYPE Doc") + db.command("sql", "CREATE PROPERTY Doc.id LONG") + db.command("sql", "CREATE PROPERTY Doc.embedding ARRAY_OF_FLOATS") + x = _vectors(32, N) + make = _make(shape) + assert ( + db.insert_many("Doc", [{"id": i, "embedding": make(x[i])} for i in range(N)]) + == N + ) + stored = _stored(db, "Doc") + for i in range(N): + assert _bits(stored[i]) == x[i].view(np.uint32).tolist() + + +# ---- the split itself (no database): the one function both wheels call + + +def _rows(x, **extra): + return [{"a": i, "b": f"s{i}", "v": x[i], **extra} for i in range(len(x))] + + +def test_take_vector_columns_takes_a_float32_column_of_any_property(): + x = _vectors(5, 7) + names, matrices, rest = tc.take_vector_columns(_rows(x), (), undeclared=True) + assert names == ["v"] and matrices[0].dtype == np.float32 + assert matrices[0].shape == (7, 5) and matrices[0].flags.c_contiguous + assert matrices[0].tobytes() == x.tobytes() # bit for bit + assert rest == [{"a": i, "b": f"s{i}"} for i in range(7)] + assert all(list(row) == ["a", "b"] for row in rest) + + +def test_take_vector_columns_leaves_an_undeclared_column_that_is_no_float32_vector(): + x = _vectors(5, 7) + as_lists = [{"v": row.tolist()} for row in x] + as_doubles = [{"v": row.astype(np.float64)} for row in x] + assert tc.take_vector_columns(as_lists, (), undeclared=True) is None + assert tc.take_vector_columns(as_doubles, (), undeclared=True) is None + assert tc.take_vector_columns(as_lists, ["v"])[1][0].shape == (7, 5) + assert tc.take_vector_columns(as_doubles, ["v"])[1][0].shape == (7, 5) + assert ( + tc.take_vector_columns(_rows(x), ()) is None + ) # undeclared names are tried only when asked + + +def test_take_vector_columns_declines_a_ragged_null_or_missing_column(): + x = _vectors(5, 4) + for rows in ( + [{"v": x[0]}, {"v": x[1][:3]}], + [{"v": x[0]}, {"v": None}], + [{"v": x[0]}, {"w": 1}], + [{"v": x[0]}, None], + [{"v": np.zeros(0, dtype=np.float32)}], + [{"v": x}], # a 2-D array per row + ): + assert tc.take_vector_columns(rows, ["v"], undeclared=True) is None + + +def test_take_vector_columns_rebuilds_the_rest_rows_in_each_rows_own_key_order(): + x = _vectors(3, 2) + rows = [{"a": 1, "v": x[0], "b": 2}, {"b": 3, "v": x[1], "a": 4}] + rest = tc.take_vector_columns(rows, (), undeclared=True)[2] + assert rest == [{"a": 1, "b": 2}, {"b": 3, "a": 4}] + assert [list(row) for row in rest] == [["a", "b"], ["b", "a"]] + only = tc.take_vector_columns([{"v": x[0]}, {"v": x[1]}], (), undeclared=True)[2] + assert only == [{}, {}] and only[0] is not only[1] + two = [{"x": r, "y": r * 2, "k": i} for i, r in enumerate(x)] + names, _, rest = tc.take_vector_columns(two, (), undeclared=True) + assert sorted(names) == ["x", "y"] and rest == [{"k": 0}, {"k": 1}] + + +def test_take_vector_columns_takes_a_strided_or_read_only_row(): + wide = _vectors(10, 6) + strided = tc.take_vector_columns( + [{"v": wide[i, ::2]} for i in range(6)], (), undeclared=True + )[1][0] + assert strided.tobytes() == np.ascontiguousarray(wide[:, ::2]).tobytes() + frozen = wide.copy() + frozen.setflags(write=False) + matrix = tc.take_vector_columns( + [{"v": frozen[i]} for i in range(6)], (), undeclared=True + )[1][0] + assert matrix.tobytes() == wide.tobytes() and matrix.flags.writeable