Merge the 26.11.1.post2 release line into main

.github/workflows/release-native.yml

@@ -25,7 +25,8 @@ name: Release native (arcadedb)
 #
 # Nothing here stores a token: the publish job authenticates with PyPI trusted publishing (OIDC), and only that job has the id-token
 # permission. The guard in the publish job refuses to publish anything that is not `arcadedb`, so this workflow cannot publish the
-# JVM `arcadedb-jvm` by accident.
+# JVM `arcadedb-jvm` by accident. The test call carries issues: write too, for the one job of test-native.yml that logs a crash retry
+# on issue 295 (see the comment at that call).
 
 on:
   push:
@@ -97,6 +98,12 @@ jobs:
     name: Native build and parity
     needs: validate-version
     uses: ./.github/workflows/test-native.yml
+    # issues: write is for the `crash-report` job of test-native.yml alone (one comment on issue 295 when a Windows test file crashed and
+    # passed on its retry). A called workflow cannot hold more permission than its caller gives it, so without this line the whole release
+    # run would refuse to start; the called workflow's own `permissions: contents: read` keeps every other job read-only.
+    permissions:
+      contents: read
+      issues: write
     with:
       version: ${{ needs.validate-version.outputs.version }}
       release: true

.github/workflows/test-native.yml

@@ -33,6 +33,12 @@
 #
 # A leg that passes uploads a marker (native-passed-<platform>-py<version>) and the wheel (native-wheel-<platform>); release-native.yml
 # reads them to decide what may be published. Do not rename these artifacts without changing release-native.yml.
+#
+# Crashes (ArcadeData/arcadedb-python#295): a test file whose process dies without a result is a CRASH row with its exit code, and fails the leg.
+# On the Windows legs only, the harness runs such a file once more in a fresh process (--retry-crash): a retry that produces a result is a
+# FLAKY-CRASH row, the leg may pass, and the `crash-report` job makes it loud (the job summary, and on a release or a dispatch one comment on
+# issue 295); a second crash, a failing test, a timeout and every Linux or macOS crash stay failures. Each Windows test process also writes a
+# faulthandler trace into the parity artifact (parity-wheel-xml/<file>.attemptN.fault.txt).
 name: Test native (arcadedb)
 
 on:
@@ -650,12 +656,16 @@ jobs:
           cores="$(nproc 2>/dev/null || sysctl -n hw.ncpu 2>/dev/null || echo 2)"
           limit=()
           if [ "$FINAL" = true ]; then limit=(--max-notyet 0); fi
+          # A file whose process dies without a result is run once more on Windows only (the crashes of issue 295 are Windows only);
+          # Linux and macOS stay strict. The harness marks a passing retry FLAKY-CRASH and the crash-report job logs it.
+          retry=()
+          if [ "$RUNNER_OS" = Windows ]; then retry=(--retry-crash); fi
           # macOS runs bash 3.2: under set -u an empty "${limit[@]}" is "unbound variable", so expand it only when set
           uv run --no-project --python "$PY" --with "$w" --with pytest --with setuptools --with numpy --with requests --with jpype1 --with pyarrow \
             --with polars --with pandas --with duckdb --with networkx --with "psycopg[binary]" --with adbc-driver-postgresql \
             --with redis --with neo4j --with ladybug \
             python native/wheel/parity_wheel.py --repo "$GITHUB_WORKSPACE" \
-              --out "$T/parity/parity-wheel.tsv" --cores "0-$((cores - 1))" ${limit[@]+"${limit[@]}"} | tee "$T/parity/parity-wheel.run.log"
+              --out "$T/parity/parity-wheel.tsv" --cores "0-$((cores - 1))" ${limit[@]+"${limit[@]}"} ${retry[@]+"${retry[@]}"} | tee "$T/parity/parity-wheel.run.log"
 
       - name: "Gate: no failure, error, crash, timeout or uncounted skip"
         run: |
@@ -665,10 +675,13 @@ jobs:
           awk -F'\t' '
             NR == 1 { next }
             $1 == "TOTAL" { passed = $2; failed = $3; err = $4; notyet = $5; na = $6; skipped = $7; tot = 1; next }
+            $8 ~ /^FLAKY-CRASH/ { flaky = flaky " " $1 "(" $8 ")"; next }
             $8 != "" { bad = bad " " $1 "(" $8 ")" }
             END {
               if (!tot) { print "no TOTAL row" > "/dev/stderr"; exit 1 }
               printf "parity %s py%s: passed=%s failed=%s error=%s notyet=%s na_jvm=%s skipped=%s\n", ENVIRON["KEY"], ENVIRON["PY"], passed, failed, err, notyet, na, skipped
+              # a file that crashed and passed when run once more: the leg may pass, the summary says it did only because of the retry
+              if (flaky != "") printf "FLAKY-CRASH (died without a result, passed on the one retry; first exit code in brackets):%s\n", flaky
               if (bad != "") { print "files with a status:" bad > "/dev/stderr"; exit 1 }
               if (passed + 0 == 0 || failed + 0 != 0 || err + 0 != 0) exit 1
               # this repository accepts no skips; NOTYET and NA-JVM are the two counted kinds of the native build
@@ -696,11 +709,44 @@ jobs:
           name: native-parity-${{ matrix.key }}-py${{ matrix.python }}
           path: |
             ${{ runner.temp }}/parity/parity-wheel.tsv
+            ${{ runner.temp }}/parity/parity-wheel.meta.json
             ${{ runner.temp }}/parity/parity-wheel.run.log
             ${{ runner.temp }}/parity/parity-wheel-xml/
           if-no-files-found: warn
           retention-days: 30
 
+  # The one job that may write to an issue: after the parity legs it reads their meta files (parity-wheel.meta.json in each native-parity-*
+  # artifact) and lists every FLAKY-CRASH and CRASH in the job summary; on a release (a tag push) or a dispatch it also posts ONE comment on
+  # issue 295 (native/wheel/report_flaky_crash.py decides: never on a pull request, only in ArcadeData/arcadedb-python). A log is not a gate: a token
+  # that cannot write issues, a missing artifact or a gh failure print a warning and the job still ends green. release-native.yml grants the
+  # issues permission to its call of this workflow, because a called workflow cannot hold more permission than its caller gives it.
+  crash-report:
+    name: Report parity crashes
+    needs: [plan, test]
+    if: ${{ !cancelled() && needs.plan.outputs.run == 'true' }}
+    runs-on: ubuntu-24.04
+    timeout-minutes: 10
+    continue-on-error: true
+    permissions:
+      contents: read
+      issues: write
+    steps:
+      - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
+        with:
+          persist-credentials: false
+
+      - uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333 # v8.0.2
+        continue-on-error: true
+        with:
+          pattern: native-parity-*
+          path: ${{ runner.temp }}/parity-artifacts
+
+      - name: List the crashes and log them on issue 295
+        continue-on-error: true
+        env:
+          GH_TOKEN: ${{ github.token }}
+        run: python3 -I native/wheel/report_flaky_crash.py --artifacts "$RUNNER_TEMP/parity-artifacts"
+
   # The examples of examples/ against the INSTALLED native wheel, so the scripts a new user copies are run on what `uv add arcadedb` installs.
   # Nothing is built here: the wheel is the one the build job uploaded (native-wheel-<key>), the file the parity job installs. The platforms are
   # those of the parity job and the Pythons are 3.12 and 3.14 (scripts/select_native_matrix.py, `examples`). The runner is scripts/run_examples.sh, the
@@ -1031,10 +1077,13 @@ jobs:
           awk -F'\t' '
             NR == 1 { next }
             $1 == "TOTAL" { passed = $2; failed = $3; err = $4; notyet = $5; na = $6; skipped = $7; tot = 1; next }
+            $8 ~ /^FLAKY-CRASH/ { flaky = flaky " " $1 "(" $8 ")"; next }
             $8 != "" { bad = bad " " $1 "(" $8 ")" }
             END {
               if (!tot) { print "no TOTAL row" > "/dev/stderr"; exit 1 }
               printf "parity %s py%s: passed=%s failed=%s error=%s notyet=%s na_jvm=%s skipped=%s\n", ENVIRON["KEY"], ENVIRON["PY"], passed, failed, err, notyet, na, skipped
+              # a file that crashed and passed when run once more: the leg may pass, the summary says it did only because of the retry
+              if (flaky != "") printf "FLAKY-CRASH (died without a result, passed on the one retry; first exit code in brackets):%s\n", flaky
               if (bad != "") { print "files with a status:" bad > "/dev/stderr"; exit 1 }
               if (passed + 0 == 0 || failed + 0 != 0 || err + 0 != 0) exit 1
               # this repository accepts no skips; NOTYET and NA-JVM are the two counted kinds of the native build

docs/modules/ROOT/pages/development/ci.adoc

@@ -100,6 +100,7 @@ The workflow `test-native.yml` builds the native image of the `arcadedb` package
 * A pull request runs it only when native paths change: `native/**`, `src/arcadedb/**`, `tests/**`, `scripts/select_native_matrix.py` or `test-native.yml` itself. Changes under `tests/` and `src/arcadedb/` are included because both backends share those files, so a shared test or source can break native parity without touching a native file; docs outside those paths do not run it. It then runs on `linux-x86_64` with Python 3.12 only.
 * A push to `main`, the nightly run, a manual run (dispatch) run all four platforms: `linux-x86_64`, `linux-aarch64`, `macos-arm64` and `windows-x64` (Python 3.12 and 3.14). A release (`release-native.yml` calls the workflow with `release: true`) runs the same four platforms on all four Python versions, 16 legs.
 * The `pyver` job installs the already built `linux-x86_64` wheel (no rebuild) into the Pythons that the full parity runs of the event leave out (3.11, 3.13 and 3.14 on a pull request; 3.11 and 3.13 on a push to `main`, the nightly run and a dispatch; none on a release) into 3.15 and into the free-threaded 3.14t, and runs 13 test files there (14 where polars installs; 3.15 has no jpype1 wheel, so it leaves out the two files that need jpype1), about 20 seconds of tests per Python. It is informational: `release-native.yml` reads only the `native-passed-*` markers and the `native-wheel-*` wheels, so a red `pyver` leg never blocks a release. The 3.14t leg has no polars (no free-threaded wheel exists), so it skips `test_dataframe_ingest.py`. `scripts/select_native_matrix.py` decides its legs.
+* Every test file runs in its own process (`native/wheel/parity_wheel.py`). A process that dies without writing its result is a `CRASH` row in `parity-wheel.tsv` with its exit code, for example `CRASH(rc=0xC0000005,3221225477)`, and it fails the leg. On the Windows legs only (the crashes of issue 295 happen there), the harness runs such a file once more in a fresh process (`--retry-crash`). A retry that produces a result is a `FLAKY-CRASH` row: the gate lets the leg pass, the job summary lists the file with the first exit code, and on a release or a dispatch the `crash-report` job adds one comment to issue 295 (run URL, Python, wheel version, exit codes). A second crash, a failing test, a timeout and every crash on Linux or macOS stay failures. A process that wrote its result and then died in shutdown is a `DIRTY-EXIT` record (exit code, file, attempt) in the metadata, the job summary and the issue comment: it is not retried and does not fail the leg. Each Windows test process also writes a faulthandler trace, `parity-wheel-xml/<file>.attemptN.fault.txt`, into the parity artifact of the leg. Only a retry hides a crash, so treat a `FLAKY-CRASH` as an open bug: read the trace in the artifact before a release is announced.
 * A native image build takes about 5 to 30 minutes for each platform (macOS is the slowest). The organisation shares one runner limit with upstream ArcadeDB, so do not add runs lightly. Change the paths or the matrix of `test-native.yml` only for a reason that you can write down.
 
 == Change the selection

docs/modules/ROOT/pages/how-to/dataframes-and-columns.adoc

@@ -1,7 +1,7 @@
 = DataFrames, Arrow and columns
 :description: Load a pandas or Polars frame, an Arrow table or whole columns with insert_dataframe, insert_arrow and insert_columns, what each Arrow type becomes, how nulls and NaN are stored, what a declared property accepts, how many rows are converted at a time, what a failed load leaves, and what to_columns, to_arrow, to_dataframe and to_polars return, including the difference between the two builds.
 
-// Sources: src/arcadedb/core.py (insert_columns, insert_dataframe, insert_arrow), dataframe.py, results.py and _native/columns.py (to_columns, to_arrow, to_dataframe, to_polars). The statements below are pinned by tests/test_dataframe_columns_behavior.py, and for the Arrow types, the declared types, the column checks, the readers and the dates also by tests/test_dataframe_ingest.py, tests/test_insert_columns.py, tests/test_resultset_arrow.py, tests/test_columnar_readers.py and tests/test_resultset.py.
+// Sources: src/arcadedb/core.py (insert_columns, insert_dataframe, insert_arrow), dataframe.py, results.py, _columnar.py and _native/colframes.py (to_columns, to_arrow, to_dataframe, to_polars). The statements below are pinned by tests/test_dataframe_columns_behavior.py, and for the Arrow types, the declared types, the column checks, the readers and the dates also by tests/test_dataframe_ingest.py, tests/test_insert_columns.py, tests/test_resultset_arrow.py, tests/test_columnar_readers.py and tests/test_resultset.py.
 
 This page covers column-shaped data in and out of a database. Load it with `insert_dataframe` (a pandas or Polars frame), `insert_arrow` (an Arrow table) or `insert_columns` (numpy arrays and lists). Read a result as columns with `to_columns`, `to_arrow`, `to_dataframe` or `to_polars`. For a list of Python dicts use `insert_many`, and for a file use the importer: xref:how-to/bulk-loads.adoc[Bulk loads and commit size] and xref:how-to/import-export.adoc[Importing and exporting data]. The calls here are bindings calls and have no statement in SQL, so the main manual has nothing to say about them. The types that a stored value can have are the engine's: {arcadedb-docs-url}/reference/datatypes[Data types].
 
@@ -337,12 +337,7 @@ A missing library does not fail the same way everywhere. `to_columns()` returns
 
 == Differences between the two builds
 
-The calls above, the types and the results are the same on `arcadedb` and `arcadedb-jvm`. Two things differ. The first is how the readers get the rows:
-
-* On `arcadedb-jvm`, `to_columns()` and `to_arrow()` read the result from the engine in batches of `batch_size` rows, as packed columns, so `batch_size` sets how many rows cross at a time.
-* On `arcadedb` (native), they read every row into Python dicts first and build the columns from them. `batch_size` is accepted and not used, and the rows exist as Python objects while the columns are built.
-
-The second is the message of a missing pandas: the JVM package adds the `uv add` hint to the `ImportError` of `to_dataframe()`, and the native package raises the plain `ImportError`.
+The calls above, the types and the results are the same on `arcadedb` and `arcadedb-jvm`, and so is how `to_columns()` and `to_arrow()` get the rows: both read the result from the engine as packed columns, in requests of at most `batch_size` rows, so `batch_size` sets how many rows cross at a time. One thing differs, the message of a missing pandas: the JVM package adds the `uv add` hint to the `ImportError` of `to_dataframe()`, and the native package raises the plain `ImportError`.
 
 == Related pages
 

docs/modules/ROOT/pages/how-to/vectors.adoc

@@ -110,9 +110,9 @@ print(db.count_type("Chunk"))
 200
 ----
 
-`GraphBatch.create_vertices` rows and the properties of `new_edge` and `new_edges` take the same arrays. On the `arcadedb` (native) package, a non-empty one-dimensional `float32` array in a row or an edge property crosses in a binary side buffer, and the stored values are bit-identical to the ones the text path gave. A list or a `float64` array still crosses as number text, so make the array with `np.asarray(v, dtype=np.float32)` before the load. A laptop A/B on the native wheel (100,000 vertices for each load, `ab.py --noise`, ABBA order, 12 clean pairs for each figure, clean A/A control) gave this wall time of the new path against the old one (95 % interval of the median pair ratio): 0.517 [0.515, 0.520] at 128 dimensions, 0.218 [0.217, 0.219] at 96, and 0.271 [0.270, 0.272] at 64. A load without vectors was unchanged (1.009 [1.003, 1.012]).
+`GraphBatch.create_vertices` rows and the properties of `new_edge` and `new_edges` take the same arrays. On the `arcadedb` (native) package, a non-empty one-dimensional `float32` array in a row or an edge property crosses in a binary side buffer, and the stored values are bit-identical to the ones the text path gave. A list or a `float64` array still crosses as number text, so make the array with `np.asarray(v, dtype=np.float32)` before the load. When a property holds a `float32` array of one length in every row of a `create_vertices` call, the whole column crosses as one float32 matrix in that buffer, with no tag or Python call for each row. A laptop A/B on the native wheel (100,000 vertices for each load, `ab.py --noise`, ABBA order, 12 clean pairs for each figure, clean A/A control) gave this wall time of the new path against the old one (95 % interval of the median pair ratio): 0.517 [0.515, 0.520] at 128 dimensions, 0.218 [0.217, 0.219] at 96, and 0.271 [0.270, 0.272] at 64. A load without vectors was unchanged (1.009 [1.003, 1.012]).
 
-On the `arcadedb-jvm` package, `create_vertices` already sends the values of a property declared `ARRAY_OF_FLOATS` as one binary float32 matrix when every row holds a list or array of numbers of one length. So `float32` arrays in a property declared `ARRAY_OF_FLOATS` take the binary path on both packages.
+On the `arcadedb-jvm` package, `create_vertices` sends the values of a property declared `ARRAY_OF_FLOATS` as one binary float32 matrix when every row holds a list, a numpy array or a Java `float[]` or `double[]` of numbers of one length. A property that is not declared takes the same matrix when every row holds a `float32` numpy array or a Java `float[]` (what `to_java_float_array` returns on this package) of one length, and the stored value is a `float[]`, as before. A list or a `float64` array on an undeclared property keeps its doubles and the older path. So `float32` arrays and `to_java_float_array` values take the binary path on both packages, declared or not.
 
 == Search
 

docs/modules/ROOT/pages/installation.adoc

@@ -157,6 +157,11 @@ WARNING: *One JVM configuration per process.* JVM options are locked after the J
 
 Each Python process starts its own JVM, and each JVM sees the whole container limit. With several Python processes in one container, give each process its own `heap_size`.
 
+[[jvm-cds]]
+=== Class-data sharing
+
+The bundled JRE ships the JVM's default class-data-sharing archive, dumped with `-XX:+UseCompactObjectHeaders`, the flag that `start_jvm` sets. The JVM finds the archive by itself and maps the JDK's classes from it instead of loading and verifying them at every start. You do nothing to use it. The JVM ignores the archive, and starts as it did before, when your `jvm_args` change a setting that the archive depends on (`-XX:-UseCompactObjectHeaders`, `-Xshare:off`) or when the heap is larger than 32 GB (the default is half of the RAM, so a machine with more than 64 GB of RAM starts without the archive unless you set `heap_size`). The archive holds the JDK's classes only, not the engine's jars.
+
 [[jvm-aot]]
 === Faster start-up with the AOT cache