CI: let the native lane build against the 26.12.1-SNAPSHOT engine (refs #289)

.github/workflows/test-native.yml

@@ -82,10 +82,16 @@ on:
         default: "26.12.1.dev0"
         type: string
       engine:
-        description: "Engine release (Maven Central) the image links; empty = the release segment of a final version, else the newest release"
+        description: "Engine the image links: a release (Maven Central), or for a devN version A.B.C-SNAPSHOT (Sonatype snapshot repository); empty = the release segment of a final version, else DEV_ENGINE"
         required: false
         default: ""
         type: string
+      legs:
+        description: "full = all four platforms, Python 3.12 and 3.14; pull-request = linux-x86_64 and Python 3.12 only (what a pull request runs)"
+        required: false
+        default: "full"
+        type: choice
+        options: [full, pull-request]
 
 # The group names the workflow literally: github.workflow is the CALLER's name in a workflow_call, and the group of release-native.yml
 # must differ from this one or the two would wait for each other.
@@ -142,6 +148,7 @@ jobs:
         env:
           INPUT_VERSION: ${{ inputs.version }}
           INPUT_ENGINE: ${{ inputs.engine }}
+          INPUT_RELEASE: ${{ inputs.release }}
         run: |
           set -euo pipefail
           version="${INPUT_VERSION:-$DEFAULT_VERSION}"
@@ -157,6 +164,13 @@ jobs:
             # a final or post wheel links exactly the engine release it is named after
             [ "$engine" = "$base" ] || { echo "::error::Version $version links engine $base, not $engine" >&2; exit 1; }
           fi
+          # A SNAPSHOT engine is for a manual devN run only: a final or post version is refused above (it links exactly its release), and a release
+          # run (release-native.yml passes `release: true`) never builds from a snapshot, so nothing it publishes can depend on one.
+          if [[ "$engine" == *-SNAPSHOT ]]; then
+            [[ "$engine" =~ ^[0-9]+\.[0-9]+\.[0-9]+-SNAPSHOT$ ]] || { echo "::error::Engine '$engine' is not A.B.C or A.B.C-SNAPSHOT" >&2; exit 1; }
+            [ "$final" = false ] || { echo "::error::A SNAPSHOT engine is for a devN version only" >&2; exit 1; }
+            [ "${INPUT_RELEASE:-}" != true ] || { echo "::error::A release never links a SNAPSHOT engine ($engine)" >&2; exit 1; }
+          fi
           { echo "version=$version"; echo "engine=$engine"; echo "final=$final"; } >> "$GITHUB_OUTPUT"
           echo "wheel version $version, engine $engine, final=$final"
 
@@ -165,11 +179,14 @@ jobs:
         env:
           EVENT: ${{ github.event_name }}
           RELEASE: ${{ inputs.release }}
+          LEGS: ${{ inputs.legs }}
         run: |
           set -euo pipefail
           run=true
           event="$EVENT"
           if [ "$RELEASE" = true ]; then event=release; fi
+          # a manual run may ask for the legs of a pull request (one platform, one Python): the runner cap is shared with upstream
+          if [ "$EVENT" = workflow_dispatch ] && [ "${LEGS:-full}" = pull-request ]; then event=pull_request; fi
           if [ "$EVENT" = "schedule" ]; then
             changed="$(git log --since='25 hours ago' --format=%h -- src native tests scripts examples pyproject.toml .github/workflows/test-native.yml | wc -l)"
             monday="$(date -u +%u)"
@@ -372,9 +389,13 @@ jobs:
             *) m2="$HOME/.m2/repository" ;;
           esac
           mkdir -p "$T/record"
+          # THIRD_PARTY_NOTICES.md names the engine release and its jars, so it cannot match a SNAPSHOT build (timestamped jars, no tag):
+          # a SNAPSHOT run skips that comparison. Such a wheel is never published (see the plan job).
+          extra=()
+          if [[ "$ENGINE" == *-SNAPSHOT ]]; then extra=(--skip-notices-check); fi
           uv run --no-project --python 3.12 python native/wheel/build_wheel.py --lib-dir "$T/lib" --version "$VERSION" --engine "$ENGINE" \
             --graalvm-home "$GRAALVM_HOME_P5" --m2-repo "$m2" \
-            --notices-out "$T/record/THIRD_PARTY_NOTICES.generated.md" --modules-out "$T/record/jdk-modules.txt" --out "$T/wheel"
+            --notices-out "$T/record/THIRD_PARTY_NOTICES.generated.md" --modules-out "$T/record/jdk-modules.txt" --out "$T/wheel" ${extra[@]+"${extra[@]}"}
           ls -l "$T/wheel/dist"
           shopt -s nullglob
           wheels=("$T"/wheel/dist/*.whl)

docs/modules/ROOT/pages/development/ci.adoc

@@ -65,6 +65,8 @@ The two packages get their engine from different places, so between releases the
 * `test-jvm.yml` and `test-python-examples.yml` take the engine jars from the `arcadedata/arcadedb` image tag, `latest` by default. `latest` is upstream's newest snapshot and moves nightly, which is why both workflows also run every night.
 * `test-native.yml` links `DEV_ENGINE`, the newest release on Maven Central, unless the tag is a final or post version (then it links the release the tag is named after) or the `engine` input of a manual run says otherwise.
 
+A manual run of `test-native.yml` with a `devN` version may set `engine` to `A.B.C-SNAPSHOT`: `native/pom.xml` then resolves the engine jars from the Sonatype snapshot repository (snapshots on, releases off, so a release version still comes from Maven Central alone). The notices comparison of `build_wheel.py` is skipped for such a run, because `THIRD_PARTY_NOTICES.md` names a release and its jars. The plan job refuses a SNAPSHOT engine for a final or post version and for a release run, so nothing that is published depends on one. The `legs` input of a manual run chooses all four platforms (`full`, the default) or the one platform and one Python of a pull request (`pull-request`).
+
 The dev-engine bump after each release, described in xref:development/release.adoc[Release], moves `DEV_ENGINE` to the new release and closes the gap until upstream publishes the next one.
 
 == Other rules

examples/15_import_database_vs_transactional_table_ingest.py

@@ -552,6 +552,33 @@ def validate_parity_or_fail(
         raise RuntimeError("Invalid benchmark run:\n- " + "\n- ".join(problems))
 
 
+_PARALLEL_NOTE_SHOWN = False
+
+
+def _parallel_setting(db, parallel) -> str:
+    """The `parallel` setting of IMPORT DATABASE, as ", `parallel` = N" or "".
+
+    `parallel` is a reserved word of the SQL grammar, so it can only be written in backticks. The 26.12.1 engine refuses
+    the quoted name as an unknown importer setting (ArcadeData/arcadedb#11367), so from that version on the setting is
+    left out and the importer uses its default.
+    """
+    global _PARALLEL_NOTE_SHOWN
+    raw = str(db.query("sql", "SELECT version() AS v").first().get("v"))
+    version = tuple(
+        int(p) if p.isdigit() else 0
+        for p in raw.split()[0].split("-")[0].split(".")[:3]
+    )
+    if version < (26, 12, 1):
+        return f", `parallel` = {parallel}"
+    if not _PARALLEL_NOTE_SHOWN:
+        _PARALLEL_NOTE_SHOWN = True
+        print(
+            "  note: this engine refuses a quoted `parallel` importer setting "
+            "(ArcadeData/arcadedb#11367); the importer's default is used"
+        )
+    return ""
+
+
 def run_import_database_load(
     db_path: Path,
     table_csv_map: Dict[str, Path],
@@ -587,8 +614,8 @@ def run_import_database_load(
                     f"documents = '{csv_url}', "
                     "documentsFileType = 'csv', "
                     f"documentType = '{table_name}', "
-                    f"commitEvery = {batch_size}, "
-                    f"`parallel` = {parallel}"
+                    f"commitEvery = {batch_size}"
+                    f"{_parallel_setting(db, parallel)}"
                 ),
             ).one()
 

examples/16_import_database_vs_transactional_graph_ingest.py

@@ -788,6 +788,33 @@ def run_graph_batch_graph_load(
     }
 
 
+_PARALLEL_NOTE_SHOWN = False
+
+
+def _parallel_setting(db, parallel) -> str:
+    """The `parallel` setting of IMPORT DATABASE, as ", `parallel` = N" or "".
+
+    `parallel` is a reserved word of the SQL grammar, so it can only be written in backticks. The 26.12.1 engine refuses
+    the quoted name as an unknown importer setting (ArcadeData/arcadedb#11367), so from that version on the setting is
+    left out and the importer uses its default.
+    """
+    global _PARALLEL_NOTE_SHOWN
+    raw = str(db.query("sql", "SELECT version() AS v").first().get("v"))
+    version = tuple(
+        int(p) if p.isdigit() else 0
+        for p in raw.split()[0].split("-")[0].split(".")[:3]
+    )
+    if version < (26, 12, 1):
+        return f", `parallel` = {parallel}"
+    if not _PARALLEL_NOTE_SHOWN:
+        _PARALLEL_NOTE_SHOWN = True
+        print(
+            "  note: this engine refuses a quoted `parallel` importer setting "
+            "(ArcadeData/arcadedb#11367); the importer's default is used"
+        )
+    return ""
+
+
 def run_import_database_graph_load(
     db_path: Path,
     vertices_csv_path: Path,
@@ -830,8 +857,8 @@ def run_import_database_graph_load(
                 "typeIdUnique = true, "
                 "edgeFromField = 'From', "
                 "edgeToField = 'To', "
-                f"commitEvery = {batch_size}, "
-                f"`parallel` = {parallel}"
+                f"commitEvery = {batch_size}"
+                f"{_parallel_setting(db, parallel)}"
             ),
         ).one()
 

native/pom.xml

@@ -34,6 +34,18 @@
     </dependencies>
   </dependencyManagement>
 
+  <!-- Only a manual dev run links a SNAPSHOT engine (test-native.yml, input `engine`, e.g. 26.12.1-SNAPSHOT): Maven Central has no
+       snapshots, so this repository (snapshots on, releases off) is where `-Darcadedb.version=A.B.C-SNAPSHOT` resolves. Releases are
+       never asked of it, so a release version (the default, and every release build) still comes from Maven Central alone. -->
+  <repositories>
+    <repository>
+      <id>central-snapshots</id>
+      <url>https://central.sonatype.com/repository/maven-snapshots/</url>
+      <releases><enabled>false</enabled></releases>
+      <snapshots><enabled>true</enabled></snapshots>
+    </repository>
+  </repositories>
+
   <dependencies>
     <dependency>
       <groupId>com.arcadedb</groupId>

native/scripts/build-image.sh

@@ -1,7 +1,7 @@
 #!/usr/bin/env bash
 # Build the native image (the shared library the `arcadedb` wheel ships): GraalVM CE 25.4.4.1.1, -O3, FFM unbound Highway
 # handle (fastops.mode 1), -H:-VectorAPISupport (CE crash oracle/graal#14751), engine jars of the pinned RELEASE (arcadedb.version, from
-# Maven Central, never a local SNAPSHOT), local symbols deleted (the default: a smaller image than the phase 7/8 profiling builds).
+# Maven Central; a manual dev run may pass A.B.C-SNAPSHOT, which native/pom.xml resolves from the Sonatype snapshot repository; never a local SNAPSHOT), local symbols deleted (the default: a smaller image than the phase 7/8 profiling builds).
 # MARCH (default compatibility = baseline x86-64, what a manylinux wheel promises, PEP 600; x86-64-v3 is the GraalVM CE 25.4 default and the faster reference).
 # usage: [MARCH=x86-64-v3] [XMX=12g] [NI_EXTRA="-H:NumberOfThreads=2"] build-image.sh [OUTDIR]     A laptop with other jobs runs it through its job gate; the CI calls it directly.
 # Platforms: Linux (libarcadedb.so; also jvector's Highway library, libjvector-x86_64.so or libjvector-aarch64.so, both are in the jvector 4.0.2 jar), macOS

tests/test_columnar_engine_smoke.py

@@ -0,0 +1,103 @@
+"""Columnar faceting in the engine: the ``FACETS`` clause and ``CREATE INDEX ... COLUMNAR`` (ArcadeDB 26.12.1).
+
+Both are new in engine 26.12.1 (ArcadeData/arcadedb pull requests #11338 to #11346). The engine of a lane is whatever the
+lane links: Maven Central's 26.11.1 for the native pull request gate, the ``latest`` image for the JVM lane, a
+``26.12.1-SNAPSHOT`` for a manual native run. The version is read through the test's database inside each test
+(``SELECT version()``, so the same check runs on the JVM and the native build), and both engines are asserted on: an older
+engine must refuse the new statements with its own error, a newer one must run them. The repository accepts no skips
+(scripts/check_test_skips.py).
+
+This is a smoke test: it checks that the rows of a ``FACETS`` query come back and that the statements do not raise. The facet
+buckets themselves (``ResultSet.getFacets()``) have no Python API yet; the engine's own tests (``query/facet``) cover them.
+On the native build it is also the first proof that the image knows the classes of the new parser and index code.
+"""
+
+import re
+
+import pytest
+
+from arcadedb import create_database
+
+FACETS_SINCE = (26, 12, 1)
+
+FACETS_QUERY = (
+    "SELECT id, category FROM FacetDoc WHERE score >= 20 ORDER BY id "
+    "FACETS category TERMS TOP 5, score TERMS LIMIT 3"
+)
+
+
+@pytest.fixture
+def test_db(tmp_path):
+    """Create a temporary test database."""
+    db = create_database(str(tmp_path / "test_columnar_engine_smoke_db"))
+    yield db
+    db.drop()
+
+
+def _engine_version(db):
+    """The engine's x.y.z through SQL `SELECT version()`; a SNAPSHOT or a build suffix is dropped."""
+    raw = str(db.query("sql", "SELECT version() AS v").first().get("v"))
+    parts = [
+        int(p) if p.isdigit() else 0
+        for p in raw.split()[0].split("-")[0].split(".")[:3]
+    ]
+    return tuple(parts + [0] * (3 - len(parts)))
+
+
+def _make_docs(db):
+    db.command("sql", "CREATE DOCUMENT TYPE FacetDoc")
+    db.command("sql", "CREATE PROPERTY FacetDoc.id INTEGER")
+    db.command("sql", "CREATE PROPERTY FacetDoc.category STRING")
+    db.command("sql", "CREATE PROPERTY FacetDoc.score INTEGER")
+    categories = ["a", "b", "a", "c", "a"]
+    with db.transaction():
+        for i, category in enumerate(categories):
+            db.command(
+                "sql",
+                "INSERT INTO FacetDoc SET id = ?, category = ?, score = ?",
+                i,
+                category,
+                10 * (i + 1),
+            )
+
+
+class TestColumnarEngineSmoke:
+    def test_facets_clause_returns_the_rows(self, test_db):
+        _make_docs(test_db)
+
+        if _engine_version(test_db) < FACETS_SINCE:
+            # An engine without the clause reads FACETS as a table alias and fails on the facet that follows it.
+            with pytest.raises(Exception, match="SQL syntax error"):
+                test_db.query("sql", FACETS_QUERY).to_list()
+            return
+
+        test_db.command("sql", "CREATE INDEX ON FacetDoc (category) COLUMNAR")
+        test_db.command("sql", "CREATE INDEX ON FacetDoc (score) COLUMNAR")
+
+        rows = test_db.query("sql", FACETS_QUERY).to_list()
+
+        # the rows are the page of the WHERE matches (ids 1 to 4, LIMIT 3); the facets cover all four matches
+        assert [r["id"] for r in rows] == [1, 2, 3]
+        assert [r["category"] for r in rows] == ["b", "a", "c"]
+
+    def test_columnar_index_create_and_drop(self, test_db):
+        _make_docs(test_db)
+
+        if _engine_version(test_db) < FACETS_SINCE:
+            with pytest.raises(
+                Exception, match=re.escape("Index type 'COLUMNAR' is not supported")
+            ):
+                test_db.command("sql", "CREATE INDEX ON FacetDoc (category) COLUMNAR")
+            return
+
+        test_db.command("sql", "CREATE INDEX ON FacetDoc (category) COLUMNAR")
+        # a COLUMNAR index is listed under the name `Type[property]_columnar` (and one row per bucket), not `Type[property]`
+        rows = test_db.query("sql", "SELECT FROM schema:indexes").to_list()
+        names = sorted(
+            (r.get("name"), r.get("indexType"), r.get("typeName")) for r in rows
+        )
+        assert "FacetDoc[category]_columnar" in [r.get("name") for r in rows], names
+        assert test_db.schema.exists_index("FacetDoc[category]_columnar")
+
+        test_db.command("sql", "DROP INDEX `FacetDoc[category]_columnar`")
+        assert not test_db.schema.exists_index("FacetDoc[category]_columnar")

tests/test_import_database.py

@@ -67,10 +67,38 @@ def _resource_path(name: str) -> Path:
     return Path(__file__).resolve().parent / "fixtures" / name
 
 
+# `parallel` is a reserved word in the SQL grammar, so an IMPORT DATABASE setting of that name can only be written in backticks. Since the engine
+# change that refuses unknown importer names (ArcadeData/arcadedb#11363) the quoted name reaches the importer with its backticks and is refused
+# (ArcadeData/arcadedb#11367). The two tests below pin the behaviour of each engine; when #11367 is fixed they go back to the plain form.
+QUOTED_PARALLEL_REFUSED_SINCE = (26, 12, 1)
+
+
+def _engine_version(db):
+    """The engine's x.y.z through SQL `SELECT version()`; a SNAPSHOT or a build suffix is dropped."""
+    raw = str(db.query("sql", "SELECT version() AS v").first().get("v"))
+    parts = [
+        int(p) if p.isdigit() else 0
+        for p in raw.split()[0].split("-")[0].split(".")[:3]
+    ]
+    return tuple(parts + [0] * (3 - len(parts)))
+
+
+def _quoted_parallel_refused(db, result_set):
+    """True, after checking the importer's reason, on an engine that refuses a quoted `parallel`."""
+    if _engine_version(db) < QUOTED_PARALLEL_REFUSED_SINCE:
+        return False
+    row = result_set.one()
+    assert row.get("result") == "FAIL", row.to_dict()
+    assert "Unknown importer setting" in str(row.get("reason")), row.to_dict()
+    return True
+
+
 def _import_result_ok(result_set):
     assert result_set is not None
     row = result_set.one()
-    assert row.get("result") == "OK"
+    assert (
+        row.get("result") == "OK"
+    ), row.to_dict()  # the whole row: the importer says why on a FAIL
     return row
 
 
@@ -97,6 +125,8 @@ def test_import_database_csv_documents_with_quoted_parallel_setting(
                 "`parallel` = 1"
             ),
         )
+        if _quoted_parallel_refused(db, result):
+            return
         _import_result_ok(result)
 
         count = db.query("sql", "SELECT count(*) as c FROM `Document`").one().get("c")
@@ -170,6 +200,8 @@ def test_import_database_csv_graph_vertices_and_edges_with_quoted_parallel(
                 "`parallel` = 1"
             ),
         )
+        if _quoted_parallel_refused(db, result):
+            return
         _import_result_ok(result)
 
         vertices_count = (
@@ -188,7 +220,7 @@ def test_import_database_xml_vertices(temp_db_path, sample_xml_path):
             "sql",
             (
                 f"IMPORT DATABASE {_file_url(sample_xml_path)} "
-                "WITH objectNestLevel = 1, entityType = 'VERTEX'"
+                "WITH objectNestLevel = 1"
             ),
         )
         _import_result_ok(result)