diff --git a/.github/workflows/test-native.yml b/.github/workflows/test-native.yml index aa3f14ec..dcf953e0 100644 --- a/.github/workflows/test-native.yml +++ b/.github/workflows/test-native.yml @@ -82,10 +82,16 @@ on: default: "26.12.1.dev0" type: string engine: - description: "Engine release (Maven Central) the image links; empty = the release segment of a final version, else the newest release" + description: "Engine the image links: a release (Maven Central), or for a devN version A.B.C-SNAPSHOT (Sonatype snapshot repository); empty = the release segment of a final version, else DEV_ENGINE" required: false default: "" type: string + legs: + description: "full = all four platforms, Python 3.12 and 3.14; pull-request = linux-x86_64 and Python 3.12 only (what a pull request runs)" + required: false + default: "full" + type: choice + options: [full, pull-request] # The group names the workflow literally: github.workflow is the CALLER's name in a workflow_call, and the group of release-native.yml # must differ from this one or the two would wait for each other. @@ -142,6 +148,7 @@ jobs: env: INPUT_VERSION: ${{ inputs.version }} INPUT_ENGINE: ${{ inputs.engine }} + INPUT_RELEASE: ${{ inputs.release }} run: | set -euo pipefail version="${INPUT_VERSION:-$DEFAULT_VERSION}" @@ -157,6 +164,13 @@ jobs: # a final or post wheel links exactly the engine release it is named after [ "$engine" = "$base" ] || { echo "::error::Version $version links engine $base, not $engine" >&2; exit 1; } fi + # A SNAPSHOT engine is for a manual devN run only: a final or post version is refused above (it links exactly its release), and a release + # run (release-native.yml passes `release: true`) never builds from a snapshot, so nothing it publishes can depend on one. + if [[ "$engine" == *-SNAPSHOT ]]; then + [[ "$engine" =~ ^[0-9]+\.[0-9]+\.[0-9]+-SNAPSHOT$ ]] || { echo "::error::Engine '$engine' is not A.B.C or A.B.C-SNAPSHOT" >&2; exit 1; } + [ "$final" = false ] || { echo "::error::A SNAPSHOT engine is for a devN version only" >&2; exit 1; } + [ "${INPUT_RELEASE:-}" != true ] || { echo "::error::A release never links a SNAPSHOT engine ($engine)" >&2; exit 1; } + fi { echo "version=$version"; echo "engine=$engine"; echo "final=$final"; } >> "$GITHUB_OUTPUT" echo "wheel version $version, engine $engine, final=$final" @@ -165,11 +179,14 @@ jobs: env: EVENT: ${{ github.event_name }} RELEASE: ${{ inputs.release }} + LEGS: ${{ inputs.legs }} run: | set -euo pipefail run=true event="$EVENT" if [ "$RELEASE" = true ]; then event=release; fi + # a manual run may ask for the legs of a pull request (one platform, one Python): the runner cap is shared with upstream + if [ "$EVENT" = workflow_dispatch ] && [ "${LEGS:-full}" = pull-request ]; then event=pull_request; fi if [ "$EVENT" = "schedule" ]; then changed="$(git log --since='25 hours ago' --format=%h -- src native tests scripts examples pyproject.toml .github/workflows/test-native.yml | wc -l)" monday="$(date -u +%u)" @@ -372,9 +389,13 @@ jobs: *) m2="$HOME/.m2/repository" ;; esac mkdir -p "$T/record" + # THIRD_PARTY_NOTICES.md names the engine release and its jars, so it cannot match a SNAPSHOT build (timestamped jars, no tag): + # a SNAPSHOT run skips that comparison. Such a wheel is never published (see the plan job). + extra=() + if [[ "$ENGINE" == *-SNAPSHOT ]]; then extra=(--skip-notices-check); fi uv run --no-project --python 3.12 python native/wheel/build_wheel.py --lib-dir "$T/lib" --version "$VERSION" --engine "$ENGINE" \ --graalvm-home "$GRAALVM_HOME_P5" --m2-repo "$m2" \ - --notices-out "$T/record/THIRD_PARTY_NOTICES.generated.md" --modules-out "$T/record/jdk-modules.txt" --out "$T/wheel" + --notices-out "$T/record/THIRD_PARTY_NOTICES.generated.md" --modules-out "$T/record/jdk-modules.txt" --out "$T/wheel" ${extra[@]+"${extra[@]}"} ls -l "$T/wheel/dist" shopt -s nullglob wheels=("$T"/wheel/dist/*.whl) diff --git a/docs/modules/ROOT/pages/development/ci.adoc b/docs/modules/ROOT/pages/development/ci.adoc index e8d92e94..57faa665 100644 --- a/docs/modules/ROOT/pages/development/ci.adoc +++ b/docs/modules/ROOT/pages/development/ci.adoc @@ -65,6 +65,8 @@ The two packages get their engine from different places, so between releases the * `test-jvm.yml` and `test-python-examples.yml` take the engine jars from the `arcadedata/arcadedb` image tag, `latest` by default. `latest` is upstream's newest snapshot and moves nightly, which is why both workflows also run every night. * `test-native.yml` links `DEV_ENGINE`, the newest release on Maven Central, unless the tag is a final or post version (then it links the release the tag is named after) or the `engine` input of a manual run says otherwise. +A manual run of `test-native.yml` with a `devN` version may set `engine` to `A.B.C-SNAPSHOT`: `native/pom.xml` then resolves the engine jars from the Sonatype snapshot repository (snapshots on, releases off, so a release version still comes from Maven Central alone). The notices comparison of `build_wheel.py` is skipped for such a run, because `THIRD_PARTY_NOTICES.md` names a release and its jars. The plan job refuses a SNAPSHOT engine for a final or post version and for a release run, so nothing that is published depends on one. The `legs` input of a manual run chooses all four platforms (`full`, the default) or the one platform and one Python of a pull request (`pull-request`). + The dev-engine bump after each release, described in xref:development/release.adoc[Release], moves `DEV_ENGINE` to the new release and closes the gap until upstream publishes the next one. == Other rules diff --git a/examples/15_import_database_vs_transactional_table_ingest.py b/examples/15_import_database_vs_transactional_table_ingest.py index af79a0ad..2aee7d7a 100644 --- a/examples/15_import_database_vs_transactional_table_ingest.py +++ b/examples/15_import_database_vs_transactional_table_ingest.py @@ -552,6 +552,33 @@ def validate_parity_or_fail( raise RuntimeError("Invalid benchmark run:\n- " + "\n- ".join(problems)) +_PARALLEL_NOTE_SHOWN = False + + +def _parallel_setting(db, parallel) -> str: + """The `parallel` setting of IMPORT DATABASE, as ", `parallel` = N" or "". + + `parallel` is a reserved word of the SQL grammar, so it can only be written in backticks. The 26.12.1 engine refuses + the quoted name as an unknown importer setting (ArcadeData/arcadedb#11367), so from that version on the setting is + left out and the importer uses its default. + """ + global _PARALLEL_NOTE_SHOWN + raw = str(db.query("sql", "SELECT version() AS v").first().get("v")) + version = tuple( + int(p) if p.isdigit() else 0 + for p in raw.split()[0].split("-")[0].split(".")[:3] + ) + if version < (26, 12, 1): + return f", `parallel` = {parallel}" + if not _PARALLEL_NOTE_SHOWN: + _PARALLEL_NOTE_SHOWN = True + print( + " note: this engine refuses a quoted `parallel` importer setting " + "(ArcadeData/arcadedb#11367); the importer's default is used" + ) + return "" + + def run_import_database_load( db_path: Path, table_csv_map: Dict[str, Path], @@ -587,8 +614,8 @@ def run_import_database_load( f"documents = '{csv_url}', " "documentsFileType = 'csv', " f"documentType = '{table_name}', " - f"commitEvery = {batch_size}, " - f"`parallel` = {parallel}" + f"commitEvery = {batch_size}" + f"{_parallel_setting(db, parallel)}" ), ).one() diff --git a/examples/16_import_database_vs_transactional_graph_ingest.py b/examples/16_import_database_vs_transactional_graph_ingest.py index 7f8dc619..5a7d0a50 100644 --- a/examples/16_import_database_vs_transactional_graph_ingest.py +++ b/examples/16_import_database_vs_transactional_graph_ingest.py @@ -788,6 +788,33 @@ def run_graph_batch_graph_load( } +_PARALLEL_NOTE_SHOWN = False + + +def _parallel_setting(db, parallel) -> str: + """The `parallel` setting of IMPORT DATABASE, as ", `parallel` = N" or "". + + `parallel` is a reserved word of the SQL grammar, so it can only be written in backticks. The 26.12.1 engine refuses + the quoted name as an unknown importer setting (ArcadeData/arcadedb#11367), so from that version on the setting is + left out and the importer uses its default. + """ + global _PARALLEL_NOTE_SHOWN + raw = str(db.query("sql", "SELECT version() AS v").first().get("v")) + version = tuple( + int(p) if p.isdigit() else 0 + for p in raw.split()[0].split("-")[0].split(".")[:3] + ) + if version < (26, 12, 1): + return f", `parallel` = {parallel}" + if not _PARALLEL_NOTE_SHOWN: + _PARALLEL_NOTE_SHOWN = True + print( + " note: this engine refuses a quoted `parallel` importer setting " + "(ArcadeData/arcadedb#11367); the importer's default is used" + ) + return "" + + def run_import_database_graph_load( db_path: Path, vertices_csv_path: Path, @@ -830,8 +857,8 @@ def run_import_database_graph_load( "typeIdUnique = true, " "edgeFromField = 'From', " "edgeToField = 'To', " - f"commitEvery = {batch_size}, " - f"`parallel` = {parallel}" + f"commitEvery = {batch_size}" + f"{_parallel_setting(db, parallel)}" ), ).one() diff --git a/native/pom.xml b/native/pom.xml index cd83d1ff..5d78fd36 100644 --- a/native/pom.xml +++ b/native/pom.xml @@ -34,6 +34,18 @@ + + + + central-snapshots + https://central.sonatype.com/repository/maven-snapshots/ + false + true + + + com.arcadedb diff --git a/native/scripts/build-image.sh b/native/scripts/build-image.sh index 11897802..fe88cd1d 100755 --- a/native/scripts/build-image.sh +++ b/native/scripts/build-image.sh @@ -1,7 +1,7 @@ #!/usr/bin/env bash # Build the native image (the shared library the `arcadedb` wheel ships): GraalVM CE 25.4.4.1.1, -O3, FFM unbound Highway # handle (fastops.mode 1), -H:-VectorAPISupport (CE crash oracle/graal#14751), engine jars of the pinned RELEASE (arcadedb.version, from -# Maven Central, never a local SNAPSHOT), local symbols deleted (the default: a smaller image than the phase 7/8 profiling builds). +# Maven Central; a manual dev run may pass A.B.C-SNAPSHOT, which native/pom.xml resolves from the Sonatype snapshot repository; never a local SNAPSHOT), local symbols deleted (the default: a smaller image than the phase 7/8 profiling builds). # MARCH (default compatibility = baseline x86-64, what a manylinux wheel promises, PEP 600; x86-64-v3 is the GraalVM CE 25.4 default and the faster reference). # usage: [MARCH=x86-64-v3] [XMX=12g] [NI_EXTRA="-H:NumberOfThreads=2"] build-image.sh [OUTDIR] A laptop with other jobs runs it through its job gate; the CI calls it directly. # Platforms: Linux (libarcadedb.so; also jvector's Highway library, libjvector-x86_64.so or libjvector-aarch64.so, both are in the jvector 4.0.2 jar), macOS diff --git a/tests/test_columnar_engine_smoke.py b/tests/test_columnar_engine_smoke.py new file mode 100644 index 00000000..f2568101 --- /dev/null +++ b/tests/test_columnar_engine_smoke.py @@ -0,0 +1,103 @@ +"""Columnar faceting in the engine: the ``FACETS`` clause and ``CREATE INDEX ... COLUMNAR`` (ArcadeDB 26.12.1). + +Both are new in engine 26.12.1 (ArcadeData/arcadedb pull requests #11338 to #11346). The engine of a lane is whatever the +lane links: Maven Central's 26.11.1 for the native pull request gate, the ``latest`` image for the JVM lane, a +``26.12.1-SNAPSHOT`` for a manual native run. The version is read through the test's database inside each test +(``SELECT version()``, so the same check runs on the JVM and the native build), and both engines are asserted on: an older +engine must refuse the new statements with its own error, a newer one must run them. The repository accepts no skips +(scripts/check_test_skips.py). + +This is a smoke test: it checks that the rows of a ``FACETS`` query come back and that the statements do not raise. The facet +buckets themselves (``ResultSet.getFacets()``) have no Python API yet; the engine's own tests (``query/facet``) cover them. +On the native build it is also the first proof that the image knows the classes of the new parser and index code. +""" + +import re + +import pytest + +from arcadedb import create_database + +FACETS_SINCE = (26, 12, 1) + +FACETS_QUERY = ( + "SELECT id, category FROM FacetDoc WHERE score >= 20 ORDER BY id " + "FACETS category TERMS TOP 5, score TERMS LIMIT 3" +) + + +@pytest.fixture +def test_db(tmp_path): + """Create a temporary test database.""" + db = create_database(str(tmp_path / "test_columnar_engine_smoke_db")) + yield db + db.drop() + + +def _engine_version(db): + """The engine's x.y.z through SQL `SELECT version()`; a SNAPSHOT or a build suffix is dropped.""" + raw = str(db.query("sql", "SELECT version() AS v").first().get("v")) + parts = [ + int(p) if p.isdigit() else 0 + for p in raw.split()[0].split("-")[0].split(".")[:3] + ] + return tuple(parts + [0] * (3 - len(parts))) + + +def _make_docs(db): + db.command("sql", "CREATE DOCUMENT TYPE FacetDoc") + db.command("sql", "CREATE PROPERTY FacetDoc.id INTEGER") + db.command("sql", "CREATE PROPERTY FacetDoc.category STRING") + db.command("sql", "CREATE PROPERTY FacetDoc.score INTEGER") + categories = ["a", "b", "a", "c", "a"] + with db.transaction(): + for i, category in enumerate(categories): + db.command( + "sql", + "INSERT INTO FacetDoc SET id = ?, category = ?, score = ?", + i, + category, + 10 * (i + 1), + ) + + +class TestColumnarEngineSmoke: + def test_facets_clause_returns_the_rows(self, test_db): + _make_docs(test_db) + + if _engine_version(test_db) < FACETS_SINCE: + # An engine without the clause reads FACETS as a table alias and fails on the facet that follows it. + with pytest.raises(Exception, match="SQL syntax error"): + test_db.query("sql", FACETS_QUERY).to_list() + return + + test_db.command("sql", "CREATE INDEX ON FacetDoc (category) COLUMNAR") + test_db.command("sql", "CREATE INDEX ON FacetDoc (score) COLUMNAR") + + rows = test_db.query("sql", FACETS_QUERY).to_list() + + # the rows are the page of the WHERE matches (ids 1 to 4, LIMIT 3); the facets cover all four matches + assert [r["id"] for r in rows] == [1, 2, 3] + assert [r["category"] for r in rows] == ["b", "a", "c"] + + def test_columnar_index_create_and_drop(self, test_db): + _make_docs(test_db) + + if _engine_version(test_db) < FACETS_SINCE: + with pytest.raises( + Exception, match=re.escape("Index type 'COLUMNAR' is not supported") + ): + test_db.command("sql", "CREATE INDEX ON FacetDoc (category) COLUMNAR") + return + + test_db.command("sql", "CREATE INDEX ON FacetDoc (category) COLUMNAR") + # a COLUMNAR index is listed under the name `Type[property]_columnar` (and one row per bucket), not `Type[property]` + rows = test_db.query("sql", "SELECT FROM schema:indexes").to_list() + names = sorted( + (r.get("name"), r.get("indexType"), r.get("typeName")) for r in rows + ) + assert "FacetDoc[category]_columnar" in [r.get("name") for r in rows], names + assert test_db.schema.exists_index("FacetDoc[category]_columnar") + + test_db.command("sql", "DROP INDEX `FacetDoc[category]_columnar`") + assert not test_db.schema.exists_index("FacetDoc[category]_columnar") diff --git a/tests/test_import_database.py b/tests/test_import_database.py index 9f1803fd..206833f9 100644 --- a/tests/test_import_database.py +++ b/tests/test_import_database.py @@ -67,10 +67,38 @@ def _resource_path(name: str) -> Path: return Path(__file__).resolve().parent / "fixtures" / name +# `parallel` is a reserved word in the SQL grammar, so an IMPORT DATABASE setting of that name can only be written in backticks. Since the engine +# change that refuses unknown importer names (ArcadeData/arcadedb#11363) the quoted name reaches the importer with its backticks and is refused +# (ArcadeData/arcadedb#11367). The two tests below pin the behaviour of each engine; when #11367 is fixed they go back to the plain form. +QUOTED_PARALLEL_REFUSED_SINCE = (26, 12, 1) + + +def _engine_version(db): + """The engine's x.y.z through SQL `SELECT version()`; a SNAPSHOT or a build suffix is dropped.""" + raw = str(db.query("sql", "SELECT version() AS v").first().get("v")) + parts = [ + int(p) if p.isdigit() else 0 + for p in raw.split()[0].split("-")[0].split(".")[:3] + ] + return tuple(parts + [0] * (3 - len(parts))) + + +def _quoted_parallel_refused(db, result_set): + """True, after checking the importer's reason, on an engine that refuses a quoted `parallel`.""" + if _engine_version(db) < QUOTED_PARALLEL_REFUSED_SINCE: + return False + row = result_set.one() + assert row.get("result") == "FAIL", row.to_dict() + assert "Unknown importer setting" in str(row.get("reason")), row.to_dict() + return True + + def _import_result_ok(result_set): assert result_set is not None row = result_set.one() - assert row.get("result") == "OK" + assert ( + row.get("result") == "OK" + ), row.to_dict() # the whole row: the importer says why on a FAIL return row @@ -97,6 +125,8 @@ def test_import_database_csv_documents_with_quoted_parallel_setting( "`parallel` = 1" ), ) + if _quoted_parallel_refused(db, result): + return _import_result_ok(result) count = db.query("sql", "SELECT count(*) as c FROM `Document`").one().get("c") @@ -170,6 +200,8 @@ def test_import_database_csv_graph_vertices_and_edges_with_quoted_parallel( "`parallel` = 1" ), ) + if _quoted_parallel_refused(db, result): + return _import_result_ok(result) vertices_count = ( @@ -188,7 +220,7 @@ def test_import_database_xml_vertices(temp_db_path, sample_xml_path): "sql", ( f"IMPORT DATABASE {_file_url(sample_xml_path)} " - "WITH objectNestLevel = 1, entityType = 'VERTEX'" + "WITH objectNestLevel = 1" ), ) _import_result_ok(result)