From 3491149691fe03c657d40402be09e859460d7a18 Mon Sep 17 00:00:00 2001 From: Taewoon Kim Date: Sat, 7 Nov 2026 02:32:43 +0900 Subject: [PATCH 1/7] CI: let the native lane build against the 26.12.1-SNAPSHOT engine (refs #289) Columnar faceting (FACETS, CREATE INDEX ... COLUMNAR) is merged in the engine but unreleased; Maven Central's newest arcadedb-engine is 26.11.1. The JVM lane already runs it (the `latest` image), the native lane could not: native/pom.xml has no snapshot repository. - native/pom.xml: a snapshots-on, releases-off repository for the Sonatype snapshot URL. A release version is never asked of it, so the PR gate (DEV_ENGINE 26.11.1) and every release build still resolve from Maven Central alone. - test-native.yml, plan job: a manual devN run may pass engine=A.B.C-SNAPSHOT; a final or post version, and a release run (release: true), are refused with it, so release-native.yml cannot publish from a snapshot even if DEV_ENGINE is later flipped. - test-native.yml, wheel step: a SNAPSHOT run skips the THIRD_PARTY_NOTICES.md comparison (it names the engine release and its jars; snapshot jars are timestamped and there is no tag for git show). The wheel version stays 26.12.1.dev0. - test-native.yml, new dispatch input `legs` (full | pull-request): a proof run needs one platform and one Python, not four platforms. - tests/test_columnar_engine_smoke.py: version-gated like test_vector_sql.py; an engine older than 26.12.1 must refuse FACETS and COLUMNAR with its own error, a newer one must return the rows and create and drop the index. No skips. On the native image it is the first proof of the reflection metadata for the new parser classes. DEV_ENGINE is not changed: flipping the gate is a later one-line change, pending the answer on #289. Co-Authored-By: Claude Sonnet 5.5 --- .github/workflows/test-native.yml | 25 +++++- docs/modules/ROOT/pages/development/ci.adoc | 2 + native/pom.xml | 12 +++ native/scripts/build-image.sh | 2 +- tests/test_columnar_engine_smoke.py | 97 +++++++++++++++++++++ 5 files changed, 135 insertions(+), 3 deletions(-) create mode 100644 tests/test_columnar_engine_smoke.py diff --git a/.github/workflows/test-native.yml b/.github/workflows/test-native.yml index aa3f14ec..dcf953e0 100644 --- a/.github/workflows/test-native.yml +++ b/.github/workflows/test-native.yml @@ -82,10 +82,16 @@ on: default: "26.12.1.dev0" type: string engine: - description: "Engine release (Maven Central) the image links; empty = the release segment of a final version, else the newest release" + description: "Engine the image links: a release (Maven Central), or for a devN version A.B.C-SNAPSHOT (Sonatype snapshot repository); empty = the release segment of a final version, else DEV_ENGINE" required: false default: "" type: string + legs: + description: "full = all four platforms, Python 3.12 and 3.14; pull-request = linux-x86_64 and Python 3.12 only (what a pull request runs)" + required: false + default: "full" + type: choice + options: [full, pull-request] # The group names the workflow literally: github.workflow is the CALLER's name in a workflow_call, and the group of release-native.yml # must differ from this one or the two would wait for each other. @@ -142,6 +148,7 @@ jobs: env: INPUT_VERSION: ${{ inputs.version }} INPUT_ENGINE: ${{ inputs.engine }} + INPUT_RELEASE: ${{ inputs.release }} run: | set -euo pipefail version="${INPUT_VERSION:-$DEFAULT_VERSION}" @@ -157,6 +164,13 @@ jobs: # a final or post wheel links exactly the engine release it is named after [ "$engine" = "$base" ] || { echo "::error::Version $version links engine $base, not $engine" >&2; exit 1; } fi + # A SNAPSHOT engine is for a manual devN run only: a final or post version is refused above (it links exactly its release), and a release + # run (release-native.yml passes `release: true`) never builds from a snapshot, so nothing it publishes can depend on one. + if [[ "$engine" == *-SNAPSHOT ]]; then + [[ "$engine" =~ ^[0-9]+\.[0-9]+\.[0-9]+-SNAPSHOT$ ]] || { echo "::error::Engine '$engine' is not A.B.C or A.B.C-SNAPSHOT" >&2; exit 1; } + [ "$final" = false ] || { echo "::error::A SNAPSHOT engine is for a devN version only" >&2; exit 1; } + [ "${INPUT_RELEASE:-}" != true ] || { echo "::error::A release never links a SNAPSHOT engine ($engine)" >&2; exit 1; } + fi { echo "version=$version"; echo "engine=$engine"; echo "final=$final"; } >> "$GITHUB_OUTPUT" echo "wheel version $version, engine $engine, final=$final" @@ -165,11 +179,14 @@ jobs: env: EVENT: ${{ github.event_name }} RELEASE: ${{ inputs.release }} + LEGS: ${{ inputs.legs }} run: | set -euo pipefail run=true event="$EVENT" if [ "$RELEASE" = true ]; then event=release; fi + # a manual run may ask for the legs of a pull request (one platform, one Python): the runner cap is shared with upstream + if [ "$EVENT" = workflow_dispatch ] && [ "${LEGS:-full}" = pull-request ]; then event=pull_request; fi if [ "$EVENT" = "schedule" ]; then changed="$(git log --since='25 hours ago' --format=%h -- src native tests scripts examples pyproject.toml .github/workflows/test-native.yml | wc -l)" monday="$(date -u +%u)" @@ -372,9 +389,13 @@ jobs: *) m2="$HOME/.m2/repository" ;; esac mkdir -p "$T/record" + # THIRD_PARTY_NOTICES.md names the engine release and its jars, so it cannot match a SNAPSHOT build (timestamped jars, no tag): + # a SNAPSHOT run skips that comparison. Such a wheel is never published (see the plan job). + extra=() + if [[ "$ENGINE" == *-SNAPSHOT ]]; then extra=(--skip-notices-check); fi uv run --no-project --python 3.12 python native/wheel/build_wheel.py --lib-dir "$T/lib" --version "$VERSION" --engine "$ENGINE" \ --graalvm-home "$GRAALVM_HOME_P5" --m2-repo "$m2" \ - --notices-out "$T/record/THIRD_PARTY_NOTICES.generated.md" --modules-out "$T/record/jdk-modules.txt" --out "$T/wheel" + --notices-out "$T/record/THIRD_PARTY_NOTICES.generated.md" --modules-out "$T/record/jdk-modules.txt" --out "$T/wheel" ${extra[@]+"${extra[@]}"} ls -l "$T/wheel/dist" shopt -s nullglob wheels=("$T"/wheel/dist/*.whl) diff --git a/docs/modules/ROOT/pages/development/ci.adoc b/docs/modules/ROOT/pages/development/ci.adoc index e8d92e94..57faa665 100644 --- a/docs/modules/ROOT/pages/development/ci.adoc +++ b/docs/modules/ROOT/pages/development/ci.adoc @@ -65,6 +65,8 @@ The two packages get their engine from different places, so between releases the * `test-jvm.yml` and `test-python-examples.yml` take the engine jars from the `arcadedata/arcadedb` image tag, `latest` by default. `latest` is upstream's newest snapshot and moves nightly, which is why both workflows also run every night. * `test-native.yml` links `DEV_ENGINE`, the newest release on Maven Central, unless the tag is a final or post version (then it links the release the tag is named after) or the `engine` input of a manual run says otherwise. +A manual run of `test-native.yml` with a `devN` version may set `engine` to `A.B.C-SNAPSHOT`: `native/pom.xml` then resolves the engine jars from the Sonatype snapshot repository (snapshots on, releases off, so a release version still comes from Maven Central alone). The notices comparison of `build_wheel.py` is skipped for such a run, because `THIRD_PARTY_NOTICES.md` names a release and its jars. The plan job refuses a SNAPSHOT engine for a final or post version and for a release run, so nothing that is published depends on one. The `legs` input of a manual run chooses all four platforms (`full`, the default) or the one platform and one Python of a pull request (`pull-request`). + The dev-engine bump after each release, described in xref:development/release.adoc[Release], moves `DEV_ENGINE` to the new release and closes the gap until upstream publishes the next one. == Other rules diff --git a/native/pom.xml b/native/pom.xml index cd83d1ff..5d78fd36 100644 --- a/native/pom.xml +++ b/native/pom.xml @@ -34,6 +34,18 @@ + + + + central-snapshots + https://central.sonatype.com/repository/maven-snapshots/ + false + true + + + com.arcadedb diff --git a/native/scripts/build-image.sh b/native/scripts/build-image.sh index 11897802..fe88cd1d 100755 --- a/native/scripts/build-image.sh +++ b/native/scripts/build-image.sh @@ -1,7 +1,7 @@ #!/usr/bin/env bash # Build the native image (the shared library the `arcadedb` wheel ships): GraalVM CE 25.4.4.1.1, -O3, FFM unbound Highway # handle (fastops.mode 1), -H:-VectorAPISupport (CE crash oracle/graal#14751), engine jars of the pinned RELEASE (arcadedb.version, from -# Maven Central, never a local SNAPSHOT), local symbols deleted (the default: a smaller image than the phase 7/8 profiling builds). +# Maven Central; a manual dev run may pass A.B.C-SNAPSHOT, which native/pom.xml resolves from the Sonatype snapshot repository; never a local SNAPSHOT), local symbols deleted (the default: a smaller image than the phase 7/8 profiling builds). # MARCH (default compatibility = baseline x86-64, what a manylinux wheel promises, PEP 600; x86-64-v3 is the GraalVM CE 25.4 default and the faster reference). # usage: [MARCH=x86-64-v3] [XMX=12g] [NI_EXTRA="-H:NumberOfThreads=2"] build-image.sh [OUTDIR] A laptop with other jobs runs it through its job gate; the CI calls it directly. # Platforms: Linux (libarcadedb.so; also jvector's Highway library, libjvector-x86_64.so or libjvector-aarch64.so, both are in the jvector 4.0.2 jar), macOS diff --git a/tests/test_columnar_engine_smoke.py b/tests/test_columnar_engine_smoke.py new file mode 100644 index 00000000..dc9ec4cf --- /dev/null +++ b/tests/test_columnar_engine_smoke.py @@ -0,0 +1,97 @@ +"""Columnar faceting in the engine: the ``FACETS`` clause and ``CREATE INDEX ... COLUMNAR`` (ArcadeDB 26.12.1). + +Both are new in engine 26.12.1 (ArcadeData/arcadedb pull requests #11338 to #11346). The engine of a lane is whatever the +lane links: Maven Central's 26.11.1 for the native pull request gate, the ``latest`` image for the JVM lane, a +``26.12.1-SNAPSHOT`` for a manual native run. The version is read through the test's database inside each test +(``SELECT version()``, so the same check runs on the JVM and the native build), and both engines are asserted on: an older +engine must refuse the new statements with its own error, a newer one must run them. The repository accepts no skips +(scripts/check_test_skips.py). + +This is a smoke test: it checks that the rows of a ``FACETS`` query come back and that the statements do not raise. The facet +buckets themselves (``ResultSet.getFacets()``) have no Python API yet; the engine's own tests (``query/facet``) cover them. +On the native build it is also the first proof that the image knows the classes of the new parser and index code. +""" + +import re + +import pytest + +from arcadedb import create_database + +FACETS_SINCE = (26, 12, 1) + +FACETS_QUERY = ( + "SELECT id, category FROM FacetDoc WHERE score >= 20 ORDER BY id " + "FACETS category TERMS TOP 5, score TERMS LIMIT 3" +) + + +@pytest.fixture +def test_db(tmp_path): + """Create a temporary test database.""" + db = create_database(str(tmp_path / "test_columnar_engine_smoke_db")) + yield db + db.drop() + + +def _engine_version(db): + """The engine's x.y.z through SQL `SELECT version()`; a SNAPSHOT or a build suffix is dropped.""" + raw = str(db.query("sql", "SELECT version() AS v").first().get("v")) + parts = [ + int(p) if p.isdigit() else 0 + for p in raw.split()[0].split("-")[0].split(".")[:3] + ] + return tuple(parts + [0] * (3 - len(parts))) + + +def _make_docs(db): + db.command("sql", "CREATE DOCUMENT TYPE FacetDoc") + db.command("sql", "CREATE PROPERTY FacetDoc.id INTEGER") + db.command("sql", "CREATE PROPERTY FacetDoc.category STRING") + db.command("sql", "CREATE PROPERTY FacetDoc.score INTEGER") + categories = ["a", "b", "a", "c", "a"] + with db.transaction(): + for i, category in enumerate(categories): + db.command( + "sql", + "INSERT INTO FacetDoc SET id = ?, category = ?, score = ?", + i, + category, + 10 * (i + 1), + ) + + +class TestColumnarEngineSmoke: + def test_facets_clause_returns_the_rows(self, test_db): + _make_docs(test_db) + + if _engine_version(test_db) < FACETS_SINCE: + # An engine without the clause reads FACETS as a table alias and fails on the facet that follows it. + with pytest.raises(Exception, match="SQL syntax error"): + test_db.query("sql", FACETS_QUERY).to_list() + return + + test_db.command("sql", "CREATE INDEX ON FacetDoc (category) COLUMNAR") + test_db.command("sql", "CREATE INDEX ON FacetDoc (score) COLUMNAR") + + rows = test_db.query("sql", FACETS_QUERY).to_list() + + # the rows are the page of the WHERE matches (ids 1 to 4, LIMIT 3); the facets cover all four matches + assert [r["id"] for r in rows] == [1, 2, 3] + assert [r["category"] for r in rows] == ["b", "a", "c"] + + def test_columnar_index_create_and_drop(self, test_db): + _make_docs(test_db) + + if _engine_version(test_db) < FACETS_SINCE: + with pytest.raises( + Exception, match=re.escape("Index type 'COLUMNAR' is not supported") + ): + test_db.command("sql", "CREATE INDEX ON FacetDoc (category) COLUMNAR") + return + + test_db.command("sql", "CREATE INDEX ON FacetDoc (category) COLUMNAR") + assert test_db.schema.exists_index("FacetDoc[category]") + + test_db.command("sql", "DROP INDEX `FacetDoc[category]`") + assert not test_db.schema.exists_index("FacetDoc[category]") From 2d93bb15262535758d8d34ebe79f047fa463f2e5 Mon Sep 17 00:00:00 2001 From: Taewoon Kim Date: Sat, 7 Nov 2026 05:12:04 +0900 Subject: [PATCH 2/7] columnar smoke test: name the schema:indexes rows when the COLUMNAR index is not listed as FacetDoc[category] The first native snapshot run passed the FACETS test and failed only on exists_index('FacetDoc[category]'), while the JVM lane passed. The message now shows what schema:indexes lists, so the next run says whether the native build lists the index under another name or not at all (a finding for #292). Co-Authored-By: Claude Sonnet 5.5 --- tests/test_columnar_engine_smoke.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/tests/test_columnar_engine_smoke.py b/tests/test_columnar_engine_smoke.py index dc9ec4cf..1f7f307b 100644 --- a/tests/test_columnar_engine_smoke.py +++ b/tests/test_columnar_engine_smoke.py @@ -91,6 +91,13 @@ def test_columnar_index_create_and_drop(self, test_db): return test_db.command("sql", "CREATE INDEX ON FacetDoc (category) COLUMNAR") + # the rows of schema:indexes are the same on both builds; the message names them if the index is not listed as expected + rows = test_db.query("sql", "SELECT FROM schema:indexes").to_list() + names = sorted( + (r.get("name"), r.get("indexType"), r.get("typeName")) for r in rows + ) + assert any(r.get("typeName") == "FacetDoc" for r in rows), names + assert "FacetDoc[category]" in [r.get("name") for r in rows], names assert test_db.schema.exists_index("FacetDoc[category]") test_db.command("sql", "DROP INDEX `FacetDoc[category]`") From d20fd80bae1747a66adfbdeacdfd4ea95d70fd5e Mon Sep 17 00:00:00 2001 From: Taewoon Kim Date: Sat, 7 Nov 2026 05:29:12 +0900 Subject: [PATCH 3/7] columnar smoke test: a COLUMNAR index is listed as Type[property]_columnar The native snapshot run showed schema:indexes holds FacetDoc[category]_columnar (type level) and one FacetDoc_0_ row per bucket; the inferred name FacetDoc[category] was wrong on any build, so this is not a native gap. Co-Authored-By: Claude Sonnet 5.5 --- tests/test_columnar_engine_smoke.py | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/tests/test_columnar_engine_smoke.py b/tests/test_columnar_engine_smoke.py index 1f7f307b..f2568101 100644 --- a/tests/test_columnar_engine_smoke.py +++ b/tests/test_columnar_engine_smoke.py @@ -91,14 +91,13 @@ def test_columnar_index_create_and_drop(self, test_db): return test_db.command("sql", "CREATE INDEX ON FacetDoc (category) COLUMNAR") - # the rows of schema:indexes are the same on both builds; the message names them if the index is not listed as expected + # a COLUMNAR index is listed under the name `Type[property]_columnar` (and one row per bucket), not `Type[property]` rows = test_db.query("sql", "SELECT FROM schema:indexes").to_list() names = sorted( (r.get("name"), r.get("indexType"), r.get("typeName")) for r in rows ) - assert any(r.get("typeName") == "FacetDoc" for r in rows), names - assert "FacetDoc[category]" in [r.get("name") for r in rows], names - assert test_db.schema.exists_index("FacetDoc[category]") + assert "FacetDoc[category]_columnar" in [r.get("name") for r in rows], names + assert test_db.schema.exists_index("FacetDoc[category]_columnar") - test_db.command("sql", "DROP INDEX `FacetDoc[category]`") - assert not test_db.schema.exists_index("FacetDoc[category]") + test_db.command("sql", "DROP INDEX `FacetDoc[category]_columnar`") + assert not test_db.schema.exists_index("FacetDoc[category]_columnar") From 0fb83fb6b013ac8229e22e1874f0024e87b312e9 Mon Sep 17 00:00:00 2001 From: Taewoon Kim Date: Sat, 7 Nov 2026 05:50:30 +0900 Subject: [PATCH 4/7] import tests: the assertion shows the whole result row, so a FAIL names its cause On the 26.12.1-SNAPSHOT native run three IMPORT DATABASE tests answer FAIL (CSV documents and CSV graph with a quoted parallel setting, XML vertices); the engine now refuses an import that read nothing (ArcadeData/arcadedb 4a91847a1f). The helper only compared the status word. Co-Authored-By: Claude Sonnet 5.5 --- tests/test_import_database.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/tests/test_import_database.py b/tests/test_import_database.py index 9f1803fd..1ddae83b 100644 --- a/tests/test_import_database.py +++ b/tests/test_import_database.py @@ -70,7 +70,9 @@ def _resource_path(name: str) -> Path: def _import_result_ok(result_set): assert result_set is not None row = result_set.one() - assert row.get("result") == "OK" + assert ( + row.get("result") == "OK" + ), row.to_dict() # the whole row: the importer says why on a FAIL return row From 553d8e288dbb8d148f9d6b01c40fe6d0369fdca4 Mon Sep 17 00:00:00 2001 From: Taewoon Kim Date: Sat, 7 Nov 2026 06:09:02 +0900 Subject: [PATCH 5/7] import test: drop entityType, which is not an importer setting 26.11.1 ignored unknown importer names; the 26.12.1-SNAPSHOT engine refuses them (Unknown importer setting 'entityType'). The XML import is driven by objectNestLevel; the statement behaves the same on 26.11.1 without it. The two tests that quote `parallel` are left for ArcadeData/arcadedb#11367. Co-Authored-By: Claude Sonnet 5.5 --- tests/test_import_database.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_import_database.py b/tests/test_import_database.py index 1ddae83b..92b03160 100644 --- a/tests/test_import_database.py +++ b/tests/test_import_database.py @@ -190,7 +190,7 @@ def test_import_database_xml_vertices(temp_db_path, sample_xml_path): "sql", ( f"IMPORT DATABASE {_file_url(sample_xml_path)} " - "WITH objectNestLevel = 1, entityType = 'VERTEX'" + "WITH objectNestLevel = 1" ), ) _import_result_ok(result) From a7490f4dc4353b9f50ac3039bdb7d6d69e2a27bc Mon Sep 17 00:00:00 2001 From: Taewoon Kim Date: Sat, 7 Nov 2026 06:18:33 +0900 Subject: [PATCH 6/7] import tests: pin the quoted parallel setting per engine until ArcadeData/arcadedb#11367 is answered PARALLEL is a reserved word, so an IMPORT DATABASE setting of that name can only be written in backticks (unquoted is a syntax error on 26.11.1). The 26.12.1 engine refuses the quoted name as unknown, which turned the JVM gate (it runs the latest engine image) red for both tests. Below 26.12.1 the import must answer OK as before; from 26.12.1 it must answer FAIL with the importer's reason. Co-Authored-By: Claude Sonnet 5.5 --- tests/test_import_database.py | 30 ++++++++++++++++++++++++++++++ 1 file changed, 30 insertions(+) diff --git a/tests/test_import_database.py b/tests/test_import_database.py index 92b03160..206833f9 100644 --- a/tests/test_import_database.py +++ b/tests/test_import_database.py @@ -67,6 +67,32 @@ def _resource_path(name: str) -> Path: return Path(__file__).resolve().parent / "fixtures" / name +# `parallel` is a reserved word in the SQL grammar, so an IMPORT DATABASE setting of that name can only be written in backticks. Since the engine +# change that refuses unknown importer names (ArcadeData/arcadedb#11363) the quoted name reaches the importer with its backticks and is refused +# (ArcadeData/arcadedb#11367). The two tests below pin the behaviour of each engine; when #11367 is fixed they go back to the plain form. +QUOTED_PARALLEL_REFUSED_SINCE = (26, 12, 1) + + +def _engine_version(db): + """The engine's x.y.z through SQL `SELECT version()`; a SNAPSHOT or a build suffix is dropped.""" + raw = str(db.query("sql", "SELECT version() AS v").first().get("v")) + parts = [ + int(p) if p.isdigit() else 0 + for p in raw.split()[0].split("-")[0].split(".")[:3] + ] + return tuple(parts + [0] * (3 - len(parts))) + + +def _quoted_parallel_refused(db, result_set): + """True, after checking the importer's reason, on an engine that refuses a quoted `parallel`.""" + if _engine_version(db) < QUOTED_PARALLEL_REFUSED_SINCE: + return False + row = result_set.one() + assert row.get("result") == "FAIL", row.to_dict() + assert "Unknown importer setting" in str(row.get("reason")), row.to_dict() + return True + + def _import_result_ok(result_set): assert result_set is not None row = result_set.one() @@ -99,6 +125,8 @@ def test_import_database_csv_documents_with_quoted_parallel_setting( "`parallel` = 1" ), ) + if _quoted_parallel_refused(db, result): + return _import_result_ok(result) count = db.query("sql", "SELECT count(*) as c FROM `Document`").one().get("c") @@ -172,6 +200,8 @@ def test_import_database_csv_graph_vertices_and_edges_with_quoted_parallel( "`parallel` = 1" ), ) + if _quoted_parallel_refused(db, result): + return _import_result_ok(result) vertices_count = ( From c05d1bb7b63119c026413719107982387295e567 Mon Sep 17 00:00:00 2001 From: Taewoon Kim Date: Sat, 7 Nov 2026 06:45:33 +0900 Subject: [PATCH 7/7] examples 15 and 16: leave out the quoted parallel setting on an engine that refuses it PARALLEL is a reserved word, so IMPORT DATABASE can only take it in backticks, and the 26.12.1 engine refuses the quoted name as an unknown importer setting (ArcadeData/arcadedb#11367). The Python examples job runs the latest engine and went red on both examples (the import answered FAIL, so the type was never created). From 26.12.1 the setting is left out and the importer default is used, with a one-line note; below it nothing changes. Co-Authored-By: Claude Sonnet 5.5 --- ..._database_vs_transactional_table_ingest.py | 31 +++++++++++++++++-- ..._database_vs_transactional_graph_ingest.py | 31 +++++++++++++++++-- 2 files changed, 58 insertions(+), 4 deletions(-) diff --git a/examples/15_import_database_vs_transactional_table_ingest.py b/examples/15_import_database_vs_transactional_table_ingest.py index af79a0ad..2aee7d7a 100644 --- a/examples/15_import_database_vs_transactional_table_ingest.py +++ b/examples/15_import_database_vs_transactional_table_ingest.py @@ -552,6 +552,33 @@ def validate_parity_or_fail( raise RuntimeError("Invalid benchmark run:\n- " + "\n- ".join(problems)) +_PARALLEL_NOTE_SHOWN = False + + +def _parallel_setting(db, parallel) -> str: + """The `parallel` setting of IMPORT DATABASE, as ", `parallel` = N" or "". + + `parallel` is a reserved word of the SQL grammar, so it can only be written in backticks. The 26.12.1 engine refuses + the quoted name as an unknown importer setting (ArcadeData/arcadedb#11367), so from that version on the setting is + left out and the importer uses its default. + """ + global _PARALLEL_NOTE_SHOWN + raw = str(db.query("sql", "SELECT version() AS v").first().get("v")) + version = tuple( + int(p) if p.isdigit() else 0 + for p in raw.split()[0].split("-")[0].split(".")[:3] + ) + if version < (26, 12, 1): + return f", `parallel` = {parallel}" + if not _PARALLEL_NOTE_SHOWN: + _PARALLEL_NOTE_SHOWN = True + print( + " note: this engine refuses a quoted `parallel` importer setting " + "(ArcadeData/arcadedb#11367); the importer's default is used" + ) + return "" + + def run_import_database_load( db_path: Path, table_csv_map: Dict[str, Path], @@ -587,8 +614,8 @@ def run_import_database_load( f"documents = '{csv_url}', " "documentsFileType = 'csv', " f"documentType = '{table_name}', " - f"commitEvery = {batch_size}, " - f"`parallel` = {parallel}" + f"commitEvery = {batch_size}" + f"{_parallel_setting(db, parallel)}" ), ).one() diff --git a/examples/16_import_database_vs_transactional_graph_ingest.py b/examples/16_import_database_vs_transactional_graph_ingest.py index 7f8dc619..5a7d0a50 100644 --- a/examples/16_import_database_vs_transactional_graph_ingest.py +++ b/examples/16_import_database_vs_transactional_graph_ingest.py @@ -788,6 +788,33 @@ def run_graph_batch_graph_load( } +_PARALLEL_NOTE_SHOWN = False + + +def _parallel_setting(db, parallel) -> str: + """The `parallel` setting of IMPORT DATABASE, as ", `parallel` = N" or "". + + `parallel` is a reserved word of the SQL grammar, so it can only be written in backticks. The 26.12.1 engine refuses + the quoted name as an unknown importer setting (ArcadeData/arcadedb#11367), so from that version on the setting is + left out and the importer uses its default. + """ + global _PARALLEL_NOTE_SHOWN + raw = str(db.query("sql", "SELECT version() AS v").first().get("v")) + version = tuple( + int(p) if p.isdigit() else 0 + for p in raw.split()[0].split("-")[0].split(".")[:3] + ) + if version < (26, 12, 1): + return f", `parallel` = {parallel}" + if not _PARALLEL_NOTE_SHOWN: + _PARALLEL_NOTE_SHOWN = True + print( + " note: this engine refuses a quoted `parallel` importer setting " + "(ArcadeData/arcadedb#11367); the importer's default is used" + ) + return "" + + def run_import_database_graph_load( db_path: Path, vertices_csv_path: Path, @@ -830,8 +857,8 @@ def run_import_database_graph_load( "typeIdUnique = true, " "edgeFromField = 'From', " "edgeToField = 'To', " - f"commitEvery = {batch_size}, " - f"`parallel` = {parallel}" + f"commitEvery = {batch_size}" + f"{_parallel_setting(db, parallel)}" ), ).one()