Dev release: pin one engine snapshot for both packages and record it in the wheels

.github/workflows/release-jvm.yml

@@ -47,7 +47,7 @@ jobs:
     outputs:
       version: ${{ steps.validate.outputs.version }}
       base-version: ${{ steps.validate.outputs.base-version }}
-      arcadedb-tag: ${{ steps.validate.outputs.arcadedb-tag }}
+      arcadedb-tag: ${{ steps.pin.outputs.arcadedb-tag || steps.validate.outputs.arcadedb-tag }}
     steps:
       - name: Validate tag
         id: validate
@@ -92,6 +92,22 @@ jobs:
           TAG: ${{ github.ref_name }}
         run: python3 -I scripts/check_version.py "$TAG" --git-remote origin --pypi "$DIST_NAME"
 
+      # A .devN tag embeds ONE upstream snapshot, the same one in both packages (scripts/dev_engine_pin.json): the image is named by digest
+      # (the tags `latest` and A.B.C-SNAPSHOT move on every upstream deploy), and the check proves that the image (amd64 and arm64) and the
+      # Sonatype jars the native build links hold the pinned jars, byte for byte. A final or .postN tag never gets here: it names its release.
+      - name: Pin the dev engine snapshot
+        id: pin
+        if: ${{ contains(github.ref_name, '.dev') }}
+        env:
+          TAG: ${{ github.ref_name }}
+        run: |
+          python3 -I scripts/dev_engine_pin.py check --tag "$TAG"
+          env_out="$(python3 -I scripts/dev_engine_pin.py shell-env "$TAG")"
+          ref="$(printf '%s\n' "$env_out" | sed -n 's|^IMAGE_REF=arcadedata/arcadedb:||p')"
+          [ -n "$ref" ] || { echo "::error::no image reference in the pin"; exit 1; }
+          echo "arcadedb-tag=$ref" >> "$GITHUB_OUTPUT"
+          echo "Engine snapshot pinned: $ref"
+
   # Builds one wheel for each platform (py3-none-<platform tag>, the same file for every Python) and
   # runs the unit tests on it, on the Pythons scripts/select_matrix.py picks. The called workflow
   # uploads the wheels as wheel-<platform> artifacts.

.github/workflows/release-native.yml

@@ -83,6 +83,14 @@ jobs:
           TAG: ${{ github.ref_name }}
         run: python3 -I scripts/check_version.py "$TAG" --git-remote origin --pypi "$DIST_NAME"
 
+      # A .devN tag links the pinned upstream snapshot (scripts/dev_engine_pin.json), the same one as arcadedb-jvm: the check proves that the
+      # JVM image and the Sonatype jars hold the pinned jars, byte for byte. test-native.yml checks again after the build.
+      - name: The dev engine snapshot is the pinned one
+        if: ${{ contains(github.ref_name, '.dev') }}
+        env:
+          TAG: ${{ github.ref_name }}
+        run: python3 -I scripts/dev_engine_pin.py check --tag "$TAG"
+
   # Builds a wheel per platform, runs the parity suite on every Python and uploads the wheels (native-wheel-<platform>) and one marker per
   # passed leg (native-passed-<platform>-py<version>).
   test:

.github/workflows/test-jvm.yml

@@ -306,10 +306,16 @@ jobs:
           echo "📌 ArcadeDB version: $ARCADEDB_TAG"
           # The build jobs pull arcadedata/arcadedb:<jar version> as their base image,
           # so a pinned tag must name that version exactly (26.10.2, not an alias).
-          if [ "$ARCADEDB_IMAGE_TAG" != "latest" ] && [ "$ARCADEDB_IMAGE_TAG" != "$ARCADEDB_TAG" ]; then
+          IMAGE_TAG_NAME="${ARCADEDB_IMAGE_TAG%%@*}"
+          if [ "$IMAGE_TAG_NAME" != "latest" ] && [ "$IMAGE_TAG_NAME" != "$ARCADEDB_TAG" ]; then
             echo "❌ arcadedb-tag '$ARCADEDB_IMAGE_TAG' holds engine $ARCADEDB_TAG; pass the exact version tag"
             exit 1
           fi
+          # A tag with a digest (<A.B.C-SNAPSHOT>@sha256:...) is the pinned snapshot of a .devN release (scripts/dev_engine_pin.json):
+          # the jars just downloaded must be byte for byte the pinned ones, the same ones the native build links.
+          if [[ "$ARCADEDB_IMAGE_TAG" == *@sha256:* ]]; then
+            python3 -I scripts/dev_engine_pin.py verify-jars src/arcadedb/jars
+          fi
           echo "arcadedb-tag=$ARCADEDB_TAG" >> "$GITHUB_OUTPUT"
 
       # Unfiltered on purpose: build-native.sh applies scripts/jar_exclusions.txt

.github/workflows/test-native.yml

@@ -157,19 +157,31 @@ jobs:
           base="${base%%.post*}"
           if [[ "$version" == *.dev* ]]; then
             final=false
-            engine="${INPUT_ENGINE:-$DEV_ENGINE}"
+            pin_engine=""
+            if [ "${INPUT_RELEASE:-}" = true ]; then
+              # a .devN RELEASE links the pinned snapshot, the one arcadedb-jvm embeds (scripts/dev_engine_pin.json), and nothing else
+              pin_env="$(python3 -I scripts/dev_engine_pin.py shell-env "$version")"
+              pin_engine="$(printf '%s\n' "$pin_env" | sed -n 's/^ENGINE=//p')"
+              [ -n "$pin_engine" ] || { echo "::error::no engine in the pin" >&2; exit 1; }
+              [ -z "$INPUT_ENGINE" ] || [ "$INPUT_ENGINE" = "$pin_engine" ] || { echo "::error::A dev release links the pinned engine $pin_engine, not $INPUT_ENGINE" >&2; exit 1; }
+              python3 -I scripts/dev_engine_pin.py check --tag "$version"
+              engine="$pin_engine"
+            else
+              engine="${INPUT_ENGINE:-$DEV_ENGINE}"
+            fi
           else
             final=true
             engine="${INPUT_ENGINE:-$base}"
             # a final or post wheel links exactly the engine release it is named after
             [ "$engine" = "$base" ] || { echo "::error::Version $version links engine $base, not $engine" >&2; exit 1; }
           fi
-          # A SNAPSHOT engine is for a manual devN run only: a final or post version is refused above (it links exactly its release), and a release
-          # run (release-native.yml passes `release: true`) never builds from a snapshot, so nothing it publishes can depend on one.
+          # A SNAPSHOT engine is for a devN version only: a final or post version is refused above (it links exactly its release). A release run
+          # (release-native.yml passes `release: true`) links a snapshot only for a .devN tag, and only the pinned one (scripts/dev_engine_pin.json,
+          # proven above); any other snapshot, and every snapshot of a final or post release, is refused. tests/test_dev_engine_pin.py runs this step.
           if [[ "$engine" == *-SNAPSHOT ]]; then
             [[ "$engine" =~ ^[0-9]+\.[0-9]+\.[0-9]+-SNAPSHOT$ ]] || { echo "::error::Engine '$engine' is not A.B.C or A.B.C-SNAPSHOT" >&2; exit 1; }
             [ "$final" = false ] || { echo "::error::A SNAPSHOT engine is for a devN version only" >&2; exit 1; }
-            [ "${INPUT_RELEASE:-}" != true ] || { echo "::error::A release never links a SNAPSHOT engine ($engine)" >&2; exit 1; }
+            [ "${INPUT_RELEASE:-}" != true ] || [ "$engine" = "${pin_engine:-}" ] || { echo "::error::A release never links a SNAPSHOT engine ($engine) except the pinned one of a .devN tag" >&2; exit 1; }
           fi
           { echo "version=$version"; echo "engine=$engine"; echo "final=$final"; } >> "$GITHUB_OUTPUT"
           echo "wheel version $version, engine $engine, final=$final"
@@ -380,6 +392,7 @@ jobs:
         env:
           VERSION: ${{ needs.plan.outputs.version }}
           ENGINE: ${{ needs.plan.outputs.engine }}
+          RELEASE: ${{ inputs.release }}
         run: |
           set -euo pipefail
           # the Linux container logged /root/.m2/repository in the native-image classpath; on the runner that is $T/m2/repository
@@ -390,9 +403,17 @@ jobs:
           esac
           mkdir -p "$T/record"
           # THIRD_PARTY_NOTICES.md names the engine release and its jars, so it cannot match a SNAPSHOT build (timestamped jars, no tag):
-          # a SNAPSHOT run skips that comparison. Such a wheel is never published (see the plan job).
+          # a SNAPSHOT run skips that comparison.
           extra=()
-          if [[ "$ENGINE" == *-SNAPSHOT ]]; then extra=(--skip-notices-check); fi
+          if [[ "$ENGINE" == *-SNAPSHOT ]]; then
+            extra=(--skip-notices-check)
+            if [ "${RELEASE:-}" = true ]; then
+              # a .devN release: the Sonatype jars the build resolved must still be the pinned ones (an upstream deploy during the build fails
+              # here, fail closed), and the wheel records the snapshot and the image digest it embeds
+              uv run --no-project --python 3.12 python -I scripts/dev_engine_pin.py check --tag "$VERSION"
+              extra+=(--engine-pin)
+            fi
+          fi
           uv run --no-project --python 3.12 python native/wheel/build_wheel.py --lib-dir "$T/lib" --version "$VERSION" --engine "$ENGINE" \
             --graalvm-home "$GRAALVM_HOME_P5" --m2-repo "$m2" \
             --notices-out "$T/record/THIRD_PARTY_NOTICES.generated.md" --modules-out "$T/record/jdk-modules.txt" --out "$T/wheel" ${extra[@]+"${extra[@]}"}

.github/workflows/test-python-examples.yml

@@ -185,10 +185,16 @@ jobs:
           echo "📌 ArcadeDB version: $ARCADEDB_TAG"
           # The build jobs pull arcadedata/arcadedb:<jar version> as their base image,
           # so a pinned tag must name that version exactly (26.10.2, not an alias).
-          if [ "$ARCADEDB_IMAGE_TAG" != "latest" ] && [ "$ARCADEDB_IMAGE_TAG" != "$ARCADEDB_TAG" ]; then
+          IMAGE_TAG_NAME="${ARCADEDB_IMAGE_TAG%%@*}"
+          if [ "$IMAGE_TAG_NAME" != "latest" ] && [ "$IMAGE_TAG_NAME" != "$ARCADEDB_TAG" ]; then
             echo "❌ arcadedb-tag '$ARCADEDB_IMAGE_TAG' holds engine $ARCADEDB_TAG; pass the exact version tag"
             exit 1
           fi
+          # A tag with a digest (<A.B.C-SNAPSHOT>@sha256:...) is the pinned snapshot of a .devN release (scripts/dev_engine_pin.json):
+          # the jars just downloaded must be byte for byte the pinned ones, the same ones the native build links.
+          if [[ "$ARCADEDB_IMAGE_TAG" == *@sha256:* ]]; then
+            python3 -I scripts/dev_engine_pin.py verify-jars src/arcadedb/jars
+          fi
           echo "arcadedb-tag=$ARCADEDB_TAG" >> "$GITHUB_OUTPUT"
 
       # Unfiltered on purpose: build-native.sh applies scripts/jar_exclusions.txt

README.md

@@ -38,6 +38,8 @@ uv add arcadedb-jvm   # JVM, for heavy long-running loads
 
 <!-- REMOVE WHEN: the old arcadedb-embedded* shims are retired and two releases have shipped with the two-package layout allow-old-name -->
 > The first release of this layout is 26.11.1, and both packages have a final release on PyPI. A plain `uv add arcadedb` installs the final, and a later pre-release (`A.B.C.devN`) needs `uv add --prerelease allow arcadedb`. The same holds for `arcadedb-jvm`.
+>
+> A `.devN` version (for example `26.12.1.dev0`, when one is published) is a development pre-release. It embeds a snapshot of the upstream engine, the same snapshot in both packages, pinned by image digest and jar checksums in `scripts/dev_engine_pin.json`, and the wheel states it (`arcadedb.ENGINE_SNAPSHOT`, `arcadedb.ENGINE_IMAGE_DIGEST`). Install it with `uv add --prerelease allow arcadedb==26.12.1.dev0` (or `arcadedb-jvm==26.12.1.dev0`).
 
 <!-- REMOVE WHEN: the old arcadedb-embedded* shims are retired and two releases have shipped with the two-package layout allow-old-name -->
 > Old names: `arcadedb-jvm` was formerly `arcadedb-embedded`, and `arcadedb` was formerly `arcadedb-embedded-native` (archived on PyPI). The last `arcadedb-embedded` release, 26.11.1, is a deprecation shim: it installs `arcadedb`, the native build, and warns once at import. It does not install `arcadedb-jvm`.

docs/modules/ROOT/pages/development/release.adoc

@@ -12,7 +12,7 @@ The tag is the bare version, with no `v`. The tag, the wheel version and the PyP
 | Version | Meaning
 
 | `A.B.C.dev0`, `A.B.C.dev1`, ...
-| A preview of engine release `A.B.C`, built from the upstream snapshot jars. `devN` starts at 0.
+| A development pre-release of engine release `A.B.C`, built from ONE pinned upstream snapshot (<<_the_dev_engine_snapshot,below>>). `devN` starts at 0.
 
 | `A.B.C`
 | The release. The wheels ship the `A.B.C` engine jar.
@@ -31,6 +31,17 @@ The rule has five parts:
 
 The base is the version of the engine. The workflow compares the tag with the version of the engine jar in the wheels, and it refuses a `devN` tag for a release jar and a release tag for a snapshot jar.
 
+[[_the_dev_engine_snapshot]]
+== The engine snapshot of a dev release
+
+Upstream publishes the snapshot engine by the day: the Docker image tags `latest` and `A.B.C-SNAPSHOT` move, and so does the Sonatype snapshot repository that the native build resolves. A `.devN` release therefore pins one snapshot for both packages in `scripts/dev_engine_pin.json`: the image by index digest, the timestamped snapshot version, and the sha256 of every `arcadedb-*.jar` in the image.
+
+* `scripts/dev_engine_pin.py check --tag <VERSION>` proves that the image (amd64 and arm64) and the Sonatype jars hold exactly the pinned jars. Both release workflows run it before anything is built, `release-native.yml` runs it again after the native build, and the JVM jobs download the image by digest and run `dev_engine_pin.py verify-jars` on the jars. A difference stops the release (exit 1); a source it cannot read stops it too (exit 2).
+* Only `A.B.C.devN` of the pinned base gets a snapshot. A final or `.postN` release refuses it, and a snapshot other than the pinned one is refused for a dev release (`tests/test_dev_engine_pin.py` runs the real step code).
+* Each wheel states what it embeds: `arcadedb.ENGINE_SNAPSHOT` (the timestamped version) and `arcadedb.ENGINE_IMAGE_DIGEST`.
+* To re-pin, run `uv run --no-project python -I scripts/dev_engine_pin.py capture <image-index-digest>` and replace the file in a pull request, before the tag.
+* The native notices comparison is skipped for a snapshot engine (`THIRD_PARTY_NOTICES.md` names a release), so a dev wheel is not checked against it.
+
 == How the rule is enforced
 
 `scripts/check_version.py` checks the rule. It reads the published versions from the Git tags of the repository and from PyPI, and it never changes anything:

native/wheel/build_wheel.py

@@ -145,6 +145,11 @@ def main():
         default="",
         help="also write the generated notices here (CI keeps it in the build record)",
     )
+    ap.add_argument(
+        "--engine-pin",
+        action="store_true",
+        help="--engine is the pinned snapshot of scripts/dev_engine_pin.json (a .devN release): record its timestamped version and image digest",
+    )
     ap.add_argument(
         "--skip-notices-check",
         action="store_true",
@@ -157,6 +162,17 @@ def main():
     )
     ap.add_argument("--plat", default="", help="override the computed platform tag")
     a = ap.parse_args()
+    pin = None
+    if a.engine_pin:
+        sys.path.insert(0, str(REPO / "scripts"))
+        import dev_engine_pin
+
+        pin = dev_engine_pin.load_pin()
+        problem = dev_engine_pin.tag_problem(a.version, pin)
+        if problem:
+            sys.exit(problem)
+        if a.engine != pin["engine"]:
+            sys.exit(f"--engine {a.engine} is not the pinned engine {pin['engine']}")
     lib_dir, out = Path(a.lib_dir), Path(a.out).resolve()
     libname = LIB_NAMES.get(sys.platform) or sys.exit(
         f"unsupported platform {sys.platform}"
@@ -246,6 +262,8 @@ def sub(m):
     (dst / "_build_info.py").write_text(
         f'"""Written by native-wheel/build_wheel.py."""\n'
         f'ENGINE_VERSION = "{a.engine}"  # the ArcadeDB engine release compiled into {libname}\n'
+        f'ENGINE_SNAPSHOT = {pin["snapshot"] if pin else None!r}  # the exact upstream snapshot (scripts/dev_engine_pin.json) of a .devN wheel\n'
+        f'ENGINE_IMAGE_DIGEST = {pin["image_digest"] if pin else None!r}  # the arcadedata/arcadedb image the same snapshot was pinned by\n'
         f'NATIVE_IMAGE = "GraalVM CE 25.4.4.1.1, -O3, -march={a.march}"\n'
         f'NATIVE_MARCH = "{a.march}"  # lib.py checks the CPU only for an x86-64-v3 image\n'
         f'SOURCE_COMMIT = "{commit}"  # ArcadeData/arcadedb-python commit of the bindings and the Java glue\n',
@@ -255,7 +273,7 @@ def sub(m):
     t = ip.read_text(encoding="utf-8")
     t = t.replace(
         "# Import async execution\n",
-        "from ._build_info import ENGINE_VERSION, NATIVE_IMAGE, NATIVE_MARCH, SOURCE_COMMIT  # noqa: F401\n\n# Import async execution\n",
+        "from ._build_info import ENGINE_IMAGE_DIGEST, ENGINE_SNAPSHOT, ENGINE_VERSION, NATIVE_IMAGE, NATIVE_MARCH, SOURCE_COMMIT  # noqa: F401\n\n# Import async execution\n",
         1,
     )
     t = t.replace(

scripts/Dockerfile.build

@@ -211,6 +211,7 @@ COPY setup.py .
 COPY scripts/setup_jars.py ./scripts/
 COPY scripts/extract_version.py ./scripts/
 COPY scripts/write_version.py ./scripts/
+COPY scripts/dev_engine_pin.py scripts/dev_engine_pin.json ./scripts/
 COPY scripts/verify_wheel_platform_tag.py ./scripts/
 COPY pyproject.toml ./
 COPY README.md ./

scripts/dev_engine_pin.json

@@ -0,0 +1,15 @@
+{
+  "engine": "26.12.1-SNAPSHOT",
+  "snapshot": "26.12.1-20261106.183100-26",
+  "image_digest": "sha256:faccacf735b4ec040092c9a3d6f5d0b63b4bd72ba7dffcc1f97b75a38e821f6b",
+  "jars": {
+    "arcadedb-console": "639c1bd077df6d334c2baddee22a956aa737999e38bb37e51c64bce227b9cd42",
+    "arcadedb-engine": "7e5443d0da08be895c0bc7cf6c9ce63dda750389672cca6d2ed41d99dea47a5f",
+    "arcadedb-graphql": "90bb2b2d0c184d6399a92bd61c427ca8d67dacc0abfb91d5fdac518baa09ad10",
+    "arcadedb-integration": "3f1584737f72e4f761374751c47ce5bf3deacaaaaec4a23d66d4ac3badb4a9ac",
+    "arcadedb-mcp": "b35ab1b04919148ef4e667ab95ddbaa286c6cc4d382af698a62dd8fa633f0349",
+    "arcadedb-network": "b23403fbdd585395717796d9943f20a12473cba3df1bf082e7ceb83e6de6a082",
+    "arcadedb-server": "0c1430856817df018135c30c1df9fc730a3a401a378309553f1ba4c5f8bb25ef",
+    "arcadedb-studio": "351a767ede7affd7921b2c4b820a6eba63ef0326c3737ed329c82474f3baecc1"
+  }
+}