diff --git a/.github/workflows/release-jvm.yml b/.github/workflows/release-jvm.yml index adeb2a9..9b6bb24 100644 --- a/.github/workflows/release-jvm.yml +++ b/.github/workflows/release-jvm.yml @@ -47,7 +47,7 @@ jobs: outputs: version: ${{ steps.validate.outputs.version }} base-version: ${{ steps.validate.outputs.base-version }} - arcadedb-tag: ${{ steps.validate.outputs.arcadedb-tag }} + arcadedb-tag: ${{ steps.pin.outputs.arcadedb-tag || steps.validate.outputs.arcadedb-tag }} steps: - name: Validate tag id: validate @@ -92,6 +92,22 @@ jobs: TAG: ${{ github.ref_name }} run: python3 -I scripts/check_version.py "$TAG" --git-remote origin --pypi "$DIST_NAME" + # A .devN tag embeds ONE upstream snapshot, the same one in both packages (scripts/dev_engine_pin.json): the image is named by digest + # (the tags `latest` and A.B.C-SNAPSHOT move on every upstream deploy), and the check proves that the image (amd64 and arm64) and the + # Sonatype jars the native build links hold the pinned jars, byte for byte. A final or .postN tag never gets here: it names its release. + - name: Pin the dev engine snapshot + id: pin + if: ${{ contains(github.ref_name, '.dev') }} + env: + TAG: ${{ github.ref_name }} + run: | + python3 -I scripts/dev_engine_pin.py check --tag "$TAG" + env_out="$(python3 -I scripts/dev_engine_pin.py shell-env "$TAG")" + ref="$(printf '%s\n' "$env_out" | sed -n 's|^IMAGE_REF=arcadedata/arcadedb:||p')" + [ -n "$ref" ] || { echo "::error::no image reference in the pin"; exit 1; } + echo "arcadedb-tag=$ref" >> "$GITHUB_OUTPUT" + echo "Engine snapshot pinned: $ref" + # Builds one wheel for each platform (py3-none-, the same file for every Python) and # runs the unit tests on it, on the Pythons scripts/select_matrix.py picks. The called workflow # uploads the wheels as wheel- artifacts. diff --git a/.github/workflows/release-native.yml b/.github/workflows/release-native.yml index 4fee617..0e52fb4 100644 --- a/.github/workflows/release-native.yml +++ b/.github/workflows/release-native.yml @@ -83,6 +83,14 @@ jobs: TAG: ${{ github.ref_name }} run: python3 -I scripts/check_version.py "$TAG" --git-remote origin --pypi "$DIST_NAME" + # A .devN tag links the pinned upstream snapshot (scripts/dev_engine_pin.json), the same one as arcadedb-jvm: the check proves that the + # JVM image and the Sonatype jars hold the pinned jars, byte for byte. test-native.yml checks again after the build. + - name: The dev engine snapshot is the pinned one + if: ${{ contains(github.ref_name, '.dev') }} + env: + TAG: ${{ github.ref_name }} + run: python3 -I scripts/dev_engine_pin.py check --tag "$TAG" + # Builds a wheel per platform, runs the parity suite on every Python and uploads the wheels (native-wheel-) and one marker per # passed leg (native-passed--py). test: diff --git a/.github/workflows/test-jvm.yml b/.github/workflows/test-jvm.yml index 590285a..25fd86f 100644 --- a/.github/workflows/test-jvm.yml +++ b/.github/workflows/test-jvm.yml @@ -306,10 +306,16 @@ jobs: echo "📌 ArcadeDB version: $ARCADEDB_TAG" # The build jobs pull arcadedata/arcadedb: as their base image, # so a pinned tag must name that version exactly (26.10.2, not an alias). - if [ "$ARCADEDB_IMAGE_TAG" != "latest" ] && [ "$ARCADEDB_IMAGE_TAG" != "$ARCADEDB_TAG" ]; then + IMAGE_TAG_NAME="${ARCADEDB_IMAGE_TAG%%@*}" + if [ "$IMAGE_TAG_NAME" != "latest" ] && [ "$IMAGE_TAG_NAME" != "$ARCADEDB_TAG" ]; then echo "❌ arcadedb-tag '$ARCADEDB_IMAGE_TAG' holds engine $ARCADEDB_TAG; pass the exact version tag" exit 1 fi + # A tag with a digest (@sha256:...) is the pinned snapshot of a .devN release (scripts/dev_engine_pin.json): + # the jars just downloaded must be byte for byte the pinned ones, the same ones the native build links. + if [[ "$ARCADEDB_IMAGE_TAG" == *@sha256:* ]]; then + python3 -I scripts/dev_engine_pin.py verify-jars src/arcadedb/jars + fi echo "arcadedb-tag=$ARCADEDB_TAG" >> "$GITHUB_OUTPUT" # Unfiltered on purpose: build-native.sh applies scripts/jar_exclusions.txt diff --git a/.github/workflows/test-native.yml b/.github/workflows/test-native.yml index 1779d9a..c3c9cc2 100644 --- a/.github/workflows/test-native.yml +++ b/.github/workflows/test-native.yml @@ -157,19 +157,31 @@ jobs: base="${base%%.post*}" if [[ "$version" == *.dev* ]]; then final=false - engine="${INPUT_ENGINE:-$DEV_ENGINE}" + pin_engine="" + if [ "${INPUT_RELEASE:-}" = true ]; then + # a .devN RELEASE links the pinned snapshot, the one arcadedb-jvm embeds (scripts/dev_engine_pin.json), and nothing else + pin_env="$(python3 -I scripts/dev_engine_pin.py shell-env "$version")" + pin_engine="$(printf '%s\n' "$pin_env" | sed -n 's/^ENGINE=//p')" + [ -n "$pin_engine" ] || { echo "::error::no engine in the pin" >&2; exit 1; } + [ -z "$INPUT_ENGINE" ] || [ "$INPUT_ENGINE" = "$pin_engine" ] || { echo "::error::A dev release links the pinned engine $pin_engine, not $INPUT_ENGINE" >&2; exit 1; } + python3 -I scripts/dev_engine_pin.py check --tag "$version" + engine="$pin_engine" + else + engine="${INPUT_ENGINE:-$DEV_ENGINE}" + fi else final=true engine="${INPUT_ENGINE:-$base}" # a final or post wheel links exactly the engine release it is named after [ "$engine" = "$base" ] || { echo "::error::Version $version links engine $base, not $engine" >&2; exit 1; } fi - # A SNAPSHOT engine is for a manual devN run only: a final or post version is refused above (it links exactly its release), and a release - # run (release-native.yml passes `release: true`) never builds from a snapshot, so nothing it publishes can depend on one. + # A SNAPSHOT engine is for a devN version only: a final or post version is refused above (it links exactly its release). A release run + # (release-native.yml passes `release: true`) links a snapshot only for a .devN tag, and only the pinned one (scripts/dev_engine_pin.json, + # proven above); any other snapshot, and every snapshot of a final or post release, is refused. tests/test_dev_engine_pin.py runs this step. if [[ "$engine" == *-SNAPSHOT ]]; then [[ "$engine" =~ ^[0-9]+\.[0-9]+\.[0-9]+-SNAPSHOT$ ]] || { echo "::error::Engine '$engine' is not A.B.C or A.B.C-SNAPSHOT" >&2; exit 1; } [ "$final" = false ] || { echo "::error::A SNAPSHOT engine is for a devN version only" >&2; exit 1; } - [ "${INPUT_RELEASE:-}" != true ] || { echo "::error::A release never links a SNAPSHOT engine ($engine)" >&2; exit 1; } + [ "${INPUT_RELEASE:-}" != true ] || [ "$engine" = "${pin_engine:-}" ] || { echo "::error::A release never links a SNAPSHOT engine ($engine) except the pinned one of a .devN tag" >&2; exit 1; } fi { echo "version=$version"; echo "engine=$engine"; echo "final=$final"; } >> "$GITHUB_OUTPUT" echo "wheel version $version, engine $engine, final=$final" @@ -380,6 +392,7 @@ jobs: env: VERSION: ${{ needs.plan.outputs.version }} ENGINE: ${{ needs.plan.outputs.engine }} + RELEASE: ${{ inputs.release }} run: | set -euo pipefail # the Linux container logged /root/.m2/repository in the native-image classpath; on the runner that is $T/m2/repository @@ -390,9 +403,17 @@ jobs: esac mkdir -p "$T/record" # THIRD_PARTY_NOTICES.md names the engine release and its jars, so it cannot match a SNAPSHOT build (timestamped jars, no tag): - # a SNAPSHOT run skips that comparison. Such a wheel is never published (see the plan job). + # a SNAPSHOT run skips that comparison. extra=() - if [[ "$ENGINE" == *-SNAPSHOT ]]; then extra=(--skip-notices-check); fi + if [[ "$ENGINE" == *-SNAPSHOT ]]; then + extra=(--skip-notices-check) + if [ "${RELEASE:-}" = true ]; then + # a .devN release: the Sonatype jars the build resolved must still be the pinned ones (an upstream deploy during the build fails + # here, fail closed), and the wheel records the snapshot and the image digest it embeds + uv run --no-project --python 3.12 python -I scripts/dev_engine_pin.py check --tag "$VERSION" + extra+=(--engine-pin) + fi + fi uv run --no-project --python 3.12 python native/wheel/build_wheel.py --lib-dir "$T/lib" --version "$VERSION" --engine "$ENGINE" \ --graalvm-home "$GRAALVM_HOME_P5" --m2-repo "$m2" \ --notices-out "$T/record/THIRD_PARTY_NOTICES.generated.md" --modules-out "$T/record/jdk-modules.txt" --out "$T/wheel" ${extra[@]+"${extra[@]}"} diff --git a/.github/workflows/test-python-examples.yml b/.github/workflows/test-python-examples.yml index ed797f6..73c6db6 100644 --- a/.github/workflows/test-python-examples.yml +++ b/.github/workflows/test-python-examples.yml @@ -185,10 +185,16 @@ jobs: echo "📌 ArcadeDB version: $ARCADEDB_TAG" # The build jobs pull arcadedata/arcadedb: as their base image, # so a pinned tag must name that version exactly (26.10.2, not an alias). - if [ "$ARCADEDB_IMAGE_TAG" != "latest" ] && [ "$ARCADEDB_IMAGE_TAG" != "$ARCADEDB_TAG" ]; then + IMAGE_TAG_NAME="${ARCADEDB_IMAGE_TAG%%@*}" + if [ "$IMAGE_TAG_NAME" != "latest" ] && [ "$IMAGE_TAG_NAME" != "$ARCADEDB_TAG" ]; then echo "❌ arcadedb-tag '$ARCADEDB_IMAGE_TAG' holds engine $ARCADEDB_TAG; pass the exact version tag" exit 1 fi + # A tag with a digest (@sha256:...) is the pinned snapshot of a .devN release (scripts/dev_engine_pin.json): + # the jars just downloaded must be byte for byte the pinned ones, the same ones the native build links. + if [[ "$ARCADEDB_IMAGE_TAG" == *@sha256:* ]]; then + python3 -I scripts/dev_engine_pin.py verify-jars src/arcadedb/jars + fi echo "arcadedb-tag=$ARCADEDB_TAG" >> "$GITHUB_OUTPUT" # Unfiltered on purpose: build-native.sh applies scripts/jar_exclusions.txt diff --git a/README.md b/README.md index 987e100..86b2ef3 100644 --- a/README.md +++ b/README.md @@ -38,6 +38,8 @@ uv add arcadedb-jvm # JVM, for heavy long-running loads > The first release of this layout is 26.11.1, and both packages have a final release on PyPI. A plain `uv add arcadedb` installs the final, and a later pre-release (`A.B.C.devN`) needs `uv add --prerelease allow arcadedb`. The same holds for `arcadedb-jvm`. +> +> A `.devN` version (for example `26.12.1.dev0`, when one is published) is a development pre-release. It embeds a snapshot of the upstream engine, the same snapshot in both packages, pinned by image digest and jar checksums in `scripts/dev_engine_pin.json`, and the wheel states it (`arcadedb.ENGINE_SNAPSHOT`, `arcadedb.ENGINE_IMAGE_DIGEST`). Install it with `uv add --prerelease allow arcadedb==26.12.1.dev0` (or `arcadedb-jvm==26.12.1.dev0`). > Old names: `arcadedb-jvm` was formerly `arcadedb-embedded`, and `arcadedb` was formerly `arcadedb-embedded-native` (archived on PyPI). The last `arcadedb-embedded` release, 26.11.1, is a deprecation shim: it installs `arcadedb`, the native build, and warns once at import. It does not install `arcadedb-jvm`. diff --git a/docs/modules/ROOT/pages/development/release.adoc b/docs/modules/ROOT/pages/development/release.adoc index 02efd36..36139ca 100644 --- a/docs/modules/ROOT/pages/development/release.adoc +++ b/docs/modules/ROOT/pages/development/release.adoc @@ -12,7 +12,7 @@ The tag is the bare version, with no `v`. The tag, the wheel version and the PyP | Version | Meaning | `A.B.C.dev0`, `A.B.C.dev1`, ... -| A preview of engine release `A.B.C`, built from the upstream snapshot jars. `devN` starts at 0. +| A development pre-release of engine release `A.B.C`, built from ONE pinned upstream snapshot (<<_the_dev_engine_snapshot,below>>). `devN` starts at 0. | `A.B.C` | The release. The wheels ship the `A.B.C` engine jar. @@ -31,6 +31,17 @@ The rule has five parts: The base is the version of the engine. The workflow compares the tag with the version of the engine jar in the wheels, and it refuses a `devN` tag for a release jar and a release tag for a snapshot jar. +[[_the_dev_engine_snapshot]] +== The engine snapshot of a dev release + +Upstream publishes the snapshot engine by the day: the Docker image tags `latest` and `A.B.C-SNAPSHOT` move, and so does the Sonatype snapshot repository that the native build resolves. A `.devN` release therefore pins one snapshot for both packages in `scripts/dev_engine_pin.json`: the image by index digest, the timestamped snapshot version, and the sha256 of every `arcadedb-*.jar` in the image. + +* `scripts/dev_engine_pin.py check --tag ` proves that the image (amd64 and arm64) and the Sonatype jars hold exactly the pinned jars. Both release workflows run it before anything is built, `release-native.yml` runs it again after the native build, and the JVM jobs download the image by digest and run `dev_engine_pin.py verify-jars` on the jars. A difference stops the release (exit 1); a source it cannot read stops it too (exit 2). +* Only `A.B.C.devN` of the pinned base gets a snapshot. A final or `.postN` release refuses it, and a snapshot other than the pinned one is refused for a dev release (`tests/test_dev_engine_pin.py` runs the real step code). +* Each wheel states what it embeds: `arcadedb.ENGINE_SNAPSHOT` (the timestamped version) and `arcadedb.ENGINE_IMAGE_DIGEST`. +* To re-pin, run `uv run --no-project python -I scripts/dev_engine_pin.py capture ` and replace the file in a pull request, before the tag. +* The native notices comparison is skipped for a snapshot engine (`THIRD_PARTY_NOTICES.md` names a release), so a dev wheel is not checked against it. + == How the rule is enforced `scripts/check_version.py` checks the rule. It reads the published versions from the Git tags of the repository and from PyPI, and it never changes anything: diff --git a/native/wheel/build_wheel.py b/native/wheel/build_wheel.py index d86bbb8..7c44c24 100644 --- a/native/wheel/build_wheel.py +++ b/native/wheel/build_wheel.py @@ -145,6 +145,11 @@ def main(): default="", help="also write the generated notices here (CI keeps it in the build record)", ) + ap.add_argument( + "--engine-pin", + action="store_true", + help="--engine is the pinned snapshot of scripts/dev_engine_pin.json (a .devN release): record its timestamped version and image digest", + ) ap.add_argument( "--skip-notices-check", action="store_true", @@ -157,6 +162,17 @@ def main(): ) ap.add_argument("--plat", default="", help="override the computed platform tag") a = ap.parse_args() + pin = None + if a.engine_pin: + sys.path.insert(0, str(REPO / "scripts")) + import dev_engine_pin + + pin = dev_engine_pin.load_pin() + problem = dev_engine_pin.tag_problem(a.version, pin) + if problem: + sys.exit(problem) + if a.engine != pin["engine"]: + sys.exit(f"--engine {a.engine} is not the pinned engine {pin['engine']}") lib_dir, out = Path(a.lib_dir), Path(a.out).resolve() libname = LIB_NAMES.get(sys.platform) or sys.exit( f"unsupported platform {sys.platform}" @@ -246,6 +262,8 @@ def sub(m): (dst / "_build_info.py").write_text( f'"""Written by native-wheel/build_wheel.py."""\n' f'ENGINE_VERSION = "{a.engine}" # the ArcadeDB engine release compiled into {libname}\n' + f'ENGINE_SNAPSHOT = {pin["snapshot"] if pin else None!r} # the exact upstream snapshot (scripts/dev_engine_pin.json) of a .devN wheel\n' + f'ENGINE_IMAGE_DIGEST = {pin["image_digest"] if pin else None!r} # the arcadedata/arcadedb image the same snapshot was pinned by\n' f'NATIVE_IMAGE = "GraalVM CE 25.4.4.1.1, -O3, -march={a.march}"\n' f'NATIVE_MARCH = "{a.march}" # lib.py checks the CPU only for an x86-64-v3 image\n' f'SOURCE_COMMIT = "{commit}" # ArcadeData/arcadedb-python commit of the bindings and the Java glue\n', @@ -255,7 +273,7 @@ def sub(m): t = ip.read_text(encoding="utf-8") t = t.replace( "# Import async execution\n", - "from ._build_info import ENGINE_VERSION, NATIVE_IMAGE, NATIVE_MARCH, SOURCE_COMMIT # noqa: F401\n\n# Import async execution\n", + "from ._build_info import ENGINE_IMAGE_DIGEST, ENGINE_SNAPSHOT, ENGINE_VERSION, NATIVE_IMAGE, NATIVE_MARCH, SOURCE_COMMIT # noqa: F401\n\n# Import async execution\n", 1, ) t = t.replace( diff --git a/scripts/Dockerfile.build b/scripts/Dockerfile.build index f9ca268..de26178 100644 --- a/scripts/Dockerfile.build +++ b/scripts/Dockerfile.build @@ -211,6 +211,7 @@ COPY setup.py . COPY scripts/setup_jars.py ./scripts/ COPY scripts/extract_version.py ./scripts/ COPY scripts/write_version.py ./scripts/ +COPY scripts/dev_engine_pin.py scripts/dev_engine_pin.json ./scripts/ COPY scripts/verify_wheel_platform_tag.py ./scripts/ COPY pyproject.toml ./ COPY README.md ./ diff --git a/scripts/dev_engine_pin.json b/scripts/dev_engine_pin.json new file mode 100644 index 0000000..7b01631 --- /dev/null +++ b/scripts/dev_engine_pin.json @@ -0,0 +1,15 @@ +{ + "engine": "26.12.1-SNAPSHOT", + "snapshot": "26.12.1-20261106.183100-26", + "image_digest": "sha256:faccacf735b4ec040092c9a3d6f5d0b63b4bd72ba7dffcc1f97b75a38e821f6b", + "jars": { + "arcadedb-console": "639c1bd077df6d334c2baddee22a956aa737999e38bb37e51c64bce227b9cd42", + "arcadedb-engine": "7e5443d0da08be895c0bc7cf6c9ce63dda750389672cca6d2ed41d99dea47a5f", + "arcadedb-graphql": "90bb2b2d0c184d6399a92bd61c427ca8d67dacc0abfb91d5fdac518baa09ad10", + "arcadedb-integration": "3f1584737f72e4f761374751c47ce5bf3deacaaaaec4a23d66d4ac3badb4a9ac", + "arcadedb-mcp": "b35ab1b04919148ef4e667ab95ddbaa286c6cc4d382af698a62dd8fa633f0349", + "arcadedb-network": "b23403fbdd585395717796d9943f20a12473cba3df1bf082e7ceb83e6de6a082", + "arcadedb-server": "0c1430856817df018135c30c1df9fc730a3a401a378309553f1ba4c5f8bb25ef", + "arcadedb-studio": "351a767ede7affd7921b2c4b820a6eba63ef0326c3737ed329c82474f3baecc1" + } +} diff --git a/scripts/dev_engine_pin.py b/scripts/dev_engine_pin.py new file mode 100644 index 0000000..6ebcf62 --- /dev/null +++ b/scripts/dev_engine_pin.py @@ -0,0 +1,268 @@ +#!/usr/bin/env python3 +"""The engine snapshot a `.devN` release embeds, pinned once for BOTH packages. + +A `.devN` tag is built from the upstream SNAPSHOT engine. The JVM wheels take the jars from the Docker image +`arcadedata/arcadedb`, the native wheels resolve `arcadedb-*` from the Sonatype snapshot repository; both channels follow upstream +and move by the day. `scripts/dev_engine_pin.json` names one snapshot: the image by index digest and the sha256 of every +`arcadedb-*.jar` in it. This script proves that BOTH channels still hold exactly those jars, byte for byte, and fails closed +(exit 1 on a mismatch, exit 2 on a source it cannot read). + + dev_engine_pin.py check [--tag TAG] [--pin FILE] # image (amd64 and arm64) and Sonatype jars against the pin + dev_engine_pin.py capture IMAGE_DIGEST # print a pin for an image digest (the Sonatype jars must agree) + dev_engine_pin.py verify-jars DIR # the jars in DIR (a download of the image) are exactly the pinned ones + dev_engine_pin.py shell-env TAG # print JVM image reference / snapshot facts a workflow needs (dev tags only) + +`check --tag` applies the pin ONLY to a `.devN` tag: a final or `.postN` tag exits 1, because such a release links exactly +its release engine and never a snapshot. Tests: tests/test_dev_engine_pin.py. Output is ASCII only. +""" + +from __future__ import annotations + +import argparse +import hashlib +import json +import re +import sys +import tarfile +import urllib.error +import urllib.request +import xml.etree.ElementTree as ET # nosec B405 - parses the maven-metadata.xml of a fixed https repository +from pathlib import Path + +PIN_FILE = Path(__file__).resolve().parent / "dev_engine_pin.json" +REGISTRY = "https://registry-1.docker.io/v2/arcadedata/arcadedb" +AUTH = "https://auth.docker.io/token?service=registry.docker.io&scope=repository:arcadedata/arcadedb:pull" +SNAPSHOTS = "https://central.sonatype.com/repository/maven-snapshots/com/arcadedb" +INDEX_TYPES = "application/vnd.oci.image.index.v1+json,application/vnd.docker.distribution.manifest.list.v2+json" +MANIFEST_TYPES = "application/vnd.oci.image.manifest.v1+json,application/vnd.docker.distribution.manifest.v2+json" +LIB = re.compile(r"^(?:\./)?home/arcadedb/lib/(arcadedb-[^/]+)-(\d+\.\d+\.\d+)-SNAPSHOT\.jar$") +DEV_TAG = re.compile(r"^\d+\.\d+\.\d+\.dev\d+$") +ARCHES = ("amd64", "arm64") +TIMEOUT = 60 + + +class Unreadable(Exception): + """A source (registry, Sonatype) could not be read: exit 2, never a pass.""" + + +class _NoAuthRedirect(urllib.request.HTTPRedirectHandler): + """The registry redirects a blob to a CDN that refuses a request carrying the registry token.""" + + def redirect_request(self, req, fp, code, msg, headers, newurl): + new = super().redirect_request(req, fp, code, msg, headers, newurl) + if new is not None: + new.headers.pop("Authorization", None) + return new + + +_OPENER = urllib.request.build_opener(_NoAuthRedirect) + + +def _open(url: str, headers: dict[str, str] | None = None): + req = urllib.request.Request(url, headers=headers or {}) # noqa: S310 - fixed https hosts + try: + return _OPENER.open(req, timeout=TIMEOUT) # nosec B310 + except (urllib.error.URLError, TimeoutError) as e: + raise Unreadable(f"{url}: {e}") from e + + +def _token() -> str: + with _open(AUTH) as r: + return json.load(r)["token"] + + +def _registry_json(ref: str, accept: str, token: str) -> dict: + with _open(f"{REGISTRY}/manifests/{ref}", {"Authorization": f"Bearer {token}", "Accept": accept}) as r: + return json.load(r) + + +def image_jars(index_digest: str, arch: str, token: str) -> dict[str, str]: + """{module: sha256} of the arcadedb-*-SNAPSHOT.jar files in /home/arcadedb/lib of one platform of the image.""" + index = _registry_json(index_digest, INDEX_TYPES, token) + entries = [m for m in index.get("manifests", []) if m.get("platform", {}).get("architecture") == arch] + if len(entries) != 1: + raise Unreadable(f"image {index_digest}: {len(entries)} manifests for {arch}") + manifest = _registry_json(entries[0]["digest"], MANIFEST_TYPES, token) + found: dict[str, str] = {} + for layer in reversed(manifest["layers"]): # a later layer wins over an earlier one + with _open(f"{REGISTRY}/blobs/{layer['digest']}", {"Authorization": f"Bearer {token}"}) as blob: + with tarfile.open(fileobj=blob, mode="r|gz") as tar: + for member in tar: + m = LIB.match(member.name) + if m and m.group(1) not in found and member.isfile(): + h = hashlib.sha256() + f = tar.extractfile(member) + for chunk in iter(lambda: f.read(1 << 20), b""): + h.update(chunk) + found[m.group(1)] = h.hexdigest() + if "arcadedb-engine" not in found: + raise Unreadable(f"image {index_digest} ({arch}): no arcadedb-engine jar in /home/arcadedb/lib") + return found + + +def snapshot_jar(module: str, version: str) -> tuple[str, str | None]: + """(timestamped version, sha256) of the newest snapshot jar of a module; sha256 None when the repository has no such module.""" + try: + with _open(f"{SNAPSHOTS}/{module}/{version}/maven-metadata.xml") as r: + root = ET.fromstring(r.read()) # nosec B314 + except Unreadable as e: + if "404" in str(e): + return "", None + raise + stamp = None + for sv in root.iter("snapshotVersion"): + if sv.findtext("extension") == "jar" and sv.findtext("classifier") is None: + stamp = sv.findtext("value") + if not stamp: + raise Unreadable(f"{module}: no jar in the snapshot metadata") + h = hashlib.sha256() + with _open(f"{SNAPSHOTS}/{module}/{version}/{module}-{stamp}.jar") as r: + for chunk in iter(lambda: r.read(1 << 20), b""): + h.update(chunk) + return stamp, h.hexdigest() + + +def jar_hashes(directory: Path) -> dict[str, str]: + """{module: sha256} of the arcadedb-*-SNAPSHOT.jar files in a directory.""" + found = {} + for jar in sorted(Path(directory).glob("arcadedb-*-SNAPSHOT.jar")): + m = re.fullmatch(r"(arcadedb-.+)-\d+\.\d+\.\d+-SNAPSHOT\.jar", jar.name) + if m: + found[m.group(1)] = hashlib.sha256(jar.read_bytes()).hexdigest() + return found + + +def jars_are_pinned(hashes: dict[str, str], pin: dict, strict: bool) -> bool: + """The engine jar is there and every jar present is the pinned one; strict: the set of jars is the pinned set.""" + if hashes.get("arcadedb-engine") != pin["jars"]["arcadedb-engine"]: + return False + if any(pin["jars"].get(mod) != sha for mod, sha in hashes.items()): + return False + return not strict or set(hashes) == set(pin["jars"]) + + +def pinned_snapshot(directory: Path, pin_path: Path = PIN_FILE) -> dict | None: + """The pin when the jars of DIRECTORY are exactly the pinned snapshot (some pinned jars may be left out), else None.""" + if not Path(pin_path).is_file(): + return None + pin = load_pin(Path(pin_path)) + return pin if jars_are_pinned(jar_hashes(directory), pin, strict=False) else None + + +def verify_jars(directory: Path) -> int: + pin = load_pin() + if not jars_are_pinned(jar_hashes(directory), pin, strict=True): + print(f"FAIL the jars in {directory} are not the pinned snapshot {pin['snapshot']} (image {pin['image_digest']})") + return 1 + print(f"PASS the jars in {directory} are the pinned snapshot {pin['snapshot']}, same sha256") + return 0 + + +def load_pin(path: Path = PIN_FILE) -> dict: + pin = json.loads(path.read_text(encoding="utf-8")) + for key in ("engine", "snapshot", "image_digest", "jars"): + if key not in pin: + raise ValueError(f"{path}: missing {key}") + if not re.fullmatch(r"\d+\.\d+\.\d+-SNAPSHOT", pin["engine"]): + raise ValueError(f"{path}: engine {pin['engine']!r} is not A.B.C-SNAPSHOT") + if not re.fullmatch(r"sha256:[0-9a-f]{64}", pin["image_digest"]): + raise ValueError(f"{path}: image_digest is not a sha256 digest") + return pin + + +def tag_problem(tag: str, pin: dict) -> str | None: + """Why a snapshot pin may not be applied to TAG, or None. Only `A.B.C.devN` of the pinned base may link a snapshot.""" + if not DEV_TAG.fullmatch(tag): + return f"tag {tag} is not A.B.C.devN: a final or .postN release links exactly its release engine, never a snapshot" + if tag.split(".dev")[0] != pin["engine"].removesuffix("-SNAPSHOT"): + return f"tag {tag} has another base than the pinned engine {pin['engine']}" + return None + + +def compare(pin: dict, image: dict[str, dict[str, str]], maven: dict[str, str | None]) -> list[str]: + """Every difference between the pin and the two channels; an empty list is a pass.""" + bad = [] + for arch, jars in image.items(): + if jars != pin["jars"]: + for mod in sorted(set(jars) | set(pin["jars"])): + if jars.get(mod) != pin["jars"].get(mod): + bad.append(f"image {arch}: {mod} is {jars.get(mod)}, the pin says {pin['jars'].get(mod)}") + for mod, sha in maven.items(): + if sha is not None and sha != pin["jars"].get(mod): + bad.append(f"Sonatype: {mod} is {sha}, the pin says {pin['jars'].get(mod)}") + for mod in ("arcadedb-engine", "arcadedb-server"): + if maven.get(mod) is None: + bad.append(f"Sonatype has no {mod} {pin['engine']}") + return bad + + +def check(tag: str | None, pin_path: Path) -> int: + pin = load_pin(pin_path) + if tag: + problem = tag_problem(tag, pin) + if problem: + print(f"FAIL {problem}") + return 1 + token = _token() + image = {arch: image_jars(pin["image_digest"], arch, token) for arch in ARCHES} + maven = {mod: snapshot_jar(mod, pin["engine"])[1] for mod in pin["jars"]} + bad = compare(pin, image, maven) + for line in bad: + print(f"FAIL {line}") + if bad: + print(f"FAIL the JVM image and the native engine are not the pinned snapshot {pin['snapshot']}; re-pin (scripts/dev_engine_pin.py capture)") + return 1 + print(f"PASS image {pin['image_digest']} (amd64, arm64) and Sonatype hold the pinned snapshot {pin['snapshot']}: {len(pin['jars'])} jars, same sha256") + return 0 + + +def capture(digest: str) -> int: + token = _token() + jars = image_jars(digest, "amd64", token) + stamps = {mod: snapshot_jar(mod, "26.12.1-SNAPSHOT") for mod in jars} + engine_stamp = stamps["arcadedb-engine"][0] + base = re.fullmatch(r"(\d+\.\d+\.\d+)-\d{8}\.\d{6}-\d+", engine_stamp) + if not base: + raise Unreadable(f"unexpected snapshot version {engine_stamp!r}") + print(json.dumps({"engine": f"{base.group(1)}-SNAPSHOT", "snapshot": engine_stamp, "image_digest": digest, "jars": dict(sorted(jars.items()))}, indent=2)) + return 0 + + +def shell_env(tag: str) -> int: + pin = load_pin() + problem = tag_problem(tag, pin) + if problem: + print(f"::error::{problem}", file=sys.stderr) + return 1 + print(f"ENGINE={pin['engine']}") + print(f"SNAPSHOT={pin['snapshot']}") + print(f"IMAGE_REF=arcadedata/arcadedb:{pin['engine']}@{pin['image_digest']}") + print(f"IMAGE_DIGEST={pin['image_digest']}") + return 0 + + +def main(argv: list[str] | None = None) -> int: + ap = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) + sub = ap.add_subparsers(dest="cmd", required=True) + c = sub.add_parser("check") + c.add_argument("--tag") + c.add_argument("--pin", type=Path, default=PIN_FILE) + sub.add_parser("capture").add_argument("digest") + sub.add_parser("shell-env").add_argument("tag") + sub.add_parser("verify-jars").add_argument("directory", type=Path) + a = ap.parse_args(argv) + try: + if a.cmd == "check": + return check(a.tag, a.pin) + if a.cmd == "capture": + return capture(a.digest) + if a.cmd == "verify-jars": + return verify_jars(a.directory) + return shell_env(a.tag) + except (Unreadable, ValueError, OSError, tarfile.TarError, KeyError) as e: + print(f"FAIL cannot read a source: {e}", file=sys.stderr) + return 2 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/scripts/write_version.py b/scripts/write_version.py index 3a40629..0a03f9e 100644 --- a/scripts/write_version.py +++ b/scripts/write_version.py @@ -13,13 +13,17 @@ sys.stderr = io.TextIOWrapper(sys.stderr.buffer, encoding="utf-8", errors="replace") -def write_version_file(ver, out_path): +def write_version_file(ver, out_path, pin=None): """Write version to _version.py""" version_file_content = f'''"""Version information for arcadedb package. Generated automatically during build from the bundled ArcadeDB engine jar. """ __version__ = "{ver}" + +# The exact upstream snapshot a .devN wheel embeds (scripts/dev_engine_pin.json); None for a release or an unpinned build. +ENGINE_SNAPSHOT = {pin["snapshot"] if pin else None!r} +ENGINE_IMAGE_DIGEST = {pin["image_digest"] if pin else None!r} ''' with open(out_path, "w", encoding="utf-8") as f: @@ -43,7 +47,9 @@ def write_version_file(ver, out_path): # Write to src/arcadedb/_version.py output_path = bindings_dir / "src/arcadedb/_version.py" - write_version_file(version, output_path) + from dev_engine_pin import pinned_snapshot + + write_version_file(version, output_path, pinned_snapshot(jars_dir)) except (ValueError, FileNotFoundError, OSError) as e: print(f"Error: {e}", file=sys.stderr) diff --git a/src/arcadedb/__init__.py b/src/arcadedb/__init__.py index 7451794..a3fbc0d 100644 --- a/src/arcadedb/__init__.py +++ b/src/arcadedb/__init__.py @@ -96,9 +96,17 @@ to_python_array, ) +try: + from ._version import ENGINE_IMAGE_DIGEST, ENGINE_SNAPSHOT +except ImportError: # a source checkout without a generated _version.py, or an older build + ENGINE_SNAPSHOT = None + ENGINE_IMAGE_DIGEST = None + __all__ = [ "__version__", "__backend__", + "ENGINE_SNAPSHOT", + "ENGINE_IMAGE_DIGEST", # Exceptions "ArcadeDBError", # Core classes diff --git a/tests/conftest.py b/tests/conftest.py index 9a309f2..129dbbf 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -38,6 +38,7 @@ } if sys.platform == "win32": collect_ignore.extend(WINDOWS_NOT_APPLICABLE) + collect_ignore.append("test_dev_engine_pin.py") # runs the bash code of a workflow step if BACKEND != "native": # Java -> Python upcalls (Python functions in SQL) exist on the native build only collect_ignore.append("test_native_udf.py") diff --git a/tests/test_dev_engine_pin.py b/tests/test_dev_engine_pin.py new file mode 100644 index 0000000..0bfb360 --- /dev/null +++ b/tests/test_dev_engine_pin.py @@ -0,0 +1,173 @@ +"""The engine snapshot of a .devN release (scripts/dev_engine_pin.py, scripts/dev_engine_pin.json). + +The rule: only A.B.C.devN of the pinned base may link a snapshot engine, and then only the pinned one. A final or .postN +release refuses a snapshot. These tests run the real `run:` code of the plan step of test-native.yml (bash) for every +case that needs no network, and call the pure functions of the script for the rest. +This file is not collected on Windows (tests/conftest.py): the step code is bash. +""" + +import copy +import hashlib +import json +import subprocess # nosec B404 - runs bash on a workflow step with a fixed environment +import sys +from pathlib import Path + +import pytest + +ROOT = Path(__file__).resolve().parent.parent +sys.path.insert(0, str(ROOT / "scripts")) +import dev_engine_pin as pin_mod # noqa: E402 + +PIN = pin_mod.load_pin() + + +def _plan_step() -> str: + lines = (ROOT / ".github/workflows/test-native.yml").read_text(encoding="utf-8").splitlines() + start = next(i for i, ln in enumerate(lines) if ln.strip() == "- name: Resolve the version and the engine") + run = next(i for i in range(start, len(lines)) if lines[i].strip() == "run: |") + indent = len(lines[run + 1]) - len(lines[run + 1].lstrip()) + body = [] + for ln in lines[run + 1 :]: + if ln.strip() and len(ln) - len(ln.lstrip()) < indent: + break + body.append(ln[indent:]) + return "\n".join(body) + "\n" + + +def _run_plan(tmp_path, version, engine="", release="true"): + out = tmp_path / "out" + out.write_text("") + env = { + "PATH": "/usr/bin:/bin:/usr/local/bin", + "INPUT_VERSION": version, + "INPUT_ENGINE": engine, + "INPUT_RELEASE": release, + "DEV_ENGINE": "26.11.1", + "DEFAULT_VERSION": "26.12.1.dev0", + "GITHUB_OUTPUT": str(out), + } + r = subprocess.run( # nosec B603 B607 + ["bash", "-c", _plan_step()], env=env, cwd=ROOT, capture_output=True, text=True, timeout=60 + ) + return r, out.read_text() + + +def test_pin_file_is_well_formed(): + assert PIN["engine"] == "26.12.1-SNAPSHOT" + assert PIN["snapshot"].startswith("26.12.1-") and PIN["image_digest"].startswith("sha256:") + assert "arcadedb-engine" in PIN["jars"] and "arcadedb-server" in PIN["jars"] + + +@pytest.mark.parametrize("tag", ["26.12.1", "26.12.1.post1", "26.11.1.dev0", "26.12.1rc1", "v26.12.1.dev0"]) +def test_only_a_dev_tag_of_the_pinned_base_may_link_the_snapshot(tag): + assert pin_mod.tag_problem(tag, PIN) is not None + + +@pytest.mark.parametrize("tag", ["26.12.1.dev0", "26.12.1.dev7"]) +def test_a_dev_tag_of_the_pinned_base_may(tag): + assert pin_mod.tag_problem(tag, PIN) is None + + +@pytest.mark.parametrize("tag", ["26.12.1", "26.12.1.post1"]) +def test_check_refuses_a_final_or_post_tag_before_any_network(tag, capsys): + assert pin_mod.main(["check", "--tag", tag]) == 1 + assert "never a snapshot" in capsys.readouterr().out + + +@pytest.mark.parametrize("version", ["26.12.1", "26.12.1.post1"]) +def test_release_plan_refuses_a_snapshot_for_a_final_or_post_version(tmp_path, version): + r, _ = _run_plan(tmp_path, version, engine="26.12.1-SNAPSHOT") + assert r.returncode != 0 + assert "links engine 26.12.1, not 26.12.1-SNAPSHOT" in r.stderr + + +def test_release_plan_of_a_final_version_links_its_release_engine(tmp_path): + r, out = _run_plan(tmp_path, "26.12.1") + assert r.returncode == 0, r.stderr + assert "engine=26.12.1\n" in out and "SNAPSHOT" not in out + + +def test_release_plan_refuses_another_snapshot_for_a_dev_version(tmp_path): + r, _ = _run_plan(tmp_path, "26.12.1.dev0", engine="26.12.2-SNAPSHOT") + assert r.returncode != 0 + assert "links the pinned engine 26.12.1-SNAPSHOT, not 26.12.2-SNAPSHOT" in r.stderr + + +def test_release_plan_refuses_a_dev_version_of_another_base(tmp_path): + r, _ = _run_plan(tmp_path, "26.13.1.dev0") + assert r.returncode != 0 + assert "another base than the pinned engine" in r.stderr + + +def test_a_manual_dev_run_may_still_link_any_snapshot(tmp_path): + r, out = _run_plan(tmp_path, "26.12.1.dev0", engine="26.12.2-SNAPSHOT", release="false") + assert r.returncode == 0, r.stderr + assert "engine=26.12.2-SNAPSHOT\n" in out + + +def test_a_manual_final_run_still_refuses_a_snapshot(tmp_path): + r, _ = _run_plan(tmp_path, "26.12.1", engine="26.12.1-SNAPSHOT", release="false") + assert r.returncode != 0 + + +def _hashes(**changes): + h = copy.deepcopy(PIN["jars"]) + h.update(changes) + return h + + +def _maven(**changes): + m = {k: PIN["jars"][k] for k in ("arcadedb-engine", "arcadedb-server")} + m.update(changes) + return m + + +def test_compare_passes_when_both_channels_hold_the_pin(): + assert pin_mod.compare(PIN, {"amd64": _hashes(), "arm64": _hashes()}, _maven()) == [] + + +def test_compare_names_an_image_that_moved(): + bad = pin_mod.compare(PIN, {"amd64": _hashes(), "arm64": _hashes(**{"arcadedb-engine": "0" * 64})}, _maven()) + assert any("image arm64: arcadedb-engine" in b for b in bad) + + +def test_compare_names_a_sonatype_snapshot_that_moved(): + bad = pin_mod.compare(PIN, {"amd64": _hashes(), "arm64": _hashes()}, _maven(**{"arcadedb-engine": "1" * 64})) + assert any(b.startswith("Sonatype: arcadedb-engine") for b in bad) + + +def test_compare_needs_engine_and_server_on_sonatype(): + maven = {"arcadedb-engine": PIN["jars"]["arcadedb-engine"]} + bad = pin_mod.compare(PIN, {"amd64": _hashes(), "arm64": _hashes()}, maven) + assert any("no arcadedb-server" in b for b in bad) + + +def _pin_for(directory, jars): + path = directory / "pin.json" + path.write_text(json.dumps(dict(PIN, jars={m: hashlib.sha256(c).hexdigest() for m, c in jars.items()}))) + return path + + +def _write_jars(directory, jars): + directory.mkdir() + for mod, content in jars.items(): + (directory / f"{mod}-26.12.1-SNAPSHOT.jar").write_bytes(content) + + +def test_pinned_snapshot_is_recorded_only_for_the_pinned_jars(tmp_path): + jars = {"arcadedb-engine": b"engine", "arcadedb-server": b"server"} + d = tmp_path / "jars" + _write_jars(d, jars) + pin_path = _pin_for(tmp_path, jars) + assert pin_mod.pinned_snapshot(d, pin_path)["snapshot"] == PIN["snapshot"] + (d / "arcadedb-engine-26.12.1-SNAPSHOT.jar").write_bytes(b"another engine build") + assert pin_mod.pinned_snapshot(d, pin_path) is None + + +def test_strict_needs_the_whole_pinned_set(tmp_path): + d = tmp_path / "jars" + _write_jars(d, {"arcadedb-engine": b"engine"}) + pin = json.loads(_pin_for(tmp_path, {"arcadedb-engine": b"engine", "arcadedb-server": b"server"}).read_text()) + assert pin_mod.jars_are_pinned(pin_mod.jar_hashes(d), pin, strict=False) + assert not pin_mod.jars_are_pinned(pin_mod.jar_hashes(d), pin, strict=True)