JVM wheel: ship a class-data-sharing archive for the bundled JRE (closes #313)

docs/modules/ROOT/pages/installation.adoc

@@ -155,6 +155,11 @@ WARNING: *One JVM configuration per process.* JVM options are locked after the J
 
 Each Python process starts its own JVM, and each JVM sees the whole container limit. With several Python processes in one container, give each process its own `heap_size`.
 
+[[jvm-cds]]
+=== Class-data sharing
+
+The bundled JRE ships the JVM's default class-data-sharing archive, dumped with `-XX:+UseCompactObjectHeaders`, the flag that `start_jvm` sets. The JVM finds the archive by itself and maps the JDK's classes from it instead of loading and verifying them at every start. You do nothing to use it. The JVM ignores the archive, and starts as it did before, when your `jvm_args` change a setting that the archive depends on (`-XX:-UseCompactObjectHeaders`, `-Xshare:off`) or when the heap is larger than 32 GB (the default is half of the RAM, so a machine with more than 64 GB of RAM starts without the archive unless you set `heap_size`). The archive holds the JDK's classes only, not the engine's jars.
+
 [[jvm-aot]]
 === Faster start-up with the AOT cache
 

scripts/Dockerfile.build

@@ -181,6 +181,10 @@ RUN echo "🔍 Analyzing JARs with jdeps..." && \
         exit 1; \
     fi
 
+# The default class-data-sharing archive of the JRE, dumped last with the flags the wheel starts the JVM with (#313)
+COPY scripts/dump_cds_archive.sh /build/dump_cds_archive.sh
+RUN bash /build/dump_cds_archive.sh /build/jre
+
 # Stage 3: Build Python wheel
 
 FROM python:${PYTHON_VERSION}-slim AS python-builder

scripts/build-native.sh

@@ -275,6 +275,11 @@ if [[ "$PLATFORM" == windows/* ]]; then
     echo -e "${CYAN}📊 JRE size after Windows cleanup: ${YELLOW}${JRE_SIZE}${NC}"
 fi
 
+# The default class-data-sharing archive of the JRE, dumped last with the flags the wheel starts the JVM with (#313)
+bash "$SCRIPT_DIR/dump_cds_archive.sh" "$PY_BINDINGS_DIR/temp_jre"
+JRE_SIZE=$(du -sh "$PY_BINDINGS_DIR/temp_jre" | cut -f1)
+echo -e "${CYAN}📊 JRE size with the CDS archive: ${YELLOW}${JRE_SIZE}${NC}"
+
 # Step 3: Copy JRE to package (JARs already filtered in place)
 echo -e "${CYAN}📦 Preparing package...${NC}"
 

scripts/dump_cds_archive.sh

@@ -0,0 +1,46 @@
+#!/usr/bin/env bash
+# SPDX-FileCopyrightText: 2021-present Arcade Data Ltd (info@arcadedata.com) and HumemAI (info@humem.ai)
+# SPDX-License-Identifier: Apache-2.0
+#
+# Dump the default class-data-sharing (CDS) archive of the bundled JRE, so the JVM maps the JDK's classes at start instead of
+# loading and verifying them (#313). A JRE made by jlink has no archive; the JVM then starts without one.
+#
+# Usage: dump_cds_archive.sh JRE_DIR
+#
+# The archive is the JVM's default one (lib/server, or bin/server on Windows): the JVM looks for it by itself at every start and
+# ignores it, without an error, when it was dumped for other settings. So it must be dumped with the settings the wheel starts
+# with: -XX:+UseCompactObjectHeaders, which src/arcadedb/jvm.py sets (an archive dumped without it is ignored by a JVM that has it,
+# and the JVM names the default archive after it). -Xmx1g keeps compressed oops on whatever the RAM of this build host is: the
+# archive is the one a heap below 32 GB uses. A start with a larger heap (jvm.py gives the JVM 50 percent of the RAM) takes no
+# archive, as before. The archive holds the JDK's classes only: the engine's jars are named by absolute path and checked by
+# mtime in an archive, which an installed wheel cannot match.
+#
+# Run it last, on the finished JRE: the archive records the size of lib/modules. It checks its own result with -Xshare:on, which
+# fails the JVM start when the archive cannot be used.
+set -euo pipefail
+
+JRE_DIR="${1:?usage: dump_cds_archive.sh JRE_DIR}"
+JAVA="${JRE_DIR}/bin/java"
+CDS_FLAGS=(-XX:+UseCompactObjectHeaders -Xmx1g)
+
+if [[ ! -x "$JAVA" && ! -x "${JAVA}.exe" ]]; then
+    echo "dump_cds_archive.sh: no java in ${JRE_DIR}/bin" >&2
+    exit 1
+fi
+
+echo "Dumping the default CDS archive of ${JRE_DIR} with: ${CDS_FLAGS[*]}"
+"$JAVA" "${CDS_FLAGS[@]}" -Xshare:dump
+
+ARCHIVES="$(find "$JRE_DIR" -name '*.jsa' -type f)"
+if [[ -z "$ARCHIVES" ]]; then
+    echo "dump_cds_archive.sh: the dump wrote no .jsa file under ${JRE_DIR}" >&2
+    exit 1
+fi
+echo "CDS archive:"
+while IFS= read -r archive; do
+    echo "   ${archive} ($(du -k "$archive" | cut -f1) KiB)"
+done <<< "$ARCHIVES"
+
+# -Xshare:on: no archive, or one for other settings, is an error here instead of a quiet start without it
+"$JAVA" "${CDS_FLAGS[@]}" -Xshare:on -version
+echo "The JVM starts with the archive (-Xshare:on)"

src/arcadedb/jvm.py

@@ -889,7 +889,9 @@ def _build_jvm_args(
     ):
         merged_args.append("-Darcadedb.polyglotEngineEnabled=false")
 
-    # Enable compact object headers (JDK 25+) to reduce heap footprint
+    # Enable compact object headers (JDK 25+) to reduce heap footprint.
+    # The bundled JRE's class-data-sharing archive is dumped with this flag (scripts/dump_cds_archive.sh, #313): the JVM ignores an
+    # archive dumped with another setting of it, so a change here is a change there, and tests/test_jvm_cds.py fails when they differ.
     if not any("UseCompactObjectHeaders" in arg for arg in merged_args):
         merged_args.append("-XX:+UseCompactObjectHeaders")
 

tests/conftest.py

@@ -110,6 +110,7 @@
     "test_server_packaging.py": "JVM wheel contents",
     "test_runtime_cache.py": "JRE runtime cache of the JVM wheel",
     "test_aot.py": "JVM class-data cache",
+    "test_jvm_cds.py": "class-data-sharing archive of the JVM wheel's bundled JRE",
     "test_stage_jars.py": "staging of the JVM wheel's jars",
     "test_extract_version.py": "version of the JVM wheel's engine jar",
     "test_extract_version_classifiers.py": "version of the JVM wheel's engine jar",

tests/test_jvm_cds.py

@@ -0,0 +1,65 @@
+"""The bundled JRE ships a class-data-sharing (CDS) archive that the JVM uses at start (#313).
+
+A JRE made by jlink has no archive, so every start loaded and verified the JDK's classes again. ``scripts/dump_cds_archive.sh`` dumps the
+JVM's default archive as the last step of the JRE build, with ``-XX:+UseCompactObjectHeaders`` (which ``jvm.py`` sets: the JVM ignores an
+archive dumped with another setting, and it does so without an error). These tests run on the installed wheel, on every platform and
+Python of the JVM matrix: the archive is in the wheel, the JVM maps it with the flags ``start_jvm`` builds (``-Xshare:on`` turns an
+unusable archive into a failed start; ``-Xlog:class+load`` shows a JDK class coming from the shared objects file), and an archive does not
+serve a JVM with other settings. The archive is the JVM's own default one, so a missing or mismatched file only means a start without it;
+the JDK AOT cache (``aot.py``, ``test_aot.py``) stays opt-in.
+"""
+
+import subprocess  # nosec B404 - runs the bundled JRE's java with fixed arguments
+import sys
+
+from arcadedb import jvm
+
+
+def _java():
+    name = "java.exe" if sys.platform == "win32" else "java"
+    return str(jvm._extract_runtime_resource("jre") / "bin" / name)
+
+
+def _flags(tmp_path):
+    """The JVM arguments ``start_jvm`` passes, with a heap that keeps compressed oops on whatever the RAM of this host is."""
+    return jvm._build_jvm_args(
+        heap_size="1g", disable_xml_limits=False, jvm_args=None, log_dir=tmp_path
+    )
+
+
+def _run(*args):
+    return subprocess.run(  # nosec B603 - the bundled java, arguments of this file
+        [_java(), *args], capture_output=True, text=True, timeout=300
+    )
+
+
+def test_the_wheel_ships_the_archive():
+    archives = list(jvm._extract_runtime_resource("jre").rglob("*.jsa"))
+    assert archives, "no .jsa file in the bundled JRE"
+    assert all(a.stat().st_size > 1_000_000 for a in archives)
+
+
+def test_the_jvm_maps_the_archive_with_the_flags_the_wheel_starts_with(tmp_path):
+    flags = _flags(tmp_path)
+    assert "-XX:+UseCompactObjectHeaders" in flags
+    # -Xshare:on: no usable archive is a failed start, not a quiet start without it
+    result = _run(*flags, "-Xshare:on", "-version")
+    assert result.returncode == 0, result.stderr
+    result = _run(*flags, "-Xshare:auto", "-Xlog:class+load:stdout", "-version")
+    assert result.returncode == 0, result.stderr
+    assert "java.lang.Object source: shared objects file" in result.stdout
+
+
+def test_the_default_start_does_not_turn_the_archive_off(tmp_path):
+    for flag in _flags(tmp_path):
+        assert not flag.startswith(("-Xshare:", "-XX:-UseSharedSpaces"))
+
+
+def test_an_archive_does_not_serve_a_jvm_with_other_settings(tmp_path):
+    flags = [f for f in _flags(tmp_path) if f != "-XX:+UseCompactObjectHeaders"]
+    flags.append("-XX:-UseCompactObjectHeaders")
+    # the archive was dumped with compact object headers: without them -Xshare:on cannot use it, and a plain start goes on without it
+    assert _run(*flags, "-Xshare:on", "-version").returncode != 0
+    result = _run(*flags, "-Xshare:auto", "-Xlog:class+load:stdout", "-version")
+    assert result.returncode == 0, result.stderr
+    assert "java.lang.Object source: shared objects file" not in result.stdout