From 0d3612a3c4583619ebab4295e704f3576c6694cb Mon Sep 17 00:00:00 2001 From: Andy Grove Date: Sun, 15 Nov 2026 08:00:07 -0700 Subject: [PATCH 1/5] feat: build a jemalloc jar alongside each released jar build-release-comet.sh now builds the native library twice per architecture, with the system allocator and with jemalloc, and packages every jar twice. The jar with the system allocator stays the artifact. The jemalloc jar is installed next to it with the jemalloc classifier, for example comet-spark-spark4.1_2.13-1.2.0-jemalloc.jar, so publish-to-maven.sh uploads it without changes. The jemalloc jars are packaged after a clean build rather than installed, because installing would replace the jars that use the system allocator. The script checks that every jar holds the native libraries built for its variant. The arm64 jemalloc library is built for 64 KiB pages so that it loads on every arm64 kernel. --- dev/release/build-release-comet.sh | 110 ++++++++++++++---- .../comet-rm/build-comet-native-libs.sh | 27 ++++- .../contributor-guide/release_process.md | 16 ++- docs/source/user-guide/latest/source.md | 3 +- 4 files changed, 130 insertions(+), 26 deletions(-) diff --git a/dev/release/build-release-comet.sh b/dev/release/build-release-comet.sh index 1cebbbf46a7..450cacdb732 100755 --- a/dev/release/build-release-comet.sh +++ b/dev/release/build-release-comet.sh @@ -29,6 +29,9 @@ This script builds comet native binaries inside a docker image. The image is nam "comet-rm" and will be generated by this script. It then builds the jars and installs them to a temporary local Maven repository. +Every jar is built twice: once with the system allocator, which is the artifact, and once with +the jemalloc allocator, which is installed next to it with the "jemalloc" classifier. + The branch is resolved to a single commit when the script starts, and the docker images, the native binaries and the jars are all built from fresh clones of the repo at that commit. Apart from this script, nothing is taken from the local checkout that @@ -165,36 +168,99 @@ docker run \ $BUILDER_IMAGE_ARM64 "${REPO}" "${COMMIT}" arm64 echo "Building binaries completed" -echo "Copying to java build directories" - -JVM_TARGET_DIR=$BUILD_DIR/spark/target/classes/org/apache/comet -mkdir -p $JVM_TARGET_DIR +echo "Copying native libraries out of the builder containers" -mkdir -p $JVM_TARGET_DIR/linux/amd64 -docker cp \ - comet-amd64-builder-container:"/opt/comet-rm/comet/native/target/release/libcomet.so" \ - $JVM_TARGET_DIR/linux/amd64/ - -mkdir -p $JVM_TARGET_DIR/linux/aarch64 -docker cp \ - comet-arm64-builder-container:"/opt/comet-rm/comet/native/target/release/libcomet.so" \ - $JVM_TARGET_DIR/linux/aarch64/ +# Each container holds two builds of libcomet.so per architecture: "default" uses the system +# allocator and "jemalloc" uses the jemalloc allocator. See comet-rm/build-comet-native-libs.sh. +NATIVE_LIBS_DIR=$(mktemp -d /tmp/comet-native-libs-XXXXX) +for VARIANT in default jemalloc +do + mkdir -p "$NATIVE_LIBS_DIR/$VARIANT/linux/amd64" "$NATIVE_LIBS_DIR/$VARIANT/linux/aarch64" + docker cp \ + comet-amd64-builder-container:"/opt/comet-rm/lib/$VARIANT/libcomet.so" \ + "$NATIVE_LIBS_DIR/$VARIANT/linux/amd64/" + docker cp \ + comet-arm64-builder-container:"/opt/comet-rm/lib/$VARIANT/libcomet.so" \ + "$NATIVE_LIBS_DIR/$VARIANT/linux/aarch64/" +done -# Build final jar -echo "Building uber jar and publishing it locally" +# Build final jars +echo "Building uber jars and publishing them locally" pushd "$BUILD_DIR" LOCAL_REPO=$(mktemp -d /tmp/comet-staging-repo-XXXXX) +VERSION=$(./mvnw -q -DforceStdout help:evaluate -Dexpression=project.version | tail -n 1) +echo "Building version $VERSION" + +# Every jar is built once per allocator. The jar with the system allocator is the artifact and +# the one with the jemalloc allocator is installed next to it with the "jemalloc" classifier, for +# example comet-spark-spark4.1_2.13-1.2.0-jemalloc.jar. Only the artifact has a pom and sources. +# +# Each entry is "|". The spark-4.x profiles pin their own Scala 2.13.x +# patch versions to match the corresponding Spark release, so the scala-2.13 profile is not used +# for them. +JAR_BUILDS=( + "comet-spark-spark3.5_2.12|-P spark-3.5 -P scala-2.12" + "comet-spark-spark3.5_2.13|-P spark-3.5 -P scala-2.13" + "comet-spark-spark4.0_2.13|-P spark-4.0" + "comet-spark-spark4.1_2.13|-P spark-4.1" + "comet-spark-spark4.2_2.13|-P spark-4.2" +) + +# Fail unless the jar holds exactly the native libraries built for the variant. The Maven build +# is incremental, so this catches a jar that kept the libraries of an earlier build. +function check_native_libs() +{ + local JAR=$1 + local VARIANT=$2 + local CHECK_DIR + CHECK_DIR=$(mktemp -d /tmp/comet-jar-check-XXXXX) + for PLATFORM_ARCH in linux/amd64 linux/aarch64 + do + unzip -p "$JAR" "org/apache/comet/$PLATFORM_ARCH/libcomet.so" > "$CHECK_DIR/libcomet.so" + if ! cmp -s "$CHECK_DIR/libcomet.so" "$NATIVE_LIBS_DIR/$VARIANT/$PLATFORM_ARCH/libcomet.so" + then + echo "Error: $JAR does not contain the $VARIANT build of $PLATFORM_ARCH/libcomet.so" + rm -rf "$CHECK_DIR" + exit 1 + fi + done + rm -rf "$CHECK_DIR" +} + +# Install the jars with the system allocator, which are the artifacts +JVM_TARGET_DIR=$BUILD_DIR/spark/target/classes/org/apache/comet +mkdir -p "$JVM_TARGET_DIR" +cp -R "$NATIVE_LIBS_DIR/default/linux" "$JVM_TARGET_DIR/" -./mvnw "-Dmaven.repo.local=${LOCAL_REPO}" -P spark-3.5 -P scala-2.12 -DskipTests install -./mvnw "-Dmaven.repo.local=${LOCAL_REPO}" -P spark-3.5 -P scala-2.13 -DskipTests install -# The spark-4.x profiles pin their own Scala 2.13.x patch versions to match the -# corresponding Spark release, so the scala-2.13 profile is not used here. -./mvnw "-Dmaven.repo.local=${LOCAL_REPO}" -P spark-4.0 -DskipTests install -./mvnw "-Dmaven.repo.local=${LOCAL_REPO}" -P spark-4.1 -DskipTests install -./mvnw "-Dmaven.repo.local=${LOCAL_REPO}" -P spark-4.2 -DskipTests install +for BUILD in "${JAR_BUILDS[@]}" +do + ARTIFACT_ID=${BUILD%%|*} + PROFILES=${BUILD#*|} + ./mvnw "-Dmaven.repo.local=${LOCAL_REPO}" $PROFILES -DskipTests install + check_native_libs "spark/target/$ARTIFACT_ID-$VERSION.jar" default +done + +# Rebuild with the jemalloc libraries. Clean first, so that nothing from the builds above is +# packaged again, then place the libraries where the build expects to find them. These builds only +# package the jars: installing them would replace the jars with the system allocator. +./mvnw "-Dmaven.repo.local=${LOCAL_REPO}" clean +mkdir -p "$JVM_TARGET_DIR" +cp -R "$NATIVE_LIBS_DIR/jemalloc/linux" "$JVM_TARGET_DIR/" + +for BUILD in "${JAR_BUILDS[@]}" +do + ARTIFACT_ID=${BUILD%%|*} + PROFILES=${BUILD#*|} + ./mvnw "-Dmaven.repo.local=${LOCAL_REPO}" $PROFILES -P jemalloc -DskipTests package + JEMALLOC_JAR="spark/target/$ARTIFACT_ID-$VERSION.jar" + check_native_libs "$JEMALLOC_JAR" jemalloc + cp "$JEMALLOC_JAR" \ + "$LOCAL_REPO/org/apache/datafusion/$ARTIFACT_ID/$VERSION/$ARTIFACT_ID-$VERSION-jemalloc.jar" +done echo "Installed to local repo: ${LOCAL_REPO}" echo "Built from commit: ${COMMIT}" +rm -rf "$NATIVE_LIBS_DIR" popd diff --git a/dev/release/comet-rm/build-comet-native-libs.sh b/dev/release/comet-rm/build-comet-native-libs.sh index 1f04333ddc9..4d3fcea0bb8 100755 --- a/dev/release/comet-rm/build-comet-native-libs.sh +++ b/dev/release/comet-rm/build-comet-native-libs.sh @@ -50,6 +50,31 @@ git clone "$REPO" comet cd comet git checkout "$COMMIT" -# build comet binaries +# Build the native library twice, once with the system allocator and once with the jemalloc +# allocator, and keep each build in its own directory under /opt/comet-rm/lib. The caller copies +# them out. Cargo reuses the compiled dependencies, so the second build only compiles jemalloc and +# the comet crate again. +LIB_DIR=/opt/comet-rm/lib +mkdir -p "$LIB_DIR/default" "$LIB_DIR/jemalloc" make core-${ARCH}-libs +cp native/target/release/libcomet.so "$LIB_DIR/default/" + +# A jemalloc built on a kernel with small memory pages fails every allocation on a kernel with +# larger pages, and some arm64 distributions use 64 KiB pages. The published arm64 library must +# load on all of them, so build jemalloc for 64 KiB pages. That also works on kernels with smaller +# pages, but can raise memory use there. amd64 kernels always use 4 KiB pages. +if [ "$ARCH" = "arm64" ] +then + export JEMALLOC_SYS_WITH_LG_PAGE=16 +fi + +COMET_FEATURES=jemalloc make core-${ARCH}-libs +cp native/target/release/libcomet.so "$LIB_DIR/jemalloc/" + +# The jemalloc library must differ from the default one, otherwise the feature was not applied +if cmp -s "$LIB_DIR/default/libcomet.so" "$LIB_DIR/jemalloc/libcomet.so" +then + echo "Error: the jemalloc build of libcomet.so is identical to the default build" + exit 1 +fi diff --git a/docs/source/contributor-guide/release_process.md b/docs/source/contributor-guide/release_process.md index e0820d2fead..5c463059265 100644 --- a/docs/source/contributor-guide/release_process.md +++ b/docs/source/contributor-guide/release_process.md @@ -386,6 +386,9 @@ This script builds comet native binaries inside a docker image. The image is nam "comet-rm" and will be generated by this script. It then builds the jars and installs them to a temporary local Maven repository. +Every jar is built twice: once with the system allocator, which is the artifact, and once with +the jemalloc allocator, which is installed next to it with the "jemalloc" classifier. + The branch is resolved to a single commit when the script starts, and the docker images, the native binaries and the jars are all built from fresh clones of the repo at that commit. Apart from this script, nothing is taken from the local checkout that @@ -407,7 +410,15 @@ cd dev/release && ./build-release-comet.sh -b 0.13.0-rc1 && cd ../.. #### Build output -The build output is installed to a temporary local maven repository. The build script will print the name of the +The build output is installed to a temporary local maven repository. Every jar is built twice, because the native +library is built with two allocators. The jar with the system allocator, which is glibc malloc on Linux, is the +artifact, for example `comet-spark-spark4.1_2.13-0.13.0.jar`. The jar with the +[jemalloc](https://jemalloc.net/) allocator is installed next to it with the `jemalloc` classifier, for example +`comet-spark-spark4.1_2.13-0.13.0-jemalloc.jar`, and has no pom of its own. The two jars differ only in the native +libraries they contain, and the script checks that each jar holds the libraries built for it. The script builds +each native library for both allocators in the same container, so a build takes longer than it did with one +allocator. The arm64 jemalloc library is built for 64 KiB memory pages, so that it loads on the arm64 kernels that +use them. The build script will print the name of the repository location at the end. This location will be required at the time of deploying the artifacts to a staging repository. The script also prints the commit that the artifacts were built from, which should be the commit that the release candidate tag points to. @@ -503,7 +514,8 @@ In the Nexus repository UI (https://repository.apache.org/) locate and verify th staging (https://central.sonatype.org/publish/release/#locate-and-examine-your-staging-repository). The staging repository should hold only the `comet-spark-spark_` artifacts, one for -each Spark and Scala combination that `build-release-comet.sh` builds. The build installs `comet-common` and the +each Spark and Scala combination that `build-release-comet.sh` builds. Each artifact holds the jar with the system +allocator, the pom and the sources jar, plus the jar with the `jemalloc` classifier. The build installs `comet-common` and the parent pom too, but `publish-to-maven.sh` leaves them out of the upload: `comet-common` is shaded into the `comet-spark` jar and the published `comet-spark` pom has no parent, so nothing refers to either of them. diff --git a/docs/source/user-guide/latest/source.md b/docs/source/user-guide/latest/source.md index 49f67b91775..3e15e542f23 100644 --- a/docs/source/user-guide/latest/source.md +++ b/docs/source/user-guide/latest/source.md @@ -84,7 +84,8 @@ make release-nogit PROFILES="-Pspark-4.1" The native library allocates its memory with the system allocator, which is glibc malloc on Linux. In a long-running executor, glibc can keep several GiB of freed native memory resident (see [#7108](https://github.com/apache/datafusion-comet/issues/7108)). The published jars use the system -allocator. To build Comet with [jemalloc](https://jemalloc.net/) instead, add the `jemalloc` profile: +allocator, and each release also publishes a jar with the `jemalloc` classifier that uses +[jemalloc](https://jemalloc.net/). To build Comet with jemalloc yourself, add the `jemalloc` profile: ```console make release PROFILES="-Pspark-4.1 -Pjemalloc" From b7a8ae1689a70b3a69daf902fd7c21416f2b4508 Mon Sep 17 00:00:00 2001 From: Andy Grove Date: Sun, 15 Nov 2026 08:01:40 -0700 Subject: [PATCH 2/5] docs: build the jemalloc jars for 4 KiB pages and document 64 KiB kernels --- dev/release/comet-rm/build-comet-native-libs.sh | 13 +++++-------- docs/source/about/faq.md | 4 ++++ docs/source/contributor-guide/release_process.md | 4 ++-- docs/source/user-guide/latest/source.md | 10 +++++++--- 4 files changed, 18 insertions(+), 13 deletions(-) diff --git a/dev/release/comet-rm/build-comet-native-libs.sh b/dev/release/comet-rm/build-comet-native-libs.sh index 4d3fcea0bb8..34dea504340 100755 --- a/dev/release/comet-rm/build-comet-native-libs.sh +++ b/dev/release/comet-rm/build-comet-native-libs.sh @@ -60,14 +60,11 @@ mkdir -p "$LIB_DIR/default" "$LIB_DIR/jemalloc" make core-${ARCH}-libs cp native/target/release/libcomet.so "$LIB_DIR/default/" -# A jemalloc built on a kernel with small memory pages fails every allocation on a kernel with -# larger pages, and some arm64 distributions use 64 KiB pages. The published arm64 library must -# load on all of them, so build jemalloc for 64 KiB pages. That also works on kernels with smaller -# pages, but can raise memory use there. amd64 kernels always use 4 KiB pages. -if [ "$ARCH" = "arm64" ] -then - export JEMALLOC_SYS_WITH_LG_PAGE=16 -fi +# jemalloc takes the page size of the machine it is built on, and fails every allocation on a +# kernel whose pages are larger. Fix it at 4 KiB so that the published library does not depend on the +# machine that built it. Kernels with 64 KiB pages, which some arm64 distributions use, need a +# library built from source with JEMALLOC_SYS_WITH_LG_PAGE=16. The user guide says so. +export JEMALLOC_SYS_WITH_LG_PAGE=12 COMET_FEATURES=jemalloc make core-${ARCH}-libs cp native/target/release/libcomet.so "$LIB_DIR/jemalloc/" diff --git a/docs/source/about/faq.md b/docs/source/about/faq.md index 9937e1ea277..f6593214b9f 100644 --- a/docs/source/about/faq.md +++ b/docs/source/about/faq.md @@ -83,6 +83,10 @@ includes AVX2, on amd64, and Arm Neoverse N1 or newer cores on arm64. If Comet c (illegal instruction), the CPU is older than that. Build Comet from source for it, and please [open an issue](https://github.com/apache/datafusion-comet/issues) describing your environment. +The jars with the `jemalloc` classifier are built for 4 KiB memory pages and fail on arm64 kernels that use +64 KiB pages. On those machines, use the jar with the system allocator or +[build Comet from source](../user-guide/latest/source.md#building-with-the-jemalloc-allocator). + A jar you build yourself is smaller than the published one because it contains the native library for one platform only. diff --git a/docs/source/contributor-guide/release_process.md b/docs/source/contributor-guide/release_process.md index 5c463059265..925c90b1ebd 100644 --- a/docs/source/contributor-guide/release_process.md +++ b/docs/source/contributor-guide/release_process.md @@ -417,8 +417,8 @@ artifact, for example `comet-spark-spark4.1_2.13-0.13.0.jar`. The jar with the `comet-spark-spark4.1_2.13-0.13.0-jemalloc.jar`, and has no pom of its own. The two jars differ only in the native libraries they contain, and the script checks that each jar holds the libraries built for it. The script builds each native library for both allocators in the same container, so a build takes longer than it did with one -allocator. The arm64 jemalloc library is built for 64 KiB memory pages, so that it loads on the arm64 kernels that -use them. The build script will print the name of the +allocator. The jemalloc libraries are built for 4 KiB memory pages, so they do not load on kernels with larger pages. The +user guide tells those users to build from source. The build script will print the name of the repository location at the end. This location will be required at the time of deploying the artifacts to a staging repository. The script also prints the commit that the artifacts were built from, which should be the commit that the release candidate tag points to. diff --git a/docs/source/user-guide/latest/source.md b/docs/source/user-guide/latest/source.md index 3e15e542f23..1e7522f9a6c 100644 --- a/docs/source/user-guide/latest/source.md +++ b/docs/source/user-guide/latest/source.md @@ -108,6 +108,10 @@ with `spark.executorEnv._RJEM_MALLOC_CONF`, for example `--conf spark.executorEnv._RJEM_MALLOC_CONF=background_thread:true`. jemalloc fails every allocation on a kernel whose memory pages are larger than those of the machine -it was built on. Some arm64 distributions use 64 KiB pages. When the executors may run on one, build -with `JEMALLOC_SYS_WITH_LG_PAGE=16` set in the environment, which builds jemalloc for 64 KiB pages. -That also works on kernels with smaller pages, but can raise memory use there. +it was built on. The jemalloc jars that Comet publishes are built for 4 KiB pages, which is what most +Linux kernels use. Some arm64 kernels use 64 KiB pages, for example the default kernels of Red Hat +Enterprise Linux and its rebuilds on arm64, and some kernels for NVIDIA Grace. On those executors, use +the jar with the system allocator, or build Comet from source with `JEMALLOC_SYS_WITH_LG_PAGE=16` set in +the environment, which builds jemalloc for 64 KiB pages. `getconf PAGESIZE` prints the page size of a +machine. A build for 64 KiB pages also works on kernels with smaller pages, but can raise memory use +there. From 71ac53011d03c568c15f6cfc321a2911ab6a93c0 Mon Sep 17 00:00:00 2001 From: Andy Grove Date: Sun, 15 Nov 2026 08:14:22 -0700 Subject: [PATCH 3/5] docs: list the arm64 kernels with pages larger than 4 KiB --- docs/source/about/faq.md | 4 ++-- docs/source/user-guide/latest/source.md | 20 +++++++++++++------- 2 files changed, 15 insertions(+), 9 deletions(-) diff --git a/docs/source/about/faq.md b/docs/source/about/faq.md index f6593214b9f..278c7199d08 100644 --- a/docs/source/about/faq.md +++ b/docs/source/about/faq.md @@ -83,8 +83,8 @@ includes AVX2, on amd64, and Arm Neoverse N1 or newer cores on arm64. If Comet c (illegal instruction), the CPU is older than that. Build Comet from source for it, and please [open an issue](https://github.com/apache/datafusion-comet/issues) describing your environment. -The jars with the `jemalloc` classifier are built for 4 KiB memory pages and fail on arm64 kernels that use -64 KiB pages. On those machines, use the jar with the system allocator or +The jars with the `jemalloc` classifier are built for 4 KiB memory pages and fail on arm64 kernels with +larger pages, such as the 64 KiB kernel of RHEL 8. On those machines, use the jar with the system allocator or [build Comet from source](../user-guide/latest/source.md#building-with-the-jemalloc-allocator). A jar you build yourself is smaller than the published one because it contains the native library diff --git a/docs/source/user-guide/latest/source.md b/docs/source/user-guide/latest/source.md index 1e7522f9a6c..2bab03cc0d5 100644 --- a/docs/source/user-guide/latest/source.md +++ b/docs/source/user-guide/latest/source.md @@ -108,10 +108,16 @@ with `spark.executorEnv._RJEM_MALLOC_CONF`, for example `--conf spark.executorEnv._RJEM_MALLOC_CONF=background_thread:true`. jemalloc fails every allocation on a kernel whose memory pages are larger than those of the machine -it was built on. The jemalloc jars that Comet publishes are built for 4 KiB pages, which is what most -Linux kernels use. Some arm64 kernels use 64 KiB pages, for example the default kernels of Red Hat -Enterprise Linux and its rebuilds on arm64, and some kernels for NVIDIA Grace. On those executors, use -the jar with the system allocator, or build Comet from source with `JEMALLOC_SYS_WITH_LG_PAGE=16` set in -the environment, which builds jemalloc for 64 KiB pages. `getconf PAGESIZE` prints the page size of a -machine. A build for 64 KiB pages also works on kernels with smaller pages, but can raise memory use -there. +it was built on. The jemalloc jars that Comet publishes are built for 4 KiB pages, which is what +x86_64 always uses and what the default arm64 kernels of Ubuntu, Debian, Fedora, RHEL 9 and 10 and +Oracle Linux use. Some arm64 kernels use larger pages: + +- 64 KiB: RHEL 8 and its rebuilds, RHEL 9 and 10 when the optional `kernel-64k` is installed, and + NVIDIA Grace systems running the 64 KiB kernel that NVIDIA recommends, for example Ubuntu's + `linux-nvidia-64k`. +- 16 KiB: Raspberry Pi 5 with the stock Raspberry Pi OS kernel and Asahi Linux on Apple Silicon. + +`getconf PAGESIZE` prints the page size of a machine. On executors with larger pages, use the jar with +the system allocator, or build Comet from source with `JEMALLOC_SYS_WITH_LG_PAGE=16` set in the +environment, which builds jemalloc for 64 KiB pages. That library works on kernels with 4, 16 and +64 KiB pages, but can raise memory use on the smaller ones. From 1be1934b8229c9a069ff77a742589018c016ef51 Mon Sep 17 00:00:00 2001 From: Andy Grove Date: Sun, 15 Nov 2026 08:14:33 -0700 Subject: [PATCH 4/5] chore: say larger pages in the native build comment --- dev/release/comet-rm/build-comet-native-libs.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dev/release/comet-rm/build-comet-native-libs.sh b/dev/release/comet-rm/build-comet-native-libs.sh index 34dea504340..b5efaa5700f 100755 --- a/dev/release/comet-rm/build-comet-native-libs.sh +++ b/dev/release/comet-rm/build-comet-native-libs.sh @@ -62,7 +62,7 @@ cp native/target/release/libcomet.so "$LIB_DIR/default/" # jemalloc takes the page size of the machine it is built on, and fails every allocation on a # kernel whose pages are larger. Fix it at 4 KiB so that the published library does not depend on the -# machine that built it. Kernels with 64 KiB pages, which some arm64 distributions use, need a +# machine that built it. Kernels with larger pages, which some arm64 distributions use, need a # library built from source with JEMALLOC_SYS_WITH_LG_PAGE=16. The user guide says so. export JEMALLOC_SYS_WITH_LG_PAGE=12 From dc427e4b7ff9eac2660c1b18bbd06849506f955e Mon Sep 17 00:00:00 2001 From: Andy Grove Date: Sun, 15 Nov 2026 08:14:55 -0700 Subject: [PATCH 5/5] docs: drop the 16 KiB kernels from the jemalloc page size note --- dev/release/comet-rm/build-comet-native-libs.sh | 2 +- docs/source/about/faq.md | 4 ++-- docs/source/user-guide/latest/source.md | 13 +++++-------- 3 files changed, 8 insertions(+), 11 deletions(-) diff --git a/dev/release/comet-rm/build-comet-native-libs.sh b/dev/release/comet-rm/build-comet-native-libs.sh index b5efaa5700f..34dea504340 100755 --- a/dev/release/comet-rm/build-comet-native-libs.sh +++ b/dev/release/comet-rm/build-comet-native-libs.sh @@ -62,7 +62,7 @@ cp native/target/release/libcomet.so "$LIB_DIR/default/" # jemalloc takes the page size of the machine it is built on, and fails every allocation on a # kernel whose pages are larger. Fix it at 4 KiB so that the published library does not depend on the -# machine that built it. Kernels with larger pages, which some arm64 distributions use, need a +# machine that built it. Kernels with 64 KiB pages, which some arm64 distributions use, need a # library built from source with JEMALLOC_SYS_WITH_LG_PAGE=16. The user guide says so. export JEMALLOC_SYS_WITH_LG_PAGE=12 diff --git a/docs/source/about/faq.md b/docs/source/about/faq.md index 278c7199d08..b1b388eaa06 100644 --- a/docs/source/about/faq.md +++ b/docs/source/about/faq.md @@ -83,8 +83,8 @@ includes AVX2, on amd64, and Arm Neoverse N1 or newer cores on arm64. If Comet c (illegal instruction), the CPU is older than that. Build Comet from source for it, and please [open an issue](https://github.com/apache/datafusion-comet/issues) describing your environment. -The jars with the `jemalloc` classifier are built for 4 KiB memory pages and fail on arm64 kernels with -larger pages, such as the 64 KiB kernel of RHEL 8. On those machines, use the jar with the system allocator or +The jars with the `jemalloc` classifier are built for 4 KiB memory pages and fail on arm64 kernels that use +64 KiB pages, such as the kernel of RHEL 8. On those machines, use the jar with the system allocator or [build Comet from source](../user-guide/latest/source.md#building-with-the-jemalloc-allocator). A jar you build yourself is smaller than the published one because it contains the native library diff --git a/docs/source/user-guide/latest/source.md b/docs/source/user-guide/latest/source.md index 2bab03cc0d5..94655f26a81 100644 --- a/docs/source/user-guide/latest/source.md +++ b/docs/source/user-guide/latest/source.md @@ -110,14 +110,11 @@ with `spark.executorEnv._RJEM_MALLOC_CONF`, for example jemalloc fails every allocation on a kernel whose memory pages are larger than those of the machine it was built on. The jemalloc jars that Comet publishes are built for 4 KiB pages, which is what x86_64 always uses and what the default arm64 kernels of Ubuntu, Debian, Fedora, RHEL 9 and 10 and -Oracle Linux use. Some arm64 kernels use larger pages: +Oracle Linux use. Some arm64 kernels use 64 KiB pages: RHEL 8 and its rebuilds, RHEL 9 and 10 when +the optional `kernel-64k` is installed, and NVIDIA Grace systems running the 64 KiB kernel that NVIDIA +recommends, for example Ubuntu's `linux-nvidia-64k`. -- 64 KiB: RHEL 8 and its rebuilds, RHEL 9 and 10 when the optional `kernel-64k` is installed, and - NVIDIA Grace systems running the 64 KiB kernel that NVIDIA recommends, for example Ubuntu's - `linux-nvidia-64k`. -- 16 KiB: Raspberry Pi 5 with the stock Raspberry Pi OS kernel and Asahi Linux on Apple Silicon. - -`getconf PAGESIZE` prints the page size of a machine. On executors with larger pages, use the jar with +`getconf PAGESIZE` prints the page size of a machine. On executors with 64 KiB pages, use the jar with the system allocator, or build Comet from source with `JEMALLOC_SYS_WITH_LG_PAGE=16` set in the -environment, which builds jemalloc for 64 KiB pages. That library works on kernels with 4, 16 and +environment, which builds jemalloc for 64 KiB pages. That library works on kernels with 4 KiB and 64 KiB pages, but can raise memory use on the smaller ones.