diff --git a/dev/release/build-release-comet.sh b/dev/release/build-release-comet.sh index 1cebbbf46a7..450cacdb732 100755 --- a/dev/release/build-release-comet.sh +++ b/dev/release/build-release-comet.sh @@ -29,6 +29,9 @@ This script builds comet native binaries inside a docker image. The image is nam "comet-rm" and will be generated by this script. It then builds the jars and installs them to a temporary local Maven repository. +Every jar is built twice: once with the system allocator, which is the artifact, and once with +the jemalloc allocator, which is installed next to it with the "jemalloc" classifier. + The branch is resolved to a single commit when the script starts, and the docker images, the native binaries and the jars are all built from fresh clones of the repo at that commit. Apart from this script, nothing is taken from the local checkout that @@ -165,36 +168,99 @@ docker run \ $BUILDER_IMAGE_ARM64 "${REPO}" "${COMMIT}" arm64 echo "Building binaries completed" -echo "Copying to java build directories" - -JVM_TARGET_DIR=$BUILD_DIR/spark/target/classes/org/apache/comet -mkdir -p $JVM_TARGET_DIR +echo "Copying native libraries out of the builder containers" -mkdir -p $JVM_TARGET_DIR/linux/amd64 -docker cp \ - comet-amd64-builder-container:"/opt/comet-rm/comet/native/target/release/libcomet.so" \ - $JVM_TARGET_DIR/linux/amd64/ - -mkdir -p $JVM_TARGET_DIR/linux/aarch64 -docker cp \ - comet-arm64-builder-container:"/opt/comet-rm/comet/native/target/release/libcomet.so" \ - $JVM_TARGET_DIR/linux/aarch64/ +# Each container holds two builds of libcomet.so per architecture: "default" uses the system +# allocator and "jemalloc" uses the jemalloc allocator. See comet-rm/build-comet-native-libs.sh. +NATIVE_LIBS_DIR=$(mktemp -d /tmp/comet-native-libs-XXXXX) +for VARIANT in default jemalloc +do + mkdir -p "$NATIVE_LIBS_DIR/$VARIANT/linux/amd64" "$NATIVE_LIBS_DIR/$VARIANT/linux/aarch64" + docker cp \ + comet-amd64-builder-container:"/opt/comet-rm/lib/$VARIANT/libcomet.so" \ + "$NATIVE_LIBS_DIR/$VARIANT/linux/amd64/" + docker cp \ + comet-arm64-builder-container:"/opt/comet-rm/lib/$VARIANT/libcomet.so" \ + "$NATIVE_LIBS_DIR/$VARIANT/linux/aarch64/" +done -# Build final jar -echo "Building uber jar and publishing it locally" +# Build final jars +echo "Building uber jars and publishing them locally" pushd "$BUILD_DIR" LOCAL_REPO=$(mktemp -d /tmp/comet-staging-repo-XXXXX) +VERSION=$(./mvnw -q -DforceStdout help:evaluate -Dexpression=project.version | tail -n 1) +echo "Building version $VERSION" + +# Every jar is built once per allocator. The jar with the system allocator is the artifact and +# the one with the jemalloc allocator is installed next to it with the "jemalloc" classifier, for +# example comet-spark-spark4.1_2.13-1.2.0-jemalloc.jar. Only the artifact has a pom and sources. +# +# Each entry is "|". The spark-4.x profiles pin their own Scala 2.13.x +# patch versions to match the corresponding Spark release, so the scala-2.13 profile is not used +# for them. +JAR_BUILDS=( + "comet-spark-spark3.5_2.12|-P spark-3.5 -P scala-2.12" + "comet-spark-spark3.5_2.13|-P spark-3.5 -P scala-2.13" + "comet-spark-spark4.0_2.13|-P spark-4.0" + "comet-spark-spark4.1_2.13|-P spark-4.1" + "comet-spark-spark4.2_2.13|-P spark-4.2" +) + +# Fail unless the jar holds exactly the native libraries built for the variant. The Maven build +# is incremental, so this catches a jar that kept the libraries of an earlier build. +function check_native_libs() +{ + local JAR=$1 + local VARIANT=$2 + local CHECK_DIR + CHECK_DIR=$(mktemp -d /tmp/comet-jar-check-XXXXX) + for PLATFORM_ARCH in linux/amd64 linux/aarch64 + do + unzip -p "$JAR" "org/apache/comet/$PLATFORM_ARCH/libcomet.so" > "$CHECK_DIR/libcomet.so" + if ! cmp -s "$CHECK_DIR/libcomet.so" "$NATIVE_LIBS_DIR/$VARIANT/$PLATFORM_ARCH/libcomet.so" + then + echo "Error: $JAR does not contain the $VARIANT build of $PLATFORM_ARCH/libcomet.so" + rm -rf "$CHECK_DIR" + exit 1 + fi + done + rm -rf "$CHECK_DIR" +} + +# Install the jars with the system allocator, which are the artifacts +JVM_TARGET_DIR=$BUILD_DIR/spark/target/classes/org/apache/comet +mkdir -p "$JVM_TARGET_DIR" +cp -R "$NATIVE_LIBS_DIR/default/linux" "$JVM_TARGET_DIR/" -./mvnw "-Dmaven.repo.local=${LOCAL_REPO}" -P spark-3.5 -P scala-2.12 -DskipTests install -./mvnw "-Dmaven.repo.local=${LOCAL_REPO}" -P spark-3.5 -P scala-2.13 -DskipTests install -# The spark-4.x profiles pin their own Scala 2.13.x patch versions to match the -# corresponding Spark release, so the scala-2.13 profile is not used here. -./mvnw "-Dmaven.repo.local=${LOCAL_REPO}" -P spark-4.0 -DskipTests install -./mvnw "-Dmaven.repo.local=${LOCAL_REPO}" -P spark-4.1 -DskipTests install -./mvnw "-Dmaven.repo.local=${LOCAL_REPO}" -P spark-4.2 -DskipTests install +for BUILD in "${JAR_BUILDS[@]}" +do + ARTIFACT_ID=${BUILD%%|*} + PROFILES=${BUILD#*|} + ./mvnw "-Dmaven.repo.local=${LOCAL_REPO}" $PROFILES -DskipTests install + check_native_libs "spark/target/$ARTIFACT_ID-$VERSION.jar" default +done + +# Rebuild with the jemalloc libraries. Clean first, so that nothing from the builds above is +# packaged again, then place the libraries where the build expects to find them. These builds only +# package the jars: installing them would replace the jars with the system allocator. +./mvnw "-Dmaven.repo.local=${LOCAL_REPO}" clean +mkdir -p "$JVM_TARGET_DIR" +cp -R "$NATIVE_LIBS_DIR/jemalloc/linux" "$JVM_TARGET_DIR/" + +for BUILD in "${JAR_BUILDS[@]}" +do + ARTIFACT_ID=${BUILD%%|*} + PROFILES=${BUILD#*|} + ./mvnw "-Dmaven.repo.local=${LOCAL_REPO}" $PROFILES -P jemalloc -DskipTests package + JEMALLOC_JAR="spark/target/$ARTIFACT_ID-$VERSION.jar" + check_native_libs "$JEMALLOC_JAR" jemalloc + cp "$JEMALLOC_JAR" \ + "$LOCAL_REPO/org/apache/datafusion/$ARTIFACT_ID/$VERSION/$ARTIFACT_ID-$VERSION-jemalloc.jar" +done echo "Installed to local repo: ${LOCAL_REPO}" echo "Built from commit: ${COMMIT}" +rm -rf "$NATIVE_LIBS_DIR" popd diff --git a/dev/release/comet-rm/build-comet-native-libs.sh b/dev/release/comet-rm/build-comet-native-libs.sh index 1f04333ddc9..34dea504340 100755 --- a/dev/release/comet-rm/build-comet-native-libs.sh +++ b/dev/release/comet-rm/build-comet-native-libs.sh @@ -50,6 +50,28 @@ git clone "$REPO" comet cd comet git checkout "$COMMIT" -# build comet binaries +# Build the native library twice, once with the system allocator and once with the jemalloc +# allocator, and keep each build in its own directory under /opt/comet-rm/lib. The caller copies +# them out. Cargo reuses the compiled dependencies, so the second build only compiles jemalloc and +# the comet crate again. +LIB_DIR=/opt/comet-rm/lib +mkdir -p "$LIB_DIR/default" "$LIB_DIR/jemalloc" make core-${ARCH}-libs +cp native/target/release/libcomet.so "$LIB_DIR/default/" + +# jemalloc takes the page size of the machine it is built on, and fails every allocation on a +# kernel whose pages are larger. Fix it at 4 KiB so that the published library does not depend on the +# machine that built it. Kernels with 64 KiB pages, which some arm64 distributions use, need a +# library built from source with JEMALLOC_SYS_WITH_LG_PAGE=16. The user guide says so. +export JEMALLOC_SYS_WITH_LG_PAGE=12 + +COMET_FEATURES=jemalloc make core-${ARCH}-libs +cp native/target/release/libcomet.so "$LIB_DIR/jemalloc/" + +# The jemalloc library must differ from the default one, otherwise the feature was not applied +if cmp -s "$LIB_DIR/default/libcomet.so" "$LIB_DIR/jemalloc/libcomet.so" +then + echo "Error: the jemalloc build of libcomet.so is identical to the default build" + exit 1 +fi diff --git a/docs/source/about/faq.md b/docs/source/about/faq.md index 9937e1ea277..b1b388eaa06 100644 --- a/docs/source/about/faq.md +++ b/docs/source/about/faq.md @@ -83,6 +83,10 @@ includes AVX2, on amd64, and Arm Neoverse N1 or newer cores on arm64. If Comet c (illegal instruction), the CPU is older than that. Build Comet from source for it, and please [open an issue](https://github.com/apache/datafusion-comet/issues) describing your environment. +The jars with the `jemalloc` classifier are built for 4 KiB memory pages and fail on arm64 kernels that use +64 KiB pages, such as the kernel of RHEL 8. On those machines, use the jar with the system allocator or +[build Comet from source](../user-guide/latest/source.md#building-with-the-jemalloc-allocator). + A jar you build yourself is smaller than the published one because it contains the native library for one platform only. diff --git a/docs/source/contributor-guide/release_process.md b/docs/source/contributor-guide/release_process.md index e0820d2fead..925c90b1ebd 100644 --- a/docs/source/contributor-guide/release_process.md +++ b/docs/source/contributor-guide/release_process.md @@ -386,6 +386,9 @@ This script builds comet native binaries inside a docker image. The image is nam "comet-rm" and will be generated by this script. It then builds the jars and installs them to a temporary local Maven repository. +Every jar is built twice: once with the system allocator, which is the artifact, and once with +the jemalloc allocator, which is installed next to it with the "jemalloc" classifier. + The branch is resolved to a single commit when the script starts, and the docker images, the native binaries and the jars are all built from fresh clones of the repo at that commit. Apart from this script, nothing is taken from the local checkout that @@ -407,7 +410,15 @@ cd dev/release && ./build-release-comet.sh -b 0.13.0-rc1 && cd ../.. #### Build output -The build output is installed to a temporary local maven repository. The build script will print the name of the +The build output is installed to a temporary local maven repository. Every jar is built twice, because the native +library is built with two allocators. The jar with the system allocator, which is glibc malloc on Linux, is the +artifact, for example `comet-spark-spark4.1_2.13-0.13.0.jar`. The jar with the +[jemalloc](https://jemalloc.net/) allocator is installed next to it with the `jemalloc` classifier, for example +`comet-spark-spark4.1_2.13-0.13.0-jemalloc.jar`, and has no pom of its own. The two jars differ only in the native +libraries they contain, and the script checks that each jar holds the libraries built for it. The script builds +each native library for both allocators in the same container, so a build takes longer than it did with one +allocator. The jemalloc libraries are built for 4 KiB memory pages, so they do not load on kernels with larger pages. The +user guide tells those users to build from source. The build script will print the name of the repository location at the end. This location will be required at the time of deploying the artifacts to a staging repository. The script also prints the commit that the artifacts were built from, which should be the commit that the release candidate tag points to. @@ -503,7 +514,8 @@ In the Nexus repository UI (https://repository.apache.org/) locate and verify th staging (https://central.sonatype.org/publish/release/#locate-and-examine-your-staging-repository). The staging repository should hold only the `comet-spark-spark_` artifacts, one for -each Spark and Scala combination that `build-release-comet.sh` builds. The build installs `comet-common` and the +each Spark and Scala combination that `build-release-comet.sh` builds. Each artifact holds the jar with the system +allocator, the pom and the sources jar, plus the jar with the `jemalloc` classifier. The build installs `comet-common` and the parent pom too, but `publish-to-maven.sh` leaves them out of the upload: `comet-common` is shaded into the `comet-spark` jar and the published `comet-spark` pom has no parent, so nothing refers to either of them. diff --git a/docs/source/user-guide/latest/source.md b/docs/source/user-guide/latest/source.md index 49f67b91775..94655f26a81 100644 --- a/docs/source/user-guide/latest/source.md +++ b/docs/source/user-guide/latest/source.md @@ -84,7 +84,8 @@ make release-nogit PROFILES="-Pspark-4.1" The native library allocates its memory with the system allocator, which is glibc malloc on Linux. In a long-running executor, glibc can keep several GiB of freed native memory resident (see [#7108](https://github.com/apache/datafusion-comet/issues/7108)). The published jars use the system -allocator. To build Comet with [jemalloc](https://jemalloc.net/) instead, add the `jemalloc` profile: +allocator, and each release also publishes a jar with the `jemalloc` classifier that uses +[jemalloc](https://jemalloc.net/). To build Comet with jemalloc yourself, add the `jemalloc` profile: ```console make release PROFILES="-Pspark-4.1 -Pjemalloc" @@ -107,6 +108,13 @@ with `spark.executorEnv._RJEM_MALLOC_CONF`, for example `--conf spark.executorEnv._RJEM_MALLOC_CONF=background_thread:true`. jemalloc fails every allocation on a kernel whose memory pages are larger than those of the machine -it was built on. Some arm64 distributions use 64 KiB pages. When the executors may run on one, build -with `JEMALLOC_SYS_WITH_LG_PAGE=16` set in the environment, which builds jemalloc for 64 KiB pages. -That also works on kernels with smaller pages, but can raise memory use there. +it was built on. The jemalloc jars that Comet publishes are built for 4 KiB pages, which is what +x86_64 always uses and what the default arm64 kernels of Ubuntu, Debian, Fedora, RHEL 9 and 10 and +Oracle Linux use. Some arm64 kernels use 64 KiB pages: RHEL 8 and its rebuilds, RHEL 9 and 10 when +the optional `kernel-64k` is installed, and NVIDIA Grace systems running the 64 KiB kernel that NVIDIA +recommends, for example Ubuntu's `linux-nvidia-64k`. + +`getconf PAGESIZE` prints the page size of a machine. On executors with 64 KiB pages, use the jar with +the system allocator, or build Comet from source with `JEMALLOC_SYS_WITH_LG_PAGE=16` set in the +environment, which builds jemalloc for 64 KiB pages. That library works on kernels with 4 KiB and +64 KiB pages, but can raise memory use on the smaller ones.