From fcdd30e081a6f8771ab251c25083da2c510167d1 Mon Sep 17 00:00:00 2001 From: David Li Date: Mon, 21 Sep 2026 10:29:52 +0900 Subject: [PATCH 1/2] MINOR: Fix JNI builds Assisted-by: GPT-6 Astra --- .github/workflows/rc.yml | 20 ++++++-------------- ci/scripts/jni_macos_build.sh | 11 +++++++++++ dataset/CMakeLists.txt | 6 ++++++ 3 files changed, 23 insertions(+), 14 deletions(-) diff --git a/.github/workflows/rc.yml b/.github/workflows/rc.yml index 34623b4018..8b0e64d3af 100644 --- a/.github/workflows/rc.yml +++ b/.github/workflows/rc.yml @@ -199,18 +199,6 @@ jobs: run: pip install -e arrow/dev/archery[all] - name: Install dependencies run: | - # We want to use llvm@14 to avoid shared z3 - # dependency. llvm@14 doesn't depend on z3 and llvm depends - # on z3. And Homebrew's z3 provides only shared library. It - # doesn't provides static z3 because z3's CMake doesn't accept - # building both shared and static libraries at once. - # See also: Z3_BUILD_LIBZ3_SHARED in - # https://github.com/Z3Prover/z3/blob/master/README-CMake.md - # - # If llvm is installed, Apache Arrow C++ uses llvm rather than - # llvm@14 because llvm is newer than llvm@14. - brew uninstall llvm || : - # We can remove this when we drop support for # macos-15-intel. because macos-14 or later with arm64 uses /opt/homebrew/ # not /usr/local/. @@ -228,6 +216,9 @@ jobs: done brew install --overwrite python3 + # Use a version supported by Gandiva with prebuilt macOS bottles. + brew install --force-bottle llvm@18 + if [ "$(uname -m)" = "arm64" ]; then # pkg-config formula is deprecated but it's still installed # in GitHub Actions runner now. We can remove this once @@ -251,8 +242,9 @@ jobs: # # We don't use Homebrew's Node.js too. Homebrew doesn't provide # a bottle for it on x86_64 macOS for the same reason. - HOMEBREW_BUNDLE_BREW_SKIP="aws-sdk-cpp grpc node" \ - brew bundle --file=arrow/cpp/Brewfile + # LLVM is installed explicitly above instead of using the latest version. + HOMEBREW_BUNDLE_BREW_SKIP="aws-sdk-cpp grpc node llvm" \ + brew bundle --verbose --file=arrow/cpp/Brewfile # We want to link aws-sdk-cpp statically but Homebrew's # aws-sdk-cpp provides only shared library. If we have # Homebrew's aws-sdk-cpp, our build mix Homebrew's diff --git a/ci/scripts/jni_macos_build.sh b/ci/scripts/jni_macos_build.sh index c0040cb030..d9f26add63 100755 --- a/ci/scripts/jni_macos_build.sh +++ b/ci/scripts/jni_macos_build.sh @@ -69,12 +69,22 @@ export ARROW_PARQUET=ON export AWS_EC2_METADATA_DISABLED=TRUE +llvm_root="$(brew --prefix llvm@18)" +"${llvm_root}/bin/llvm-config" --version +llvm_cmake_args=( + "-DLLVM_ROOT=${llvm_root}" + "-DLLVM_DIR=${llvm_root}/lib/cmake/llvm" + "-DCLANG_EXECUTABLE=${llvm_root}/bin/clang" + "-DLLVM_LINK_EXECUTABLE=${llvm_root}/bin/llvm-link" +) + cmake \ -S "${arrow_dir}/cpp" \ -B "${build_dir}/cpp" \ --preset=ninja-release-jni-macos \ -Dabsl_SOURCE=BUNDLED \ -Dsimdjson_SOURCE=BUNDLED \ + "${llvm_cmake_args[@]}" \ -DCMAKE_INSTALL_PREFIX="${install_dir}" cmake --build "${build_dir}/cpp" --target install github_actions_group_end @@ -88,6 +98,7 @@ fi JAVA_JNI_CMAKE_ARGS="-DProtobuf_ROOT=${build_dir}/cpp/_deps/protobuf-build" JAVA_JNI_CMAKE_ARGS+=" -DProtobuf_SRC_ROOT_FOLDER=${build_dir}/cpp/_deps/protobuf-src" JAVA_JNI_CMAKE_ARGS+=" -DARROW_JAVA_JNI_ABSL_INCLUDE_DIR=${absl_include_dir}" +JAVA_JNI_CMAKE_ARGS+=" ${llvm_cmake_args[*]}" export JAVA_JNI_CMAKE_ARGS "${source_dir}/ci/scripts/jni_build.sh" \ "${source_dir}" \ diff --git a/dataset/CMakeLists.txt b/dataset/CMakeLists.txt index 348850c3be..5ab57734fd 100644 --- a/dataset/CMakeLists.txt +++ b/dataset/CMakeLists.txt @@ -17,6 +17,7 @@ find_package(ArrowDataset REQUIRED) find_package(ArrowSubstrait REQUIRED) +find_package(ArrowS3 QUIET) include_directories(${CMAKE_CURRENT_BINARY_DIR} ${CMAKE_CURRENT_SOURCE_DIR} ${JNI_INCLUDE_DIRS} ${JNI_HEADERS_DIR}) @@ -40,6 +41,11 @@ target_link_libraries(arrow_java_jni_dataset ArrowDataset::arrow_dataset_static ArrowSubstrait::arrow_substrait_static) +# Older Arrow releases include S3 in arrow_static instead of a separate library. +if(TARGET ArrowS3::arrow_s3_static) + target_link_libraries(arrow_java_jni_dataset ArrowS3::arrow_s3_static) +endif() + if(BUILD_TESTING) add_executable(arrow-java-jni-dataset-test src/main/cpp/jni_util_test.cc src/main/cpp/jni_util.cc) From a55c2e477aa11307080dd1d011e1903ee3880ac5 Mon Sep 17 00:00:00 2001 From: David Li Date: Mon, 21 Sep 2026 11:57:09 +0900 Subject: [PATCH 2/2] test --- .github/workflows/rc.yml | 23 ++++++++++++++--------- Brewfile | 1 - 2 files changed, 14 insertions(+), 10 deletions(-) diff --git a/.github/workflows/rc.yml b/.github/workflows/rc.yml index 8b0e64d3af..18e8ca4820 100644 --- a/.github/workflows/rc.yml +++ b/.github/workflows/rc.yml @@ -197,6 +197,11 @@ jobs: python-version: 3.12 - name: Install Archery run: pip install -e arrow/dev/archery[all] + - name: Set up Java + uses: actions/setup-java@v6 + with: + java-version: '17' + distribution: 'temurin' - name: Install dependencies run: | # We can remove this when we drop support for @@ -227,7 +232,7 @@ jobs: brew uninstall pkg-config@0.29.2 || : fi - # We don't use Homebrew's aws-sdk-cpp and gRPC. See the + # We don't use Homebrew's aws-sdk-cpp, gRPC, RE2 or Protobuf. See the # "brew uninstall" calls below for details. So we don't install # them instead of installing and uninstalling them. # @@ -243,8 +248,9 @@ jobs: # We don't use Homebrew's Node.js too. Homebrew doesn't provide # a bottle for it on x86_64 macOS for the same reason. # LLVM is installed explicitly above instead of using the latest version. - HOMEBREW_BUNDLE_BREW_SKIP="aws-sdk-cpp grpc node llvm" \ - brew bundle --verbose --file=arrow/cpp/Brewfile + # Keep installed dependencies instead of rebuilding newer releases. + HOMEBREW_BUNDLE_BREW_SKIP="aws-sdk-cpp grpc re2 protobuf node llvm" \ + brew bundle --no-upgrade --verbose --file=arrow/cpp/Brewfile # We want to link aws-sdk-cpp statically but Homebrew's # aws-sdk-cpp provides only shared library. If we have # Homebrew's aws-sdk-cpp, our build mix Homebrew's @@ -259,14 +265,15 @@ jobs: # We uninstall Homebrew's RE2 to ensure using bundled RE2. brew uninstall grpc || : # gRPC depends on RE2 brew uninstall grpc@1.54 || : # gRPC 1.54 may be installed too - brew uninstall re2 # We want to use bundled Protobuf for static linking. If # Homebrew's Protobuf is installed, its library file may be # used on test We uninstall Homebrew's Protobuf to ensure using # bundled Protobuf. - brew uninstall protobuf - - brew bundle --file=Brewfile + for package in re2 protobuf; do + if brew list --versions "${package}" >/dev/null; then + brew uninstall "${package}" + fi + done - name: Prepare ccache run: | echo "CCACHE_DIR=${PWD}/ccache" >> ${GITHUB_ENV} @@ -279,8 +286,6 @@ jobs: - name: Build run: | set -e - # make brew Java available to CMake - export JAVA_HOME=$(brew --prefix openjdk@17)/libexec/openjdk.jdk/Contents/Home ci/scripts/jni_macos_build.sh . arrow build jni - name: Compress into single artifact to keep directory structure run: tar -cvzf jni-macos-${{ matrix.platform.arch }}.tar.gz jni/ diff --git a/Brewfile b/Brewfile index 2c47a38af5..45b3c7bb9e 100644 --- a/Brewfile +++ b/Brewfile @@ -16,4 +16,3 @@ # under the License. brew "openjdk@17" -brew "sccache"