From c0f77999a7ec45e0a25b3d58feb5b9cabf854728 Mon Sep 17 00:00:00 2001 From: Abanoub Doss Date: Fri, 3 Jul 2026 14:11:23 -0500 Subject: [PATCH 1/4] ci: use conda Arrow in Unix test legs --- .github/workflows/test.yml | 33 +++++++++++++++++++++++++++++++++ ci/scripts/build_iceberg.sh | 5 +++++ 2 files changed, 38 insertions(+) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 23fcc2d1c3..a06797e5f9 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -56,6 +56,21 @@ jobs: - name: Install dependencies shell: bash run: sudo apt-get update && sudo apt-get install -y libcurl4-openssl-dev + # Provide a prebuilt Arrow/Parquet from conda-forge so the build resolves it + # via find_package(Arrow CONFIG) instead of compiling Arrow from source. + - name: Set up conda (prebuilt Arrow) + uses: conda-incubator/setup-miniconda@8ee1f361103df19b6f8c8655fd3967a8ecb162d5 # v4.0.1 + with: + miniforge-version: latest + channels: conda-forge + channel-priority: strict + activate-environment: arrow + - name: Install prebuilt Arrow + Parquet + shell: bash -el {0} + run: | + mamba install -y "libarrow=24.0.*" "libparquet=24.0.*" + echo "CMAKE_PREFIX_PATH=${CONDA_PREFIX}" >> "$GITHUB_ENV" + echo "LD_LIBRARY_PATH=${LD_LIBRARY_PATH:+${LD_LIBRARY_PATH}:}${CONDA_PREFIX}/lib" >> "$GITHUB_ENV" - name: Restore sccache cache uses: actions/cache/restore@27d5ce7f107fe9357f9df03efb73ab90386fccae # v5.0.5 with: @@ -70,6 +85,7 @@ jobs: env: CC: gcc-14 CXX: g++-14 + ICEBERG_REQUIRE_SYSTEM_ARROW: ON run: ci/scripts/build_iceberg.sh $(pwd) ON ON - name: Show sccache stats shell: bash @@ -101,6 +117,21 @@ jobs: uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 with: persist-credentials: false + # Provide a prebuilt Arrow/Parquet from conda-forge so the build resolves it + # via find_package(Arrow CONFIG) instead of compiling Arrow from source. + - name: Set up conda (prebuilt Arrow) + uses: conda-incubator/setup-miniconda@8ee1f361103df19b6f8c8655fd3967a8ecb162d5 # v4.0.1 + with: + miniforge-version: latest + channels: conda-forge + channel-priority: strict + activate-environment: arrow + - name: Install prebuilt Arrow + Parquet + shell: bash -el {0} + run: | + mamba install -y "libarrow=24.0.*" "libparquet=24.0.*" + echo "CMAKE_PREFIX_PATH=${CONDA_PREFIX}" >> "$GITHUB_ENV" + echo "DYLD_FALLBACK_LIBRARY_PATH=${DYLD_FALLBACK_LIBRARY_PATH:+${DYLD_FALLBACK_LIBRARY_PATH}:}${CONDA_PREFIX}/lib" >> "$GITHUB_ENV" - name: Restore sccache cache uses: actions/cache/restore@27d5ce7f107fe9357f9df03efb73ab90386fccae # v5.0.5 with: @@ -112,6 +143,8 @@ jobs: uses: mozilla-actions/sccache-action@9e7fa8a12102821edf02ca5dbea1acd0f89a2696 # v0.0.10 - name: Build Iceberg shell: bash + env: + ICEBERG_REQUIRE_SYSTEM_ARROW: ON run: ci/scripts/build_iceberg.sh $(pwd) OFF ON - name: Show sccache stats shell: bash diff --git a/ci/scripts/build_iceberg.sh b/ci/scripts/build_iceberg.sh index 6af0802f6b..5fdf10cc25 100755 --- a/ci/scripts/build_iceberg.sh +++ b/ci/scripts/build_iceberg.sh @@ -85,6 +85,11 @@ if [[ -n "${ICEBERG_EXTRA_CMAKE_ARGS:-}" ]]; then fi cmake "${CMAKE_ARGS[@]}" ${source_dir} +if [[ "${ICEBERG_REQUIRE_SYSTEM_ARROW:-OFF}" == "ON" && -d _deps/vendoredarrow-src ]]; then + echo "::error::Expected Arrow from CMAKE_PREFIX_PATH, but CMake configured vendored Arrow" + exit 1 +fi + if is_windows; then cmake --build . --config Release --target install if [[ "${run_tests}" == "ON" ]]; then From cec570159a9ff9a030ac1fba3dfc25d41bc83ddb Mon Sep 17 00:00:00 2001 From: Abanoub Doss Date: Fri, 3 Jul 2026 14:13:10 -0500 Subject: [PATCH 2/4] ci: share conda Arrow setup for lint and sanitizer --- .github/actions/setup-conda-arrow/action.yml | 54 ++++++++++++++++++++ .github/workflows/cpp-linter.yml | 6 +++ .github/workflows/sanitizer_test.yml | 6 +++ .github/workflows/test.yml | 37 ++------------ 4 files changed, 71 insertions(+), 32 deletions(-) create mode 100644 .github/actions/setup-conda-arrow/action.yml diff --git a/.github/actions/setup-conda-arrow/action.yml b/.github/actions/setup-conda-arrow/action.yml new file mode 100644 index 0000000000..527d352705 --- /dev/null +++ b/.github/actions/setup-conda-arrow/action.yml @@ -0,0 +1,54 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +name: Set up conda Arrow +description: Install Arrow and Parquet from conda-forge for CMake builds. + +inputs: + runtime-library-path: + description: Runtime library path variable to export. + required: false + default: LD_LIBRARY_PATH + +runs: + using: composite + steps: + - name: Set up conda + uses: conda-incubator/setup-miniconda@8ee1f361103df19b6f8c8655fd3967a8ecb162d5 # v4.0.1 + with: + miniforge-version: latest + channels: conda-forge + channel-priority: strict + activate-environment: arrow + - name: Install Arrow and Parquet + shell: bash -el {0} + env: + RUNTIME_LIBRARY_PATH: ${{ inputs.runtime-library-path }} + run: | + mamba install -y "libarrow=24.0.*" "libparquet=24.0.*" + echo "CMAKE_PREFIX_PATH=${CONDA_PREFIX}" >> "$GITHUB_ENV" + echo "ICEBERG_REQUIRE_SYSTEM_ARROW=ON" >> "$GITHUB_ENV" + + case "${RUNTIME_LIBRARY_PATH}" in + LD_LIBRARY_PATH|DYLD_FALLBACK_LIBRARY_PATH) ;; + *) + echo "::error::Unsupported runtime-library-path: ${RUNTIME_LIBRARY_PATH}" + exit 1 + ;; + esac + runtime_path_var="${RUNTIME_LIBRARY_PATH}" + echo "${runtime_path_var}=${!runtime_path_var:+${!runtime_path_var}:}${CONDA_PREFIX}/lib" >> "$GITHUB_ENV" diff --git a/.github/workflows/cpp-linter.yml b/.github/workflows/cpp-linter.yml index 304d1a58b2..8b336d0359 100644 --- a/.github/workflows/cpp-linter.yml +++ b/.github/workflows/cpp-linter.yml @@ -58,6 +58,8 @@ jobs: run: | sudo apt-get update sudo apt-get install -y libcurl4-openssl-dev libsqlite3-dev libpq-dev default-libmysqlclient-dev + - name: Set up conda Arrow + uses: ./.github/actions/setup-conda-arrow - name: Restore sccache cache uses: actions/cache/restore@27d5ce7f107fe9357f9df03efb73ab90386fccae # v5.0.5 with: @@ -80,6 +82,10 @@ jobs: -DICEBERG_SQL_SQLITE=ON \ -DICEBERG_SQL_POSTGRESQL=ON \ -DICEBERG_SQL_MYSQL=ON + if [[ -d _deps/vendoredarrow-src ]]; then + echo "::error::Expected Arrow from CMAKE_PREFIX_PATH, but CMake configured vendored Arrow" + exit 1 + fi cmake --build . - name: Show sccache stats shell: bash diff --git a/.github/workflows/sanitizer_test.yml b/.github/workflows/sanitizer_test.yml index 48eb578466..6a480135a8 100644 --- a/.github/workflows/sanitizer_test.yml +++ b/.github/workflows/sanitizer_test.yml @@ -50,6 +50,8 @@ jobs: - name: Install dependencies shell: bash run: sudo apt-get update && sudo apt-get install -y libcurl4-openssl-dev + - name: Set up conda Arrow + uses: ./.github/actions/setup-conda-arrow - name: Restore sccache cache uses: actions/cache/restore@27d5ce7f107fe9357f9df03efb73ab90386fccae # v5.0.5 with: @@ -67,6 +69,10 @@ jobs: mkdir build && cd build cmake .. -G Ninja -DCMAKE_BUILD_TYPE=Debug -DICEBERG_ENABLE_ASAN=ON -DICEBERG_ENABLE_UBSAN=ON \ -DCMAKE_C_COMPILER_LAUNCHER=sccache -DCMAKE_CXX_COMPILER_LAUNCHER=sccache + if [[ -d _deps/vendoredarrow-src ]]; then + echo "::error::Expected Arrow from CMAKE_PREFIX_PATH, but CMake configured vendored Arrow" + exit 1 + fi cmake --build . --verbose - name: Show sccache stats shell: bash diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index a06797e5f9..dcc0093871 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -56,21 +56,8 @@ jobs: - name: Install dependencies shell: bash run: sudo apt-get update && sudo apt-get install -y libcurl4-openssl-dev - # Provide a prebuilt Arrow/Parquet from conda-forge so the build resolves it - # via find_package(Arrow CONFIG) instead of compiling Arrow from source. - - name: Set up conda (prebuilt Arrow) - uses: conda-incubator/setup-miniconda@8ee1f361103df19b6f8c8655fd3967a8ecb162d5 # v4.0.1 - with: - miniforge-version: latest - channels: conda-forge - channel-priority: strict - activate-environment: arrow - - name: Install prebuilt Arrow + Parquet - shell: bash -el {0} - run: | - mamba install -y "libarrow=24.0.*" "libparquet=24.0.*" - echo "CMAKE_PREFIX_PATH=${CONDA_PREFIX}" >> "$GITHUB_ENV" - echo "LD_LIBRARY_PATH=${LD_LIBRARY_PATH:+${LD_LIBRARY_PATH}:}${CONDA_PREFIX}/lib" >> "$GITHUB_ENV" + - name: Set up conda Arrow + uses: ./.github/actions/setup-conda-arrow - name: Restore sccache cache uses: actions/cache/restore@27d5ce7f107fe9357f9df03efb73ab90386fccae # v5.0.5 with: @@ -85,7 +72,6 @@ jobs: env: CC: gcc-14 CXX: g++-14 - ICEBERG_REQUIRE_SYSTEM_ARROW: ON run: ci/scripts/build_iceberg.sh $(pwd) ON ON - name: Show sccache stats shell: bash @@ -117,21 +103,10 @@ jobs: uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 with: persist-credentials: false - # Provide a prebuilt Arrow/Parquet from conda-forge so the build resolves it - # via find_package(Arrow CONFIG) instead of compiling Arrow from source. - - name: Set up conda (prebuilt Arrow) - uses: conda-incubator/setup-miniconda@8ee1f361103df19b6f8c8655fd3967a8ecb162d5 # v4.0.1 + - name: Set up conda Arrow + uses: ./.github/actions/setup-conda-arrow with: - miniforge-version: latest - channels: conda-forge - channel-priority: strict - activate-environment: arrow - - name: Install prebuilt Arrow + Parquet - shell: bash -el {0} - run: | - mamba install -y "libarrow=24.0.*" "libparquet=24.0.*" - echo "CMAKE_PREFIX_PATH=${CONDA_PREFIX}" >> "$GITHUB_ENV" - echo "DYLD_FALLBACK_LIBRARY_PATH=${DYLD_FALLBACK_LIBRARY_PATH:+${DYLD_FALLBACK_LIBRARY_PATH}:}${CONDA_PREFIX}/lib" >> "$GITHUB_ENV" + runtime-library-path: DYLD_FALLBACK_LIBRARY_PATH - name: Restore sccache cache uses: actions/cache/restore@27d5ce7f107fe9357f9df03efb73ab90386fccae # v5.0.5 with: @@ -143,8 +118,6 @@ jobs: uses: mozilla-actions/sccache-action@9e7fa8a12102821edf02ca5dbea1acd0f89a2696 # v0.0.10 - name: Build Iceberg shell: bash - env: - ICEBERG_REQUIRE_SYSTEM_ARROW: ON run: ci/scripts/build_iceberg.sh $(pwd) OFF ON - name: Show sccache stats shell: bash From be441b39b737ceb8ab00074b8b697a9d1f1997bc Mon Sep 17 00:00:00 2001 From: Abanoub Doss Date: Fri, 3 Jul 2026 14:14:04 -0500 Subject: [PATCH 3/4] ci: use conda Arrow in SQL and AWS Unix legs --- .github/workflows/aws_test.yml | 5 +++++ .github/workflows/sql_catalog_test.yml | 9 +++++++++ 2 files changed, 14 insertions(+) diff --git a/.github/workflows/aws_test.yml b/.github/workflows/aws_test.yml index cb486034ba..bd8b7a7899 100644 --- a/.github/workflows/aws_test.yml +++ b/.github/workflows/aws_test.yml @@ -115,6 +115,11 @@ jobs: if: ${{ matrix.s3 == 'ON' }} shell: bash run: bash ci/scripts/start_minio.sh + - name: Set up conda Arrow + if: ${{ matrix.bundle_awssdk == 'OFF' }} + uses: ./.github/actions/setup-conda-arrow + with: + runtime-library-path: ${{ startsWith(matrix.runs-on, 'macos') && 'DYLD_FALLBACK_LIBRARY_PATH' || 'LD_LIBRARY_PATH' }} - name: Restore sccache cache uses: actions/cache/restore@27d5ce7f107fe9357f9df03efb73ab90386fccae # v5.0.5 with: diff --git a/.github/workflows/sql_catalog_test.yml b/.github/workflows/sql_catalog_test.yml index 905b4a06c4..480e287202 100644 --- a/.github/workflows/sql_catalog_test.yml +++ b/.github/workflows/sql_catalog_test.yml @@ -115,6 +115,11 @@ jobs: shell: pwsh run: | vcpkg install zlib:x64-windows nlohmann-json:x64-windows nanoarrow:x64-windows roaring:x64-windows sqlite3:x64-windows + - name: Set up conda Arrow + if: ${{ !startsWith(matrix.runs-on, 'windows') }} + uses: ./.github/actions/setup-conda-arrow + with: + runtime-library-path: ${{ startsWith(matrix.runs-on, 'macos') && 'DYLD_FALLBACK_LIBRARY_PATH' || 'LD_LIBRARY_PATH' }} - name: Restore sccache cache uses: actions/cache/restore@27d5ce7f107fe9357f9df03efb73ab90386fccae # v5.0.5 with: @@ -138,6 +143,10 @@ jobs: -DCMAKE_C_COMPILER_LAUNCHER=sccache \ -DCMAKE_CXX_COMPILER_LAUNCHER=sccache \ ${{ matrix.cmake_extra_args }} + if [[ "${RUNNER_OS}" != "Windows" && -d build/_deps/vendoredarrow-src ]]; then + echo "::error::Expected Arrow from CMAKE_PREFIX_PATH, but CMake configured vendored Arrow" + exit 1 + fi - name: Build SQL catalog tests shell: bash run: cmake --build build --target sql_catalog_test From b641dc71d3c5a3d78e8a2cf565dcd206e0d1cc31 Mon Sep 17 00:00:00 2001 From: Abanoub Doss Date: Fri, 3 Jul 2026 14:20:43 -0500 Subject: [PATCH 4/4] test: trigger cpp-linter validation --- README.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/README.md b/README.md index 7c9a343ec0..c8c78f064d 100644 --- a/README.md +++ b/README.md @@ -202,3 +202,5 @@ If you make improvements that could benefit all developers, please update the te ## License Licensed under the [Apache License, Version 2.0](http://www.apache.org/licenses/LICENSE-2.0) + +