diff --git a/.github/workflows/aws_test.yml b/.github/workflows/aws_test.yml index 7726b94a7a..22bce09aa5 100644 --- a/.github/workflows/aws_test.yml +++ b/.github/workflows/aws_test.yml @@ -114,6 +114,28 @@ jobs: if: ${{ matrix.s3 == 'ON' }} shell: bash run: bash ci/scripts/start_minio.sh + # Provide a prebuilt Arrow/Parquet (incl. the S3 filesystem) from conda-forge + # so the build resolves it via find_package(Arrow CONFIG) instead of compiling + # Arrow from source. + - name: Set up conda (prebuilt Arrow) + if: ${{ matrix.bundle_awssdk == 'OFF' }} + uses: conda-incubator/setup-miniconda@8ee1f361103df19b6f8c8655fd3967a8ecb162d5 # v4.0.1 + with: + miniforge-version: latest + channels: conda-forge + channel-priority: strict + activate-environment: arrow + - name: Install prebuilt Arrow + Parquet + if: ${{ matrix.bundle_awssdk == 'OFF' }} + shell: bash -el {0} + run: | + mamba install -y "libarrow=24.0.*" "libparquet=24.0.*" + echo "CMAKE_PREFIX_PATH=${CONDA_PREFIX}" >> "$GITHUB_ENV" + if [[ "${RUNNER_OS}" == "macOS" ]]; then + echo "DYLD_FALLBACK_LIBRARY_PATH=${DYLD_FALLBACK_LIBRARY_PATH:-}:${CONDA_PREFIX}/lib" >> "$GITHUB_ENV" + else + echo "LD_LIBRARY_PATH=${LD_LIBRARY_PATH:-}:${CONDA_PREFIX}/lib" >> "$GITHUB_ENV" + fi - name: Restore sccache cache uses: actions/cache/restore@27d5ce7f107fe9357f9df03efb73ab90386fccae # v5.0.5 with: diff --git a/.github/workflows/cpp-linter.yml b/.github/workflows/cpp-linter.yml index 49b056afee..ff76d96160 100644 --- a/.github/workflows/cpp-linter.yml +++ b/.github/workflows/cpp-linter.yml @@ -58,6 +58,21 @@ jobs: run: | sudo apt-get update sudo apt-get install -y libcurl4-openssl-dev libsqlite3-dev libpq-dev default-libmysqlclient-dev + # Provide a prebuilt Arrow/Parquet from conda-forge so the build resolves it + # via find_package(Arrow CONFIG) instead of compiling Arrow from source. + - name: Set up conda (prebuilt Arrow) + uses: conda-incubator/setup-miniconda@8ee1f361103df19b6f8c8655fd3967a8ecb162d5 # v4.0.1 + with: + miniforge-version: latest + channels: conda-forge + channel-priority: strict + activate-environment: arrow + - name: Install prebuilt Arrow + Parquet + shell: bash -el {0} + run: | + mamba install -y "libarrow=24.0.*" "libparquet=24.0.*" + echo "CMAKE_PREFIX_PATH=${CONDA_PREFIX}" >> "$GITHUB_ENV" + echo "LD_LIBRARY_PATH=${LD_LIBRARY_PATH:-}:${CONDA_PREFIX}/lib" >> "$GITHUB_ENV" - name: Restore sccache cache uses: actions/cache/restore@27d5ce7f107fe9357f9df03efb73ab90386fccae # v5.0.5 with: diff --git a/.github/workflows/sanitizer_test.yml b/.github/workflows/sanitizer_test.yml index f5eea55b90..3e4f6b1e87 100644 --- a/.github/workflows/sanitizer_test.yml +++ b/.github/workflows/sanitizer_test.yml @@ -58,6 +58,23 @@ jobs: sccache-sanitizer-ubuntu- - name: Setup sccache uses: mozilla-actions/sccache-action@9e7fa8a12102821edf02ca5dbea1acd0f89a2696 # v0.0.10 + # Provide a prebuilt Arrow/Parquet from conda-forge so the build resolves it + # via find_package(Arrow CONFIG) instead of compiling Arrow from source. Arrow + # itself is not sanitizer-instrumented, which is fine: ASAN/UBSAN target + # iceberg's own code (container-overflow detection is already disabled). + - name: Set up conda (prebuilt Arrow) + uses: conda-incubator/setup-miniconda@8ee1f361103df19b6f8c8655fd3967a8ecb162d5 # v4.0.1 + with: + miniforge-version: latest + channels: conda-forge + channel-priority: strict + activate-environment: arrow + - name: Install prebuilt Arrow + Parquet + shell: bash -el {0} + run: | + mamba install -y "libarrow=24.0.*" "libparquet=24.0.*" + echo "CMAKE_PREFIX_PATH=${CONDA_PREFIX}" >> "$GITHUB_ENV" + echo "LD_LIBRARY_PATH=${LD_LIBRARY_PATH:-}:${CONDA_PREFIX}/lib" >> "$GITHUB_ENV" - name: Configure and Build with ASAN & UBSAN env: CC: gcc-14 diff --git a/.github/workflows/sql_catalog_test.yml b/.github/workflows/sql_catalog_test.yml index 654a1abd76..6c5147783e 100644 --- a/.github/workflows/sql_catalog_test.yml +++ b/.github/workflows/sql_catalog_test.yml @@ -123,6 +123,27 @@ jobs: sccache-sqlcatalog-${{ matrix.runs-on }}${{ env.SCCACHE_KEY_SUFFIX }}- - name: Setup sccache uses: mozilla-actions/sccache-action@9e7fa8a12102821edf02ca5dbea1acd0f89a2696 # v0.0.10 + # Provide a prebuilt Arrow/Parquet from conda-forge so the build resolves it + # via find_package(Arrow CONFIG) instead of compiling Arrow from source. + - name: Set up conda (prebuilt Arrow) + if: ${{ !startsWith(matrix.runs-on, 'windows') }} + uses: conda-incubator/setup-miniconda@8ee1f361103df19b6f8c8655fd3967a8ecb162d5 # v4.0.1 + with: + miniforge-version: latest + channels: conda-forge + channel-priority: strict + activate-environment: arrow + - name: Install prebuilt Arrow + Parquet (Unix) + if: ${{ !startsWith(matrix.runs-on, 'windows') }} + shell: bash -el {0} + run: | + mamba install -y "libarrow=24.0.*" "libparquet=24.0.*" + echo "CMAKE_PREFIX_PATH=${CONDA_PREFIX}" >> "$GITHUB_ENV" + if [[ "${RUNNER_OS}" == "macOS" ]]; then + echo "DYLD_FALLBACK_LIBRARY_PATH=${DYLD_FALLBACK_LIBRARY_PATH:-}:${CONDA_PREFIX}/lib" >> "$GITHUB_ENV" + else + echo "LD_LIBRARY_PATH=${LD_LIBRARY_PATH:-}:${CONDA_PREFIX}/lib" >> "$GITHUB_ENV" + fi - name: Configure Iceberg shell: bash run: | diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 66222c6539..18e35a9cef 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -55,6 +55,23 @@ jobs: - name: Install dependencies shell: bash run: sudo apt-get update && sudo apt-get install -y libcurl4-openssl-dev + # Provide a prebuilt Arrow/Parquet from conda-forge so the build resolves it + # via find_package(Arrow CONFIG) instead of compiling Arrow from source. + - name: Set up conda (prebuilt Arrow) + uses: conda-incubator/setup-miniconda@8ee1f361103df19b6f8c8655fd3967a8ecb162d5 # v4.0.1 + with: + miniforge-version: latest + channels: conda-forge + channel-priority: strict + activate-environment: arrow + - name: Install prebuilt Arrow + Parquet + shell: bash -el {0} + run: | + mamba install -y "libarrow=24.0.*" "libparquet=24.0.*" + echo "CMAKE_PREFIX_PATH=${CONDA_PREFIX}" >> "$GITHUB_ENV" + echo "LD_LIBRARY_PATH=${LD_LIBRARY_PATH:-}:${CONDA_PREFIX}/lib" >> "$GITHUB_ENV" + # Test binaries are throwaway, so drop debug info to shrink objects + sccache. + echo "ICEBERG_EXTRA_CMAKE_ARGS=-DCMAKE_CXX_FLAGS_DEBUG=-g0 -DCMAKE_C_FLAGS_DEBUG=-g0" >> "$GITHUB_ENV" - name: Restore sccache cache uses: actions/cache/restore@27d5ce7f107fe9357f9df03efb73ab90386fccae # v5.0.5 with: @@ -100,6 +117,23 @@ jobs: uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 with: persist-credentials: false + # Provide a prebuilt Arrow/Parquet from conda-forge so the build resolves it + # via find_package(Arrow CONFIG) instead of compiling Arrow from source. + - name: Set up conda (prebuilt Arrow) + uses: conda-incubator/setup-miniconda@8ee1f361103df19b6f8c8655fd3967a8ecb162d5 # v4.0.1 + with: + miniforge-version: latest + channels: conda-forge + channel-priority: strict + activate-environment: arrow + - name: Install prebuilt Arrow + Parquet + shell: bash -el {0} + run: | + mamba install -y "libarrow=24.0.*" "libparquet=24.0.*" + echo "CMAKE_PREFIX_PATH=${CONDA_PREFIX}" >> "$GITHUB_ENV" + echo "DYLD_FALLBACK_LIBRARY_PATH=${DYLD_FALLBACK_LIBRARY_PATH:-}:${CONDA_PREFIX}/lib" >> "$GITHUB_ENV" + # Test binaries are throwaway, so drop debug info to shrink objects + sccache. + echo "ICEBERG_EXTRA_CMAKE_ARGS=-DCMAKE_CXX_FLAGS_DEBUG=-g0 -DCMAKE_C_FLAGS_DEBUG=-g0" >> "$GITHUB_ENV" - name: Restore sccache cache uses: actions/cache/restore@27d5ce7f107fe9357f9df03efb73ab90386fccae # v5.0.5 with: diff --git a/README.md b/README.md index 7c9a343ec0..41dbc8716e 100644 --- a/README.md +++ b/README.md @@ -202,3 +202,5 @@ If you make improvements that could benefit all developers, please update the te ## License Licensed under the [Apache License, Version 2.0](http://www.apache.org/licenses/LICENSE-2.0) + +