From 4b7d7d46839783dde1db2b56e49fe4ad76082592 Mon Sep 17 00:00:00 2001 From: Abanoub Doss Date: Mon, 27 Jul 2026 19:29:19 -0500 Subject: [PATCH] ci: extract conda Arrow install into a script and derive its pin --- .github/actions/setup-conda-arrow/action.yml | 21 +------- .../setup-conda-arrow/install-arrow.sh | 49 +++++++++++++++++++ .github/workflows/aws_test.yml | 2 - .github/workflows/sql_catalog_test.yml | 2 - .github/workflows/test.yml | 2 - 5 files changed, 50 insertions(+), 26 deletions(-) create mode 100755 .github/actions/setup-conda-arrow/install-arrow.sh diff --git a/.github/actions/setup-conda-arrow/action.yml b/.github/actions/setup-conda-arrow/action.yml index 980c6302d5..6382063cc3 100644 --- a/.github/actions/setup-conda-arrow/action.yml +++ b/.github/actions/setup-conda-arrow/action.yml @@ -19,10 +19,6 @@ name: Set up conda Arrow description: Install Arrow and Parquet from conda-forge for CMake builds. inputs: - runtime-library-path: - description: Runtime library path variable to export. - required: false - default: LD_LIBRARY_PATH extra-packages: description: Extra conda packages to install, space separated. required: false @@ -41,20 +37,5 @@ runs: - name: Install Arrow and Parquet shell: bash -el {0} env: - RUNTIME_LIBRARY_PATH: ${{ inputs.runtime-library-path }} EXTRA_PACKAGES: ${{ inputs.extra-packages }} - run: | - read -r -a extra_packages <<< "${EXTRA_PACKAGES}" - mamba install -y "libarrow=24.0.*" "libparquet=24.0.*" "${extra_packages[@]}" - echo "CMAKE_PREFIX_PATH=${CONDA_PREFIX}" >> "$GITHUB_ENV" - echo "ICEBERG_REQUIRE_SYSTEM_ARROW=ON" >> "$GITHUB_ENV" - - case "${RUNTIME_LIBRARY_PATH}" in - LD_LIBRARY_PATH|DYLD_FALLBACK_LIBRARY_PATH) ;; - *) - echo "::error::Unsupported runtime-library-path: ${RUNTIME_LIBRARY_PATH}" - exit 1 - ;; - esac - runtime_path_var="${RUNTIME_LIBRARY_PATH}" - echo "${runtime_path_var}=${!runtime_path_var:+${!runtime_path_var}:}${CONDA_PREFIX}/lib" >> "$GITHUB_ENV" + run: '"${{ github.action_path }}/install-arrow.sh"' diff --git a/.github/actions/setup-conda-arrow/install-arrow.sh b/.github/actions/setup-conda-arrow/install-arrow.sh new file mode 100755 index 0000000000..8590666f55 --- /dev/null +++ b/.github/actions/setup-conda-arrow/install-arrow.sh @@ -0,0 +1,49 @@ +#!/usr/bin/env bash +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# +# Installs Arrow and Parquet into the active conda environment and exports the +# paths CMake and the test binaries need. Run from the setup-conda-arrow action. + +set -euo pipefail + +toolchain="${GITHUB_WORKSPACE}/cmake_modules/IcebergThirdpartyToolchain.cmake" + +# Keep conda Arrow in sync with the vendored build. +arrow_version=$(sed -nE 's/^set\(ICEBERG_ARROW_BUILD_VERSION "([0-9]+\.[0-9]+\.[0-9]+)"\)$/\1/p' "${toolchain}") +if [[ -z "${arrow_version}" ]]; then + echo "::error::Could not read ICEBERG_ARROW_BUILD_VERSION from ${toolchain}" + exit 1 +fi + +packages=("libarrow==${arrow_version}" "libparquet==${arrow_version}") +if [[ -n "${EXTRA_PACKAGES:-}" ]]; then + read -r -a extra_packages <<< "${EXTRA_PACKAGES}" + packages+=("${extra_packages[@]}") +fi +mamba install -y "${packages[@]}" + +echo "CMAKE_PREFIX_PATH=${CONDA_PREFIX}" >> "${GITHUB_ENV}" +# build_iceberg.sh reads this and fails if CMake configured vendored Arrow anyway. +echo "ICEBERG_REQUIRE_SYSTEM_ARROW=ON" >> "${GITHUB_ENV}" + +if [[ "${RUNNER_OS}" == "macOS" ]]; then + echo "DYLD_FALLBACK_LIBRARY_PATH=${DYLD_FALLBACK_LIBRARY_PATH:+${DYLD_FALLBACK_LIBRARY_PATH}:}${CONDA_PREFIX}/lib" >> "${GITHUB_ENV}" +else + echo "LD_LIBRARY_PATH=${LD_LIBRARY_PATH:+${LD_LIBRARY_PATH}:}${CONDA_PREFIX}/lib" >> "${GITHUB_ENV}" +fi diff --git a/.github/workflows/aws_test.yml b/.github/workflows/aws_test.yml index d97b14dac7..85a21837f1 100644 --- a/.github/workflows/aws_test.yml +++ b/.github/workflows/aws_test.yml @@ -120,8 +120,6 @@ jobs: - name: Set up conda Arrow if: ${{ matrix.bundle_awssdk == 'OFF' }} uses: ./.github/actions/setup-conda-arrow - with: - runtime-library-path: ${{ startsWith(matrix.runs-on, 'macos') && 'DYLD_FALLBACK_LIBRARY_PATH' || 'LD_LIBRARY_PATH' }} - name: Set up sccache uses: ./.github/actions/setup-sccache with: diff --git a/.github/workflows/sql_catalog_test.yml b/.github/workflows/sql_catalog_test.yml index bb4bbe2f3b..a462a968fb 100644 --- a/.github/workflows/sql_catalog_test.yml +++ b/.github/workflows/sql_catalog_test.yml @@ -99,8 +99,6 @@ jobs: - name: Set up conda Arrow if: ${{ !startsWith(matrix.runs-on, 'windows') }} uses: ./.github/actions/setup-conda-arrow - with: - runtime-library-path: ${{ startsWith(matrix.runs-on, 'macos') && 'DYLD_FALLBACK_LIBRARY_PATH' || 'LD_LIBRARY_PATH' }} - name: Set up sccache uses: ./.github/actions/setup-sccache with: diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 94bf4bc7b1..47514c7f5c 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -143,8 +143,6 @@ jobs: persist-credentials: false - name: Set up conda Arrow uses: ./.github/actions/setup-conda-arrow - with: - runtime-library-path: DYLD_FALLBACK_LIBRARY_PATH - name: Set up sccache uses: ./.github/actions/setup-sccache with: