diff --git a/.github/workflows/aws_test.yml b/.github/workflows/aws_test.yml index 47c54241d0..7cbeccddaa 100644 --- a/.github/workflows/aws_test.yml +++ b/.github/workflows/aws_test.yml @@ -115,6 +115,26 @@ jobs: if: ${{ matrix.s3 == 'ON' }} shell: bash run: bash ci/scripts/start_minio.sh + # Provide a prebuilt Arrow/Parquet (incl. the S3 filesystem) from conda-forge + # so the build resolves it via find_package(Arrow CONFIG) instead of compiling + # Arrow from source. + - name: Set up conda (prebuilt Arrow) + uses: conda-incubator/setup-miniconda@8ee1f361103df19b6f8c8655fd3967a8ecb162d5 # v4.0.1 + with: + miniforge-version: latest + channels: conda-forge + channel-priority: strict + activate-environment: arrow + - name: Install prebuilt Arrow + Parquet + shell: bash -el {0} + run: | + mamba install -y "libarrow=24.0.*" "libparquet=24.0.*" + echo "CMAKE_PREFIX_PATH=${CONDA_PREFIX}" >> "$GITHUB_ENV" + if [[ "${RUNNER_OS}" == "macOS" ]]; then + echo "DYLD_FALLBACK_LIBRARY_PATH=${DYLD_FALLBACK_LIBRARY_PATH:-}:${CONDA_PREFIX}/lib" >> "$GITHUB_ENV" + else + echo "LD_LIBRARY_PATH=${LD_LIBRARY_PATH:-}:${CONDA_PREFIX}/lib" >> "$GITHUB_ENV" + fi - name: Restore sccache cache uses: actions/cache/restore@27d5ce7f107fe9357f9df03efb73ab90386fccae # v5.0.5 with: diff --git a/.github/workflows/cpp-linter.yml b/.github/workflows/cpp-linter.yml index 2e3e022d67..f33d7f191c 100644 --- a/.github/workflows/cpp-linter.yml +++ b/.github/workflows/cpp-linter.yml @@ -48,6 +48,21 @@ jobs: run: | sudo apt-get update sudo apt-get install -y libcurl4-openssl-dev libsqlite3-dev libpq-dev default-libmysqlclient-dev + # Provide a prebuilt Arrow/Parquet from conda-forge so the build resolves it + # via find_package(Arrow CONFIG) instead of compiling Arrow from source. + - name: Set up conda (prebuilt Arrow) + uses: conda-incubator/setup-miniconda@8ee1f361103df19b6f8c8655fd3967a8ecb162d5 # v4.0.1 + with: + miniforge-version: latest + channels: conda-forge + channel-priority: strict + activate-environment: arrow + - name: Install prebuilt Arrow + Parquet + shell: bash -el {0} + run: | + mamba install -y "libarrow=24.0.*" "libparquet=24.0.*" + echo "CMAKE_PREFIX_PATH=${CONDA_PREFIX}" >> "$GITHUB_ENV" + echo "LD_LIBRARY_PATH=${LD_LIBRARY_PATH:-}:${CONDA_PREFIX}/lib" >> "$GITHUB_ENV" - name: Run build env: CC: gcc-14 diff --git a/.github/workflows/sanitizer_test.yml b/.github/workflows/sanitizer_test.yml index 0910b92158..22e90d30f3 100644 --- a/.github/workflows/sanitizer_test.yml +++ b/.github/workflows/sanitizer_test.yml @@ -59,6 +59,23 @@ jobs: sccache-sanitizer-ubuntu- - name: Setup sccache uses: mozilla-actions/sccache-action@9e7fa8a12102821edf02ca5dbea1acd0f89a2696 # v0.0.10 + # Provide a prebuilt Arrow/Parquet from conda-forge so the build resolves it + # via find_package(Arrow CONFIG) instead of compiling Arrow from source. Arrow + # itself is not sanitizer-instrumented, which is fine: ASAN/UBSAN target + # iceberg's own code (container-overflow detection is already disabled). + - name: Set up conda (prebuilt Arrow) + uses: conda-incubator/setup-miniconda@8ee1f361103df19b6f8c8655fd3967a8ecb162d5 # v4.0.1 + with: + miniforge-version: latest + channels: conda-forge + channel-priority: strict + activate-environment: arrow + - name: Install prebuilt Arrow + Parquet + shell: bash -el {0} + run: | + mamba install -y "libarrow=24.0.*" "libparquet=24.0.*" + echo "CMAKE_PREFIX_PATH=${CONDA_PREFIX}" >> "$GITHUB_ENV" + echo "LD_LIBRARY_PATH=${LD_LIBRARY_PATH:-}:${CONDA_PREFIX}/lib" >> "$GITHUB_ENV" - name: Configure and Build with ASAN & UBSAN env: CC: gcc-14 diff --git a/.github/workflows/sql_catalog_test.yml b/.github/workflows/sql_catalog_test.yml index 7682998890..0949509adb 100644 --- a/.github/workflows/sql_catalog_test.yml +++ b/.github/workflows/sql_catalog_test.yml @@ -104,6 +104,39 @@ jobs: sccache-sqlcatalog-${{ matrix.runs-on }}- - name: Setup sccache uses: mozilla-actions/sccache-action@9e7fa8a12102821edf02ca5dbea1acd0f89a2696 # v0.0.10 + # Provide a prebuilt Arrow/Parquet from conda-forge so the build resolves it + # via find_package(Arrow CONFIG) instead of compiling Arrow from source. + - name: Set up conda (prebuilt Arrow) + uses: conda-incubator/setup-miniconda@8ee1f361103df19b6f8c8655fd3967a8ecb162d5 # v4.0.1 + with: + miniforge-version: latest + channels: conda-forge + channel-priority: strict + activate-environment: arrow + - name: Install prebuilt Arrow + Parquet (Unix) + if: ${{ !startsWith(matrix.runs-on, 'windows') }} + shell: bash -el {0} + run: | + mamba install -y "libarrow=24.0.*" "libparquet=24.0.*" + echo "CMAKE_PREFIX_PATH=${CONDA_PREFIX}" >> "$GITHUB_ENV" + if [[ "${RUNNER_OS}" == "macOS" ]]; then + echo "DYLD_FALLBACK_LIBRARY_PATH=${DYLD_FALLBACK_LIBRARY_PATH:-}:${CONDA_PREFIX}/lib" >> "$GITHUB_ENV" + else + echo "LD_LIBRARY_PATH=${LD_LIBRARY_PATH:-}:${CONDA_PREFIX}/lib" >> "$GITHUB_ENV" + fi + - name: Install prebuilt Arrow + Parquet (Windows) + if: ${{ startsWith(matrix.runs-on, 'windows') }} + shell: pwsh + run: | + $ErrorActionPreference = "Stop" + $libDir = Join-Path $env:CONDA "envs\arrow\Library" + & "$env:CONDA\condabin\conda.bat" install -n arrow -y -c conda-forge "libarrow=24.0.*" "libparquet=24.0.*" + if ($LASTEXITCODE -ne 0) { throw "conda install failed" } + if (-not (Test-Path (Join-Path $libDir "lib\cmake\Arrow\ArrowConfig.cmake"))) { + throw "prebuilt Arrow cmake config not found under $libDir" + } + Add-Content -Path $env:GITHUB_ENV -Value "CMAKE_PREFIX_PATH=$($libDir -replace '\\','/')" + Add-Content -Path $env:GITHUB_PATH -Value (Join-Path $libDir "bin") - name: Configure Iceberg shell: bash run: | diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index e631011942..8ff599db54 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -56,6 +56,23 @@ jobs: - name: Install dependencies shell: bash run: sudo apt-get update && sudo apt-get install -y libcurl4-openssl-dev + # Provide a prebuilt Arrow/Parquet from conda-forge so the build resolves it + # via find_package(Arrow CONFIG) instead of compiling Arrow from source. + - name: Set up conda (prebuilt Arrow) + uses: conda-incubator/setup-miniconda@8ee1f361103df19b6f8c8655fd3967a8ecb162d5 # v4.0.1 + with: + miniforge-version: latest + channels: conda-forge + channel-priority: strict + activate-environment: arrow + - name: Install prebuilt Arrow + Parquet + shell: bash -el {0} + run: | + mamba install -y "libarrow=24.0.*" "libparquet=24.0.*" + echo "CMAKE_PREFIX_PATH=${CONDA_PREFIX}" >> "$GITHUB_ENV" + echo "LD_LIBRARY_PATH=${LD_LIBRARY_PATH:-}:${CONDA_PREFIX}/lib" >> "$GITHUB_ENV" + # Test binaries are throwaway, so drop debug info to shrink objects + sccache. + echo "ICEBERG_EXTRA_CMAKE_ARGS=-DCMAKE_CXX_FLAGS_DEBUG=-g0 -DCMAKE_C_FLAGS_DEBUG=-g0" >> "$GITHUB_ENV" - name: Restore sccache cache uses: actions/cache/restore@27d5ce7f107fe9357f9df03efb73ab90386fccae # v5.0.5 with: @@ -101,6 +118,23 @@ jobs: uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 with: persist-credentials: false + # Provide a prebuilt Arrow/Parquet from conda-forge so the build resolves it + # via find_package(Arrow CONFIG) instead of compiling Arrow from source. + - name: Set up conda (prebuilt Arrow) + uses: conda-incubator/setup-miniconda@8ee1f361103df19b6f8c8655fd3967a8ecb162d5 # v4.0.1 + with: + miniforge-version: latest + channels: conda-forge + channel-priority: strict + activate-environment: arrow + - name: Install prebuilt Arrow + Parquet + shell: bash -el {0} + run: | + mamba install -y "libarrow=24.0.*" "libparquet=24.0.*" + echo "CMAKE_PREFIX_PATH=${CONDA_PREFIX}" >> "$GITHUB_ENV" + echo "DYLD_FALLBACK_LIBRARY_PATH=${DYLD_FALLBACK_LIBRARY_PATH:-}:${CONDA_PREFIX}/lib" >> "$GITHUB_ENV" + # Test binaries are throwaway, so drop debug info to shrink objects + sccache. + echo "ICEBERG_EXTRA_CMAKE_ARGS=-DCMAKE_CXX_FLAGS_DEBUG=-g0 -DCMAKE_C_FLAGS_DEBUG=-g0" >> "$GITHUB_ENV" - name: Restore sccache cache uses: actions/cache/restore@27d5ce7f107fe9357f9df03efb73ab90386fccae # v5.0.5 with: @@ -144,6 +178,33 @@ jobs: uses: ilammy/msvc-dev-cmd@0b201ec74fa43914dc39ae48a89fd1d8cb592756 # v1.13.0 with: arch: x64 + # Provide a prebuilt Arrow/Parquet from conda-forge so the build resolves it + # via find_package(Arrow CONFIG) instead of compiling Arrow from source. + - name: Set up conda (prebuilt Arrow) + uses: conda-incubator/setup-miniconda@8ee1f361103df19b6f8c8655fd3967a8ecb162d5 # v4.0.1 + with: + miniforge-version: latest + channels: conda-forge + channel-priority: strict + activate-environment: arrow + # Run in pwsh: setup-miniconda's env activation is unreliable in bash on the + # Windows runner. Expose the conda prefix through CMAKE_PREFIX_PATH so the + # standard find_package(Arrow CONFIG) resolves it; fail fast (with a listing) + # if the config is missing rather than silently building vendored Arrow. + - name: Install prebuilt Arrow + Parquet + shell: pwsh + run: | + $ErrorActionPreference = "Stop" + $libDir = Join-Path $env:CONDA "envs\arrow\Library" + & "$env:CONDA\condabin\conda.bat" install -n arrow -y -c conda-forge "libarrow=24.0.*" "libparquet=24.0.*" + if ($LASTEXITCODE -ne 0) { throw "conda install failed" } + if (-not (Test-Path (Join-Path $libDir "lib\cmake\Arrow\ArrowConfig.cmake"))) { + Write-Host "Arrow cmake config NOT FOUND under $libDir. *Config.cmake present:" + Get-ChildItem -Path $libDir -Recurse -Filter *Config.cmake -ErrorAction SilentlyContinue | Select-Object -First 60 | ForEach-Object { $_.FullName } + throw "prebuilt Arrow cmake config not found" + } + Add-Content -Path $env:GITHUB_ENV -Value "CMAKE_PREFIX_PATH=$($libDir -replace '\\','/')" + Add-Content -Path $env:GITHUB_PATH -Value (Join-Path $libDir "bin") - name: Cache vcpkg packages uses: actions/cache@27d5ce7f107fe9357f9df03efb73ab90386fccae # v5.0.5 id: vcpkg-cache diff --git a/src/iceberg/result.h b/src/iceberg/result.h index 8765f852a8..a2d93cf83e 100644 --- a/src/iceberg/result.h +++ b/src/iceberg/result.h @@ -142,3 +142,5 @@ template using ResultValueT = typename std::remove_cvref_t::value_type; } // namespace iceberg + +// touch: validate linter leg builds against prebuilt Arrow