diff --git a/.github/actions/save-sccache/action.yml b/.github/actions/save-sccache/action.yml new file mode 100644 index 0000000000..224547b04f --- /dev/null +++ b/.github/actions/save-sccache/action.yml @@ -0,0 +1,39 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +name: Save sccache +description: Show sccache stats and save the cache on main branch runs after setup-sccache. + +inputs: + key-prefix: + description: Cache key prefix without the run id or Windows compiler suffix. + required: true + +runs: + using: composite + steps: + - name: Show sccache stats + if: always() + shell: bash + run: sccache --show-stats + + - name: Save sccache cache + if: ${{ github.ref == 'refs/heads/main' }} + uses: actions/cache/save@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0 + with: + path: ${{ github.workspace }}/.sccache + key: ${{ inputs.key-prefix }}${{ env.SCCACHE_KEY_SUFFIX }}-${{ github.run_id }} diff --git a/.github/actions/sccache/resolve-msvc-key-suffix.ps1 b/.github/actions/sccache/resolve-msvc-key-suffix.ps1 new file mode 100644 index 0000000000..f0c0a69e84 --- /dev/null +++ b/.github/actions/sccache/resolve-msvc-key-suffix.ps1 @@ -0,0 +1,55 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +# Resolve the MSVC toolset version for use as an sccache cache key suffix. +# GitHub runner images roll out new cl.exe builds roughly weekly; keying the +# cache on the toolset version avoids silent misses after each bump. + +$PSNativeCommandUseErrorActionPreference = $false + +function Resolve-Suffix { + # Locate vswhere, which ships with every VS installation. + $vswhere = Join-Path ${env:ProgramFiles(x86)} 'Microsoft Visual Studio\Installer\vswhere.exe' + if (-not (Test-Path $vswhere)) { return $null } + + # Ask vswhere for the newest installation that includes the C++ toolset. + $installationPath = (& $vswhere -latest -products * ` + -requires Microsoft.VisualStudio.Component.VC.Tools.x86.x64 ` + -property installationPath | Select-Object -First 1) + if (-not $installationPath) { return $null } + + # Read the default toolset version (e.g. "14.44.35217"). + $versionFile = Join-Path $installationPath.Trim() 'VC\Auxiliary\Build\Microsoft.VCToolsVersion.default.txt' + if (-not (Test-Path $versionFile)) { return $null } + + $version = (Get-Content -Path $versionFile -Raw).Trim() + if ($version) { return "-$version" } + return $null +} + +$suffix = Resolve-Suffix +if (-not $suffix) { + # Degrade gracefully: a shared bucket is better than a failed build. + $suffix = '-unknown' + Write-Host "::warning::could not resolve MSVC toolset version for sccache key, using '$suffix'" +} + +# Export to both step outputs (for setup-sccache's restore key) and env (for +# save-sccache's save key later in the same job). +Add-Content -Path $env:GITHUB_OUTPUT -Value "suffix=$suffix" +Add-Content -Path $env:GITHUB_ENV -Value "SCCACHE_KEY_SUFFIX=$suffix" +Write-Host "Resolved SCCACHE_KEY_SUFFIX=$suffix" diff --git a/.github/actions/setup-sccache/action.yml b/.github/actions/setup-sccache/action.yml new file mode 100644 index 0000000000..eecf6eb66c --- /dev/null +++ b/.github/actions/setup-sccache/action.yml @@ -0,0 +1,57 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +name: Set up sccache +description: Configure sccache, restore its cache, and start the sccache action. + +inputs: + key-prefix: + description: Cache key prefix without the run id or Windows compiler suffix. + required: true + cache-size: + description: Maximum sccache cache size. + required: false + default: 2G + +runs: + using: composite + steps: + - name: Configure sccache environment + shell: bash + run: | + echo "SCCACHE_DIR=${{ github.workspace }}/.sccache" >> "$GITHUB_ENV" + echo "SCCACHE_CACHE_SIZE=${{ inputs.cache-size }}" >> "$GITHUB_ENV" + echo "SCCACHE_KEY_SUFFIX=" >> "$GITHUB_ENV" + + # Windows caches include the MSVC toolset version because GitHub runner image + # rollouts can change the compiler while keeping the same image label. + - name: Resolve MSVC toolset version for sccache key + if: ${{ runner.os == 'Windows' }} + id: msvc-key + shell: pwsh + run: '& "${{ github.action_path }}/../sccache/resolve-msvc-key-suffix.ps1"' + + - name: Restore sccache cache + uses: actions/cache/restore@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0 + with: + path: ${{ github.workspace }}/.sccache + key: ${{ inputs.key-prefix }}${{ steps.msvc-key.outputs.suffix }}-${{ github.run_id }} + restore-keys: | + ${{ inputs.key-prefix }}${{ steps.msvc-key.outputs.suffix }}- + + - name: Setup sccache + uses: mozilla-actions/sccache-action@9e7fa8a12102821edf02ca5dbea1acd0f89a2696 # v0.0.10 diff --git a/.github/dependabot.yml b/.github/dependabot.yml index fc74cd2992..325ea24247 100644 --- a/.github/dependabot.yml +++ b/.github/dependabot.yml @@ -26,3 +26,10 @@ updates: day: "sunday" cooldown: default-days: 7 + groups: + # The codeql-action init/autobuild/analyze steps must all run the + # same version - split PRs cause a version mismatch that fails the + # Analyze jobs. Group them so a single PR bumps all of them together. + codeql-action: + patterns: + - "github/codeql-action*" diff --git a/.github/workflows/asf-allowlist-check.yml b/.github/workflows/asf-allowlist-check.yml index a2a36d4bb0..d642d5c590 100644 --- a/.github/workflows/asf-allowlist-check.yml +++ b/.github/workflows/asf-allowlist-check.yml @@ -24,11 +24,11 @@ name: "ASF Allowlist Check" on: - pull_request: - types: [opened, synchronize, reopened, ready_for_review] push: branches: - main + pull_request: + types: [opened, synchronize, reopened, ready_for_review] permissions: contents: read diff --git a/.github/workflows/aws_test.yml b/.github/workflows/aws_test.yml index cb486034ba..65596464f0 100644 --- a/.github/workflows/aws_test.yml +++ b/.github/workflows/aws_test.yml @@ -20,8 +20,7 @@ name: AWS Tests on: push: branches: - - '**' - - '!dependabot/**' + - main tags: - '**' pull_request: @@ -74,8 +73,6 @@ jobs: AWS_DEFAULT_REGION: us-east-1 AWS_ENDPOINT_URL: http://127.0.0.1:9000 AWS_EC2_METADATA_DISABLED: "TRUE" - SCCACHE_DIR: ${{ github.workspace }}/.sccache - SCCACHE_CACHE_SIZE: "2G" steps: - name: Checkout iceberg-cpp uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 @@ -87,7 +84,7 @@ jobs: run: sudo apt-get update && sudo apt-get install -y libcurl4-openssl-dev libjitterentropy3-dev - name: Cache vcpkg packages if: ${{ startsWith(matrix.runs-on, 'ubuntu') && matrix.bundle_awssdk == 'OFF' }} - uses: actions/cache@27d5ce7f107fe9357f9df03efb73ab90386fccae # v5.0.5 + uses: actions/cache@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0 id: vcpkg-cache with: path: /usr/local/share/vcpkg/installed @@ -115,101 +112,16 @@ jobs: if: ${{ matrix.s3 == 'ON' }} shell: bash run: bash ci/scripts/start_minio.sh - - name: Restore sccache cache - uses: actions/cache/restore@27d5ce7f107fe9357f9df03efb73ab90386fccae # v5.0.5 + - name: Set up sccache + uses: ./.github/actions/setup-sccache with: - path: ${{ github.workspace }}/.sccache - key: sccache-aws-${{ matrix.runs-on }}-bundle${{ matrix.bundle_awssdk }}-s3${{ matrix.s3 }}-sigv4${{ matrix.sigv4 }}-${{ github.run_id }} - restore-keys: | - sccache-aws-${{ matrix.runs-on }}-bundle${{ matrix.bundle_awssdk }}-s3${{ matrix.s3 }}-sigv4${{ matrix.sigv4 }}- - - name: Setup sccache - uses: mozilla-actions/sccache-action@9e7fa8a12102821edf02ca5dbea1acd0f89a2696 # v0.0.10 + key-prefix: sccache-aws-${{ matrix.runs-on }}-bundle${{ matrix.bundle_awssdk }}-s3${{ matrix.s3 }}-sigv4${{ matrix.sigv4 }} - name: Build and test Iceberg shell: bash env: CMAKE_TOOLCHAIN_FILE: ${{ startsWith(matrix.runs-on, 'ubuntu') && matrix.bundle_awssdk == 'OFF' && '/usr/local/share/vcpkg/scripts/buildsystems/vcpkg.cmake' || '' }} run: ci/scripts/build_iceberg.sh "$(pwd)" OFF ON ${{ matrix.s3 }} ${{ matrix.sigv4 }} ${{ matrix.bundle_awssdk }} - - name: Show sccache stats - shell: bash - run: sccache --show-stats - - name: Save sccache cache - if: github.ref == 'refs/heads/main' - uses: actions/cache/save@27d5ce7f107fe9357f9df03efb73ab90386fccae # v5.0.5 - with: - path: ${{ github.workspace }}/.sccache - key: sccache-aws-${{ matrix.runs-on }}-bundle${{ matrix.bundle_awssdk }}-s3${{ matrix.s3 }}-sigv4${{ matrix.sigv4 }}-${{ github.run_id }} - - # Exercise the Meson build with SigV4 enabled (resolves aws-cpp-sdk-core via - # its CMake config, not pkg-config whose Cflags force -std=c++11). - meson-sigv4: - if: ${{ github.event_name != 'pull_request' || github.event.pull_request.draft == false }} - name: Meson SigV4 (AMD64 Ubuntu 26.04) - runs-on: ubuntu-26.04 - timeout-minutes: 45 - env: - SCCACHE_DIR: ${{ github.workspace }}/.sccache - SCCACHE_CACHE_SIZE: "2G" - steps: - - uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0 - with: - python-version: '3.x' - - name: Checkout iceberg-cpp - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 - with: - persist-credentials: false - - name: Install build dependencies - shell: bash - run: | - sudo apt-get update && sudo apt-get install -y libcurl4-openssl-dev libjitterentropy3-dev - python3 -m pip install --upgrade pip - python3 -m pip install -r requirements.txt - - name: Cache vcpkg packages - uses: actions/cache@27d5ce7f107fe9357f9df03efb73ab90386fccae # v5.0.5 - id: vcpkg-cache - with: - path: /usr/local/share/vcpkg/installed - key: vcpkg-x64-linux-aws-sdk-cpp-core-${{ hashFiles('.github/workflows/aws_test.yml') }} - - name: Install AWS SDK via vcpkg - if: ${{ steps.vcpkg-cache.outputs.cache-hit != 'true' }} - shell: bash - # Retry to ride out transient GitHub/mirror download failures (504s). - run: | - for attempt in 1 2 3; do - if vcpkg install aws-sdk-cpp[core]:x64-linux; then - exit 0 - fi - echo "::warning::vcpkg install failed (attempt ${attempt}/3), retrying in 30s" - sleep 30 - done - echo "::error::vcpkg install failed after 3 attempts" - exit 1 - - name: Set Ubuntu Compilers - # Wrap the compiler with sccache: Meson uses an explicit CC/CXX verbatim, - # so the launcher must be prepended here to route compiles through sccache. - run: | - echo "CC=sccache gcc-14" >> $GITHUB_ENV - echo "CXX=sccache g++-14" >> $GITHUB_ENV - - name: Restore sccache cache - uses: actions/cache/restore@27d5ce7f107fe9357f9df03efb73ab90386fccae # v5.0.5 - with: - path: ${{ github.workspace }}/.sccache - key: sccache-meson-sigv4-${{ github.run_id }} - restore-keys: | - sccache-meson-sigv4- - - name: Setup sccache - uses: mozilla-actions/sccache-action@9e7fa8a12102821edf02ca5dbea1acd0f89a2696 # v0.0.10 - - name: Build and test Iceberg - shell: bash - env: - CMAKE_PREFIX_PATH: /usr/local/share/vcpkg/installed/x64-linux - run: | - meson setup builddir -Dsigv4=enabled - meson compile -C builddir - sccache --show-stats - meson test -C builddir --timeout-multiplier 0 --print-errorlogs - - name: Save sccache cache - if: github.ref == 'refs/heads/main' - uses: actions/cache/save@27d5ce7f107fe9357f9df03efb73ab90386fccae # v5.0.5 + - name: Save sccache + uses: ./.github/actions/save-sccache with: - path: ${{ github.workspace }}/.sccache - key: sccache-meson-sigv4-${{ github.run_id }} + key-prefix: sccache-aws-${{ matrix.runs-on }}-bundle${{ matrix.bundle_awssdk }}-s3${{ matrix.s3 }}-sigv4${{ matrix.sigv4 }} diff --git a/.github/workflows/codeql.yml b/.github/workflows/codeql.yml index 0060e8ffc1..8220e01ea7 100644 --- a/.github/workflows/codeql.yml +++ b/.github/workflows/codeql.yml @@ -21,10 +21,12 @@ name: "CodeQL" on: push: - branches: [ "main" ] + branches: + - main pull_request: types: [opened, synchronize, reopened, ready_for_review] - branches: [ "main" ] + branches: + - main schedule: - cron: '16 4 * * 1' @@ -52,11 +54,11 @@ jobs: persist-credentials: false - name: Initialize CodeQL - uses: github/codeql-action/init@8aad20d150bbac5944a9f9d289da16a4b0d87c1e # v4.36.2 + uses: github/codeql-action/init@54f647b7e1bb85c95cddabcd46b0c578ec92bc1a # v4.36.3 with: languages: actions - name: Perform CodeQL Analysis - uses: github/codeql-action/analyze@8aad20d150bbac5944a9f9d289da16a4b0d87c1e # v4.36.2 + uses: github/codeql-action/analyze@54f647b7e1bb85c95cddabcd46b0c578ec92bc1a # v4.36.3 with: category: "/language:actions" diff --git a/.github/workflows/cpp-linter.yml b/.github/workflows/cpp-linter.yml index 304d1a58b2..c1381f136c 100644 --- a/.github/workflows/cpp-linter.yml +++ b/.github/workflows/cpp-linter.yml @@ -27,12 +27,12 @@ on: - 'cmake_modules/**' pull_request: types: [opened, synchronize, reopened, ready_for_review] + branches: + - main paths-ignore: - '.github/**' - 'ci/**' - 'cmake_modules/**' - branches: - - main concurrency: group: ${{ github.repository }}-${{ github.head_ref || github.sha }}-${{ github.workflow }} @@ -45,9 +45,6 @@ jobs: permissions: contents: read pull-requests: write - env: - SCCACHE_DIR: ${{ github.workspace }}/.sccache - SCCACHE_CACHE_SIZE: "2G" steps: - name: Checkout iceberg-cpp uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 @@ -58,15 +55,10 @@ jobs: run: | sudo apt-get update sudo apt-get install -y libcurl4-openssl-dev libsqlite3-dev libpq-dev default-libmysqlclient-dev - - name: Restore sccache cache - uses: actions/cache/restore@27d5ce7f107fe9357f9df03efb73ab90386fccae # v5.0.5 + - name: Set up sccache + uses: ./.github/actions/setup-sccache with: - path: ${{ github.workspace }}/.sccache - key: sccache-cpp-linter-ubuntu-${{ github.run_id }} - restore-keys: | - sccache-cpp-linter-ubuntu- - - name: Setup sccache - uses: mozilla-actions/sccache-action@9e7fa8a12102821edf02ca5dbea1acd0f89a2696 # v0.0.10 + key-prefix: sccache-cpp-linter-ubuntu - name: Run build env: CC: gcc-14 @@ -77,19 +69,15 @@ jobs: -DCMAKE_C_COMPILER_LAUNCHER=sccache \ -DCMAKE_CXX_COMPILER_LAUNCHER=sccache \ -DICEBERG_BUILD_SQL_CATALOG=ON \ + -DICEBERG_BUILD_BENCHMARKS=ON \ -DICEBERG_SQL_SQLITE=ON \ -DICEBERG_SQL_POSTGRESQL=ON \ -DICEBERG_SQL_MYSQL=ON cmake --build . - - name: Show sccache stats - shell: bash - run: sccache --show-stats - - name: Save sccache cache - if: github.ref == 'refs/heads/main' - uses: actions/cache/save@27d5ce7f107fe9357f9df03efb73ab90386fccae # v5.0.5 + - name: Save sccache + uses: ./.github/actions/save-sccache with: - path: ${{ github.workspace }}/.sccache - key: sccache-cpp-linter-ubuntu-${{ github.run_id }} + key-prefix: sccache-cpp-linter-ubuntu - uses: cpp-linter/cpp-linter-action@0f6d1b8d7e38b584cbee606eb23d850c217d54f8 # v2.15.1 id: linter if: github.event_name == 'pull_request' diff --git a/.github/workflows/docs.yml b/.github/workflows/docs.yml index 729d574c5c..083461083f 100644 --- a/.github/workflows/docs.yml +++ b/.github/workflows/docs.yml @@ -42,7 +42,7 @@ jobs: fetch-depth: 1 persist-credentials: false - - uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0 + - uses: actions/setup-python@ece7cb06caefa5fff74198d8649806c4678c61a1 # v6.3.0 with: python-version: '3.x' diff --git a/.github/workflows/pre-commit.yml b/.github/workflows/pre-commit.yml index 417f1d5dd0..209ac6918b 100644 --- a/.github/workflows/pre-commit.yml +++ b/.github/workflows/pre-commit.yml @@ -18,12 +18,12 @@ name: pre-commit on: - pull_request: - types: [opened, synchronize, reopened, ready_for_review] push: branches: - '**' - '!dependabot/**' + pull_request: + types: [opened, synchronize, reopened, ready_for_review] concurrency: group: ${{ github.repository }}-${{ github.head_ref || github.sha }}-${{ github.workflow }} @@ -40,5 +40,5 @@ jobs: - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 with: persist-credentials: false - - uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0 + - uses: actions/setup-python@ece7cb06caefa5fff74198d8649806c4678c61a1 # v6.3.0 - uses: pre-commit/action@2c7b3805fd2a0fd8c1884dcaebf91fc102a13ecd # v3.0.1 diff --git a/.github/workflows/sanitizer_test.yml b/.github/workflows/sanitizer_test.yml index 48eb578466..13bad0a7af 100644 --- a/.github/workflows/sanitizer_test.yml +++ b/.github/workflows/sanitizer_test.yml @@ -20,8 +20,7 @@ name: ASAN and UBSAN Tests on: push: branches: - - '**' - - '!dependabot/**' + - main tags: - '**' pull_request: @@ -39,9 +38,6 @@ jobs: if: ${{ github.event_name != 'pull_request' || github.event.pull_request.draft == false }} name: "ASAN and UBSAN Tests" runs-on: ubuntu-26.04 - env: - SCCACHE_DIR: ${{ github.workspace }}/.sccache - SCCACHE_CACHE_SIZE: "2G" steps: - name: Checkout iceberg-cpp uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 @@ -50,15 +46,10 @@ jobs: - name: Install dependencies shell: bash run: sudo apt-get update && sudo apt-get install -y libcurl4-openssl-dev - - name: Restore sccache cache - uses: actions/cache/restore@27d5ce7f107fe9357f9df03efb73ab90386fccae # v5.0.5 + - name: Set up sccache + uses: ./.github/actions/setup-sccache with: - path: ${{ github.workspace }}/.sccache - key: sccache-sanitizer-ubuntu-${{ github.run_id }} - restore-keys: | - sccache-sanitizer-ubuntu- - - name: Setup sccache - uses: mozilla-actions/sccache-action@9e7fa8a12102821edf02ca5dbea1acd0f89a2696 # v0.0.10 + key-prefix: sccache-sanitizer-ubuntu - name: Configure and Build with ASAN & UBSAN env: CC: gcc-14 @@ -68,15 +59,10 @@ jobs: cmake .. -G Ninja -DCMAKE_BUILD_TYPE=Debug -DICEBERG_ENABLE_ASAN=ON -DICEBERG_ENABLE_UBSAN=ON \ -DCMAKE_C_COMPILER_LAUNCHER=sccache -DCMAKE_CXX_COMPILER_LAUNCHER=sccache cmake --build . --verbose - - name: Show sccache stats - shell: bash - run: sccache --show-stats - - name: Save sccache cache - if: github.ref == 'refs/heads/main' - uses: actions/cache/save@27d5ce7f107fe9357f9df03efb73ab90386fccae # v5.0.5 + - name: Save sccache + uses: ./.github/actions/save-sccache with: - path: ${{ github.workspace }}/.sccache - key: sccache-sanitizer-ubuntu-${{ github.run_id }} + key-prefix: sccache-sanitizer-ubuntu - name: Run Tests working-directory: build env: diff --git a/.github/workflows/sql_catalog_test.yml b/.github/workflows/sql_catalog_test.yml index 905b4a06c4..e9a5824ac5 100644 --- a/.github/workflows/sql_catalog_test.yml +++ b/.github/workflows/sql_catalog_test.yml @@ -20,8 +20,7 @@ name: SQL Catalog Tests on: push: branches: - - '**' - - '!dependabot/**' + - main tags: - '**' pull_request: @@ -59,11 +58,8 @@ jobs: cmake_extra_args: "" - title: AMD64 Windows 2025 runs-on: windows-2025 - cmake_build_type: Release + cmake_build_type: Debug cmake_extra_args: -DCMAKE_TOOLCHAIN_FILE=C:/vcpkg/scripts/buildsystems/vcpkg.cmake - env: - SCCACHE_DIR: ${{ github.workspace }}/.sccache - SCCACHE_CACHE_SIZE: "2G" steps: - name: Checkout iceberg-cpp uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 @@ -74,26 +70,6 @@ jobs: uses: ilammy/msvc-dev-cmd@0b201ec74fa43914dc39ae48a89fd1d8cb592756 # v1.13.0 with: arch: x64 - # GitHub patches the Windows image roughly weekly via gradual rollouts, so - # back-to-back runs can hit different cl.exe builds. sccache keys on the compiler, - # so a shared key would miss after each bump; the version keeps caches separate. - # Non-Windows legs get an empty suffix, leaving their keys unchanged. - - name: Resolve MSVC version for sccache key - if: ${{ startsWith(matrix.runs-on, 'windows') }} - shell: pwsh - run: | - $PSNativeCommandUseErrorActionPreference = $false - $banner = (cl.exe 2>&1 | Out-String) - if ($banner -match 'Version ([\d.]+)') { - $suffix = "-$($Matches[1])" - } else { - # The key is only a cache hint, so degrade to a shared bucket rather than - # fail the build; the warning flags that the parse needs fixing. - $suffix = '-unknown' - Write-Host "::warning::could not parse cl.exe version for sccache key, using '$suffix'; banner was: $banner" - } - Add-Content -Path $env:GITHUB_ENV -Value "SCCACHE_KEY_SUFFIX=$suffix" - Write-Host "SCCACHE_KEY_SUFFIX=$suffix" - name: Install dependencies on Ubuntu if: ${{ startsWith(matrix.runs-on, 'ubuntu') }} shell: bash @@ -105,7 +81,7 @@ jobs: echo "CXX=${{ matrix.CXX }}" >> $GITHUB_ENV - name: Cache vcpkg packages if: ${{ startsWith(matrix.runs-on, 'windows') }} - uses: actions/cache@27d5ce7f107fe9357f9df03efb73ab90386fccae # v5.0.5 + uses: actions/cache@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0 id: vcpkg-cache with: path: C:/vcpkg/installed @@ -115,15 +91,10 @@ jobs: shell: pwsh run: | vcpkg install zlib:x64-windows nlohmann-json:x64-windows nanoarrow:x64-windows roaring:x64-windows sqlite3:x64-windows - - name: Restore sccache cache - uses: actions/cache/restore@27d5ce7f107fe9357f9df03efb73ab90386fccae # v5.0.5 + - name: Set up sccache + uses: ./.github/actions/setup-sccache with: - path: ${{ github.workspace }}/.sccache - key: sccache-sqlcatalog-${{ matrix.runs-on }}${{ env.SCCACHE_KEY_SUFFIX }}-${{ github.run_id }} - restore-keys: | - sccache-sqlcatalog-${{ matrix.runs-on }}${{ env.SCCACHE_KEY_SUFFIX }}- - - name: Setup sccache - uses: mozilla-actions/sccache-action@9e7fa8a12102821edf02ca5dbea1acd0f89a2696 # v0.0.10 + key-prefix: sccache-sqlcatalog-${{ matrix.runs-on }} - name: Configure Iceberg shell: bash run: | @@ -141,15 +112,10 @@ jobs: - name: Build SQL catalog tests shell: bash run: cmake --build build --target sql_catalog_test - - name: Show sccache stats - shell: bash - run: sccache --show-stats - - name: Save sccache cache - if: github.ref == 'refs/heads/main' - uses: actions/cache/save@27d5ce7f107fe9357f9df03efb73ab90386fccae # v5.0.5 + - name: Save sccache + uses: ./.github/actions/save-sccache with: - path: ${{ github.workspace }}/.sccache - key: sccache-sqlcatalog-${{ matrix.runs-on }}${{ env.SCCACHE_KEY_SUFFIX }}-${{ github.run_id }} + key-prefix: sccache-sqlcatalog-${{ matrix.runs-on }} - name: Run SQL catalog tests shell: bash run: ctest --test-dir build -R '^sql_catalog_test$' --output-on-failure -C ${{ matrix.cmake_build_type }} diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 23fcc2d1c3..8c53af6793 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -20,8 +20,7 @@ name: Test on: push: branches: - - '**' - - '!dependabot/**' + - main tags: - '**' pull_request: @@ -40,14 +39,15 @@ env: jobs: ubuntu: if: ${{ github.event_name != 'pull_request' || github.event.pull_request.draft == false }} - name: AMD64 Ubuntu 26.04 + name: AMD64 Ubuntu 26.04 (${{ matrix.cmake_build_type }}) runs-on: ubuntu-26.04 - timeout-minutes: 30 + timeout-minutes: 60 strategy: fail-fast: false - env: - SCCACHE_DIR: ${{ github.workspace }}/.sccache - SCCACHE_CACHE_SIZE: "2G" + matrix: + cmake_build_type: + - Debug + - RelWithDebInfo steps: - name: Checkout iceberg-cpp uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 @@ -56,30 +56,20 @@ jobs: - name: Install dependencies shell: bash run: sudo apt-get update && sudo apt-get install -y libcurl4-openssl-dev - - name: Restore sccache cache - uses: actions/cache/restore@27d5ce7f107fe9357f9df03efb73ab90386fccae # v5.0.5 + - name: Set up sccache + uses: ./.github/actions/setup-sccache with: - path: ${{ github.workspace }}/.sccache - key: sccache-test-ubuntu-${{ github.run_id }} - restore-keys: | - sccache-test-ubuntu- - - name: Setup sccache - uses: mozilla-actions/sccache-action@9e7fa8a12102821edf02ca5dbea1acd0f89a2696 # v0.0.10 + key-prefix: sccache-test-ubuntu-${{ matrix.cmake_build_type }} - name: Build Iceberg shell: bash env: CC: gcc-14 CXX: g++-14 - run: ci/scripts/build_iceberg.sh $(pwd) ON ON - - name: Show sccache stats - shell: bash - run: sccache --show-stats - - name: Save sccache cache - if: github.ref == 'refs/heads/main' - uses: actions/cache/save@27d5ce7f107fe9357f9df03efb73ab90386fccae # v5.0.5 + run: ci/scripts/build_iceberg.sh $(pwd) ON ON OFF OFF ON ${{ matrix.cmake_build_type }} + - name: Save sccache + uses: ./.github/actions/save-sccache with: - path: ${{ github.workspace }}/.sccache - key: sccache-test-ubuntu-${{ github.run_id }} + key-prefix: sccache-test-ubuntu-${{ matrix.cmake_build_type }} - name: Build Example shell: bash env: @@ -93,35 +83,22 @@ jobs: timeout-minutes: 30 strategy: fail-fast: false - env: - SCCACHE_DIR: ${{ github.workspace }}/.sccache - SCCACHE_CACHE_SIZE: "2G" steps: - name: Checkout iceberg-cpp uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 with: persist-credentials: false - - name: Restore sccache cache - uses: actions/cache/restore@27d5ce7f107fe9357f9df03efb73ab90386fccae # v5.0.5 + - name: Set up sccache + uses: ./.github/actions/setup-sccache with: - path: ${{ github.workspace }}/.sccache - key: sccache-test-macos-${{ github.run_id }} - restore-keys: | - sccache-test-macos- - - name: Setup sccache - uses: mozilla-actions/sccache-action@9e7fa8a12102821edf02ca5dbea1acd0f89a2696 # v0.0.10 + key-prefix: sccache-test-macos - name: Build Iceberg shell: bash run: ci/scripts/build_iceberg.sh $(pwd) OFF ON - - name: Show sccache stats - shell: bash - run: sccache --show-stats - - name: Save sccache cache - if: github.ref == 'refs/heads/main' - uses: actions/cache/save@27d5ce7f107fe9357f9df03efb73ab90386fccae # v5.0.5 + - name: Save sccache + uses: ./.github/actions/save-sccache with: - path: ${{ github.workspace }}/.sccache - key: sccache-test-macos-${{ github.run_id }} + key-prefix: sccache-test-macos - name: Build Example shell: bash run: ci/scripts/build_example.sh $(pwd)/example @@ -132,9 +109,6 @@ jobs: timeout-minutes: 60 strategy: fail-fast: false - env: - SCCACHE_DIR: ${{ github.workspace }}/.sccache - SCCACHE_CACHE_SIZE: "2G" steps: - name: Checkout iceberg-cpp uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 @@ -144,26 +118,8 @@ jobs: uses: ilammy/msvc-dev-cmd@0b201ec74fa43914dc39ae48a89fd1d8cb592756 # v1.13.0 with: arch: x64 - # GitHub patches the Windows image roughly weekly via gradual rollouts, so - # back-to-back runs can hit different cl.exe builds. sccache keys on the compiler, - # so a shared key would miss after each bump; the version keeps caches separate. - - name: Resolve MSVC version for sccache key - shell: pwsh - run: | - $PSNativeCommandUseErrorActionPreference = $false - $banner = (cl.exe 2>&1 | Out-String) - if ($banner -match 'Version ([\d.]+)') { - $msvc = $Matches[1] - } else { - # The key is only a cache hint, so degrade to a shared bucket rather than - # fail the build; the warning flags that the parse needs fixing. - $msvc = 'unknown' - Write-Host "::warning::could not parse cl.exe version for sccache key, using '$msvc'; banner was: $banner" - } - Add-Content -Path $env:GITHUB_ENV -Value "MSVC_VER=$msvc" - Write-Host "Resolved MSVC_VER=$msvc" - name: Cache vcpkg packages - uses: actions/cache@27d5ce7f107fe9357f9df03efb73ab90386fccae # v5.0.5 + uses: actions/cache@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0 id: vcpkg-cache with: path: C:/vcpkg/installed @@ -173,28 +129,20 @@ jobs: shell: pwsh run: | vcpkg install zlib:x64-windows nlohmann-json:x64-windows nanoarrow:x64-windows roaring:x64-windows cpr:x64-windows - - name: Restore sccache cache - uses: actions/cache/restore@27d5ce7f107fe9357f9df03efb73ab90386fccae # v5.0.5 + - name: Set up sccache + uses: ./.github/actions/setup-sccache with: - path: ${{ github.workspace }}/.sccache - key: sccache-test-windows-${{ env.MSVC_VER }}-${{ github.run_id }} - restore-keys: | - sccache-test-windows-${{ env.MSVC_VER }}- - - name: Setup sccache - uses: mozilla-actions/sccache-action@9e7fa8a12102821edf02ca5dbea1acd0f89a2696 # v0.0.10 + key-prefix: sccache-test-windows - name: Build Iceberg shell: pwsh run: | $ErrorActionPreference = "Stop" bash -lc 'ci/scripts/build_iceberg.sh $(pwd) OFF ON' if ($LASTEXITCODE -ne 0) { exit $LASTEXITCODE } - sccache --show-stats - - name: Save sccache cache - if: github.ref == 'refs/heads/main' - uses: actions/cache/save@27d5ce7f107fe9357f9df03efb73ab90386fccae # v5.0.5 + - name: Save sccache + uses: ./.github/actions/save-sccache with: - path: ${{ github.workspace }}/.sccache - key: sccache-test-windows-${{ env.MSVC_VER }}-${{ github.run_id }} + key-prefix: sccache-test-windows - name: Build Example shell: pwsh run: | @@ -205,9 +153,6 @@ jobs: name: Meson - ${{ matrix.title }} runs-on: ${{ matrix.runs-on }} timeout-minutes: 30 - env: - SCCACHE_DIR: ${{ github.workspace }}/.sccache - SCCACHE_CACHE_SIZE: "2G" strategy: max-parallel: 15 fail-fast: false @@ -224,7 +169,7 @@ jobs: - title: AArch64 macOS 26 runs-on: macos-26 steps: - - uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0 + - uses: actions/setup-python@ece7cb06caefa5fff74198d8649806c4678c61a1 # v6.3.0 with: python-version: '3.x' - name: Checkout iceberg-cpp @@ -243,27 +188,18 @@ jobs: run: | echo "CC=sccache ${{ matrix.CC }}" >> $GITHUB_ENV echo "CXX=sccache ${{ matrix.CXX }}" >> $GITHUB_ENV - - name: Restore sccache cache - uses: actions/cache/restore@27d5ce7f107fe9357f9df03efb73ab90386fccae # v5.0.5 + - name: Set up sccache + uses: ./.github/actions/setup-sccache with: - path: ${{ github.workspace }}/.sccache - key: sccache-meson-${{ matrix.runs-on }}-${{ github.run_id }} - restore-keys: | - sccache-meson-${{ matrix.runs-on }}- - - name: Setup sccache - uses: mozilla-actions/sccache-action@9e7fa8a12102821edf02ca5dbea1acd0f89a2696 # v0.0.10 + key-prefix: sccache-meson-${{ matrix.runs-on }} - name: Build Iceberg run: | meson setup builddir ${{ matrix.meson-setup-args || '' }} meson compile -C builddir - - name: Show sccache stats - run: sccache --show-stats - - name: Save sccache cache - if: github.ref == 'refs/heads/main' - uses: actions/cache/save@27d5ce7f107fe9357f9df03efb73ab90386fccae # v5.0.5 + - name: Save sccache + uses: ./.github/actions/save-sccache with: - path: ${{ github.workspace }}/.sccache - key: sccache-meson-${{ matrix.runs-on }}-${{ github.run_id }} + key-prefix: sccache-meson-${{ matrix.runs-on }} - name: Test Iceberg run: | meson test -C builddir --timeout-multiplier 0 --print-errorlogs diff --git a/.github/workflows/zizmor.yml b/.github/workflows/zizmor.yml index 9b57dad34e..8201964700 100644 --- a/.github/workflows/zizmor.yml +++ b/.github/workflows/zizmor.yml @@ -21,10 +21,10 @@ name: GitHub Actions Security Analysis with zizmor 🌈 on: push: - branches: ["main"] + branches: + - main pull_request: types: [opened, synchronize, reopened, ready_for_review] - branches: ["**"] concurrency: group: ${{ github.repository }}-${{ github.head_ref || github.sha }}-${{ github.workflow }} diff --git a/CMakeLists.txt b/CMakeLists.txt index eb13912388..7cbd5e55e8 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -28,20 +28,18 @@ project(Iceberg DESCRIPTION "Iceberg C++ Project" LANGUAGES CXX) -set(ICEBERG_VERSION_SUFFIX "-SNAPSHOT") -set(ICEBERG_VERSION_STRING "${PROJECT_VERSION}${ICEBERG_VERSION_SUFFIX}") -configure_file("${CMAKE_CURRENT_SOURCE_DIR}/src/iceberg/version.h.in" - "${CMAKE_BINARY_DIR}/src/iceberg/version.h") +include(IcebergBuildUtils) +iceberg_configure_version_header("${CMAKE_CURRENT_SOURCE_DIR}/src/iceberg/version.h.in" + "${CMAKE_BINARY_DIR}/src/iceberg/version.h") set(CMAKE_CXX_STANDARD 23) set(CMAKE_CXX_STANDARD_REQUIRED ON) set(CMAKE_CXX_EXTENSIONS OFF) set(CMAKE_EXPORT_COMPILE_COMMANDS ON) -set(CMAKE_COMPILE_WARNING_AS_ERROR ON) - option(ICEBERG_BUILD_STATIC "Build static library" ON) option(ICEBERG_BUILD_SHARED "Build shared library" OFF) option(ICEBERG_BUILD_TESTS "Build tests" ON) +option(ICEBERG_BUILD_BENCHMARKS "Build benchmarks" OFF) option(ICEBERG_BUILD_BUNDLE "Build the battery included library" ON) option(ICEBERG_BUILD_REST "Build rest catalog client" ON) option(ICEBERG_BUILD_REST_INTEGRATION_TESTS "Build rest catalog integration tests" OFF) @@ -80,7 +78,6 @@ if(ICEBERG_BUILD_REST_INTEGRATION_TESTS AND WIN32) endif() include(CMakeParseArguments) -include(IcebergBuildUtils) include(IcebergSanitizer) include(IcebergSccache) include(IcebergThirdpartyToolchain) diff --git a/LICENSE b/LICENSE index 374b7fc58f..8d8d5ff4e6 100644 --- a/LICENSE +++ b/LICENSE @@ -228,3 +228,95 @@ Home page: https://arrow.apache.org/ License: https://www.apache.org/licenses/LICENSE-2.0 -------------------------------------------------------------------------------- + +This product bundles utf8proc, which is available under the MIT License: + +utf8proc is a software package originally developed by Jan Behrens and the rest +of the Public Software Group, now maintained by the Julia-language developers. +All new work on the utf8proc library is licensed under the MIT "expat" license: + +Copyright (c) 2014-2021 by Steven G. Johnson, Jiahao Chen, Tony Kelman, Jonas +Fonseca, and other contributors listed in the git history. + +Permission is hereby granted, free of charge, to any person obtaining a +copy of this software and associated documentation files (the "Software"), +to deal in the Software without restriction, including without limitation +the rights to use, copy, modify, merge, publish, distribute, sublicense, +and/or sell copies of the Software, and to permit persons to whom the +Software is furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in +all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +DEALINGS IN THE SOFTWARE. + +The original utf8proc is licensed under the same MIT "expat" license: + +Copyright (c) 2009, 2013 Public Software Group e. V., Berlin, Germany + +Permission is hereby granted, free of charge, to any person obtaining a +copy of this software and associated documentation files (the "Software"), +to deal in the Software without restriction, including without limitation +the rights to use, copy, modify, merge, publish, distribute, sublicense, +and/or sell copies of the Software, and to permit persons to whom the +Software is furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in +all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +DEALINGS IN THE SOFTWARE. + +utf8proc also contains data derived from the Unicode data files. The following +license applies to that data: + +COPYRIGHT AND PERMISSION NOTICE + +Copyright (c) 1991-2007 Unicode, Inc. All rights reserved. Distributed under +the Terms of Use in http://www.unicode.org/copyright.html. + +Permission is hereby granted, free of charge, to any person obtaining a +copy of the Unicode data files and any associated documentation (the "Data +Files") or Unicode software and any associated documentation (the +"Software") to deal in the Data Files or Software without restriction, +including without limitation the rights to use, copy, modify, merge, +publish, distribute, and/or sell copies of the Data Files or Software, and +to permit persons to whom the Data Files or Software are furnished to do +so, provided that (a) the above copyright notice(s) and this permission +notice appear with all copies of the Data Files or Software, (b) both the +above copyright notice(s) and this permission notice appear in associated +documentation, and (c) there is clear notice in each modified Data File or +in the Software as well as in the documentation associated with the Data +File(s) or Software that the data or software has been modified. + +THE DATA FILES AND SOFTWARE ARE PROVIDED "AS IS", WITHOUT WARRANTY OF ANY +KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF +MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT OF +THIRD PARTY RIGHTS. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR HOLDERS +INCLUDED IN THIS NOTICE BE LIABLE FOR ANY CLAIM, OR ANY SPECIAL INDIRECT OR +CONSEQUENTIAL DAMAGES, OR ANY DAMAGES WHATSOEVER RESULTING FROM LOSS OF +USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR OTHER +TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR +PERFORMANCE OF THE DATA FILES OR SOFTWARE. + +Except as contained in this notice, the name of a copyright holder shall +not be used in advertising or otherwise to promote the sale, use or other +dealings in these Data Files or Software without prior written +authorization of the copyright holder. + +Unicode and the Unicode logo are trademarks of Unicode, Inc., and may be +registered in some jurisdictions. All other trademarks and registered +trademarks mentioned herein are the property of their respective owners. + +-------------------------------------------------------------------------------- diff --git a/Makefile b/Makefile index 1000695453..99ca5bbaa3 100644 --- a/Makefile +++ b/Makefile @@ -33,6 +33,7 @@ install-deps: build-api-docs: cd mkdocs && \ + rm -rf docs/api && \ mkdir -p docs/api && \ doxygen Doxyfile && \ echo "Doxygen output created in docs/api/" diff --git a/README.md b/README.md index 7c9a343ec0..1318933d9a 100644 --- a/README.md +++ b/README.md @@ -151,6 +151,7 @@ If you experience network issues when downloading dependencies, you can customiz - `ICEBERG_AVRO_GIT_URL`: Apache Avro git repository URL - `ICEBERG_NANOARROW_URL`: Nanoarrow tarball URL - `ICEBERG_CROARING_URL`: CRoaring tarball URL +- `ICEBERG_UTF8PROC_URL`: utf8proc tarball URL - `ICEBERG_NLOHMANN_JSON_URL`: nlohmann-json tarball URL - `ICEBERG_SPDLOG_URL`: spdlog tarball URL - `ICEBERG_CPR_URL`: cpr tarball URL diff --git a/ci/scripts/build_example.sh b/ci/scripts/build_example.sh index 41714cf32c..7ccdf4a8f7 100755 --- a/ci/scripts/build_example.sh +++ b/ci/scripts/build_example.sh @@ -23,7 +23,18 @@ source_dir=${1} build_dir=${1}/build run_example=${ICEBERG_RUN_EXAMPLE:-OFF} -rm -rf "${build_dir}" +# Clean up before configuring. If Windows still holds a just-built exe/dll +# after the retries, let mkdir fail rather than reuse a half-deleted tree. +for attempt in 1 2 3; do + if rm -rf "${build_dir}"; then + break + fi + if [[ "${attempt}" != "3" ]]; then + sleep 2 + else + echo "Failed to remove build directory after 3 attempts: ${build_dir}" >&2 + fi +done mkdir "${build_dir}" pushd ${build_dir} @@ -38,26 +49,19 @@ CMAKE_ARGS=( if is_windows; then CMAKE_ARGS+=("-DCMAKE_TOOLCHAIN_FILE=C:/vcpkg/scripts/buildsystems/vcpkg.cmake") - CMAKE_ARGS+=("-DCMAKE_BUILD_TYPE=Release") -else - CMAKE_ARGS+=("-DCMAKE_BUILD_TYPE=Debug") fi +build_type="${ICEBERG_BUILD_TYPE:-Debug}" +CMAKE_ARGS+=("-DCMAKE_BUILD_TYPE=${build_type}") + cmake "${CMAKE_ARGS[@]}" ${source_dir} -if is_windows; then - cmake --build . --config Release - if [[ "${run_example}" == "ON" ]]; then - if [[ -x ./demo_example.exe ]]; then - ./demo_example.exe +cmake --build . +if [[ "${run_example}" == "ON" ]]; then + if is_windows; then + ./demo_example.exe else - ./Release/demo_example.exe + ./demo_example fi - fi -else - cmake --build . - if [[ "${run_example}" == "ON" ]]; then - ./demo_example - fi fi popd diff --git a/ci/scripts/build_iceberg.sh b/ci/scripts/build_iceberg.sh index 6af0802f6b..6a7dc607a2 100755 --- a/ci/scripts/build_iceberg.sh +++ b/ci/scripts/build_iceberg.sh @@ -17,7 +17,7 @@ # specific language governing permissions and limitations # under the License. # -# Usage: build_iceberg.sh [rest_integration_tests=OFF] [sccache=OFF] [s3=OFF] [sigv4=OFF] [bundle_awssdk=ON] +# Usage: build_iceberg.sh [rest_integration_tests=OFF] [sccache=OFF] [s3=OFF] [sigv4=OFF] [bundle_awssdk=ON] [build_type=Debug] set -eux @@ -37,6 +37,8 @@ is_windows() { [[ "${OSTYPE}" == "msys" || "${OSTYPE}" == "win32" || "${OSTYPE}" == "cygwin" ]] } +build_type=${7:-Debug} + CMAKE_ARGS=( "-G Ninja" "-DCMAKE_INSTALL_PREFIX=${CMAKE_INSTALL_PREFIX:-${ICEBERG_HOME}}" @@ -64,16 +66,16 @@ else fi if is_windows; then - CMAKE_ARGS+=("-DCMAKE_TOOLCHAIN_FILE=C:/vcpkg/scripts/buildsystems/vcpkg.cmake") - CMAKE_ARGS+=("-DCMAKE_BUILD_TYPE=Release") -else - # Pass an externally provided toolchain (e.g. vcpkg for the SigV4 job) - if [[ -n "${CMAKE_TOOLCHAIN_FILE:-}" ]]; then - CMAKE_ARGS+=("-DCMAKE_TOOLCHAIN_FILE=${CMAKE_TOOLCHAIN_FILE}") - fi - CMAKE_ARGS+=("-DCMAKE_BUILD_TYPE=Debug") + CMAKE_TOOLCHAIN_FILE="${CMAKE_TOOLCHAIN_FILE:-C:/vcpkg/scripts/buildsystems/vcpkg.cmake}" fi +# Pass an externally provided toolchain, or the default Windows vcpkg toolchain. +if [[ -n "${CMAKE_TOOLCHAIN_FILE:-}" ]]; then + CMAKE_ARGS+=("-DCMAKE_TOOLCHAIN_FILE=${CMAKE_TOOLCHAIN_FILE}") +fi + +CMAKE_ARGS+=("-DCMAKE_BUILD_TYPE=${build_type}") + if [[ "${build_enable_sccache}" == "ON" ]]; then CMAKE_ARGS+=("-DCMAKE_CXX_COMPILER_LAUNCHER=sccache") CMAKE_ARGS+=("-DCMAKE_C_COMPILER_LAUNCHER=sccache") @@ -85,19 +87,23 @@ if [[ -n "${ICEBERG_EXTRA_CMAKE_ARGS:-}" ]]; then fi cmake "${CMAKE_ARGS[@]}" ${source_dir} -if is_windows; then - cmake --build . --config Release --target install - if [[ "${run_tests}" == "ON" ]]; then - ctest --output-on-failure -C Release - fi -else - cmake --build . --target install - if [[ "${run_tests}" == "ON" ]]; then + +cmake --build . --target install +if [[ "${run_tests}" == "ON" ]]; then ctest --output-on-failure - fi fi popd -# clean up between builds -rm -rf ${build_dir} +# Clean up after the build. Windows can briefly hold a just-built exe/dll, +# so retry but do not fail an otherwise successful CI job. +for attempt in 1 2 3; do + if rm -rf "${build_dir}"; then + break + fi + if [[ "${attempt}" != "3" ]]; then + sleep 2 + else + echo "Failed to remove build directory after 3 attempts: ${build_dir}" >&2 + fi +done diff --git a/cmake_modules/IcebergBuildUtils.cmake b/cmake_modules/IcebergBuildUtils.cmake index 74b0297088..689ec44b7d 100644 --- a/cmake_modules/IcebergBuildUtils.cmake +++ b/cmake_modules/IcebergBuildUtils.cmake @@ -20,6 +20,28 @@ include(CMakePackageConfigHelpers) +function(iceberg_configure_version_header INPUT_FILE OUTPUT_FILE) + set(ICEBERG_VERSION_SUFFIX "-SNAPSHOT") + set(ICEBERG_VERSION_STRING "${PROJECT_VERSION}${ICEBERG_VERSION_SUFFIX}") + set(ICEBERG_GIT_COMMIT_ID "unknown") + + find_package(Git QUIET) + if(GIT_FOUND) + execute_process(COMMAND "${GIT_EXECUTABLE}" rev-parse HEAD + WORKING_DIRECTORY "${PROJECT_SOURCE_DIR}" + OUTPUT_VARIABLE ICEBERG_GIT_COMMIT_ID + RESULT_VARIABLE ICEBERG_GIT_RESULT + OUTPUT_STRIP_TRAILING_WHITESPACE ERROR_QUIET) + if(NOT ICEBERG_GIT_RESULT EQUAL 0 OR NOT ICEBERG_GIT_COMMIT_ID) + set(ICEBERG_GIT_COMMIT_ID "unknown") + endif() + endif() + + set(ICEBERG_FULL_VERSION_STRING + "Apache Iceberg-CPP ${ICEBERG_VERSION_STRING} (commit ${ICEBERG_GIT_COMMIT_ID})") + configure_file("${INPUT_FILE}" "${OUTPUT_FILE}" @ONLY) +endfunction() + function(iceberg_install_cmake_package PACKAGE_NAME EXPORT_NAME) set(CONFIG_CMAKE "${PACKAGE_NAME}-config.cmake") set(BUILT_CONFIG_CMAKE "${CMAKE_CURRENT_BINARY_DIR}/${CONFIG_CMAKE}") @@ -113,6 +135,7 @@ function(add_iceberg_lib LIB_NAME) if(BUILD_SHARED) add_library(${LIB_NAME}_shared SHARED) + set_target_properties(${LIB_NAME}_shared PROPERTIES COMPILE_WARNING_AS_ERROR ON) if(LIB_DEPS) target_sources(${LIB_NAME}_shared PRIVATE ${LIB_DEPS}) @@ -172,6 +195,7 @@ function(add_iceberg_lib LIB_NAME) if(BUILD_STATIC) add_library(${LIB_NAME}_static STATIC) + set_target_properties(${LIB_NAME}_static PROPERTIES COMPILE_WARNING_AS_ERROR ON) if(LIB_DEPS) target_sources(${LIB_NAME}_static PRIVATE ${LIB_DEPS}) diff --git a/cmake_modules/IcebergSccache.cmake b/cmake_modules/IcebergSccache.cmake index d02e65df53..a618a48629 100644 --- a/cmake_modules/IcebergSccache.cmake +++ b/cmake_modules/IcebergSccache.cmake @@ -18,19 +18,12 @@ if(MSVC_TOOLCHAIN AND "${CMAKE_CXX_COMPILER_LAUNCHER}" STREQUAL "sccache") message(STATUS "Configuring sccache for MSVC") - # Remove /Zi or /ZI - string(REGEX REPLACE "/Z[iI]" "" CMAKE_C_FLAGS_DEBUG "${CMAKE_C_FLAGS_DEBUG}") - string(REGEX REPLACE "/Z[iI]" "" CMAKE_CXX_FLAGS_DEBUG "${CMAKE_CXX_FLAGS_DEBUG}") + # Keep MSVC Debug objects cacheable by sccache without affecting Release. + set(CMAKE_MSVC_DEBUG_INFORMATION_FORMAT "$<$:Embedded>") - string(REGEX REPLACE "/Z[iI]" "" CMAKE_C_FLAGS_RELWITHDEBINFO - "${CMAKE_C_FLAGS_RELWITHDEBINFO}") - string(REGEX REPLACE "/Z[iI]" "" CMAKE_CXX_FLAGS_RELWITHDEBINFO - "${CMAKE_CXX_FLAGS_RELWITHDEBINFO}") - - # Add /Z7 - set(CMAKE_C_FLAGS_DEBUG "${CMAKE_C_FLAGS_DEBUG} /Z7") - set(CMAKE_CXX_FLAGS_DEBUG "${CMAKE_CXX_FLAGS_DEBUG} /Z7") - - set(CMAKE_C_FLAGS_RELWITHDEBINFO "${CMAKE_C_FLAGS_RELWITHDEBINFO} /Z7") - set(CMAKE_CXX_FLAGS_RELWITHDEBINFO "${CMAKE_CXX_FLAGS_RELWITHDEBINFO} /Z7") + # CMP0141 normally handles these flags; normalize any flags injected elsewhere. + foreach(flag_var CMAKE_C_FLAGS_DEBUG CMAKE_CXX_FLAGS_DEBUG CMAKE_C_FLAGS_RELWITHDEBINFO + CMAKE_CXX_FLAGS_RELWITHDEBINFO) + string(REGEX REPLACE "/Z[iI]" "/Z7" ${flag_var} "${${flag_var}}") + endforeach() endif() diff --git a/cmake_modules/IcebergThirdpartyToolchain.cmake b/cmake_modules/IcebergThirdpartyToolchain.cmake index 825a5a0548..86e8400975 100644 --- a/cmake_modules/IcebergThirdpartyToolchain.cmake +++ b/cmake_modules/IcebergThirdpartyToolchain.cmake @@ -83,6 +83,7 @@ endfunction() # ICEBERG_AVRO_GIT_URL - Apache Avro git repository URL # ICEBERG_NANOARROW_URL - Nanoarrow tarball URL # ICEBERG_CROARING_URL - CRoaring tarball URL +# ICEBERG_UTF8PROC_URL - utf8proc tarball URL # ICEBERG_NLOHMANN_JSON_URL - nlohmann-json tarball URL # ICEBERG_SPDLOG_URL - spdlog tarball URL # ICEBERG_CPR_URL - cpr tarball URL @@ -120,6 +121,20 @@ else() ) endif() +set(ICEBERG_UTF8PROC_BUILD_VERSION "2.10.0") +set(ICEBERG_UTF8PROC_BUILD_SHA256_CHECKSUM + "276a37dc4d1dd24d7896826a579f4439d1e5fe33603add786bb083cab802e23e") + +if(DEFINED ENV{ICEBERG_UTF8PROC_URL}) + set(UTF8PROC_SOURCE_URL "$ENV{ICEBERG_UTF8PROC_URL}") +else() + # Use the release asset (stable bytes, matching subprojects/utf8proc.wrap) rather + # than the auto-generated tag archive, whose contents GitHub does not guarantee. + set(UTF8PROC_SOURCE_URL + "https://github.com/JuliaStrings/utf8proc/releases/download/v${ICEBERG_UTF8PROC_BUILD_VERSION}/utf8proc-${ICEBERG_UTF8PROC_BUILD_VERSION}.tar.gz" + ) +endif() + # ---------------------------------------------------------------------- # FetchContent @@ -151,6 +166,10 @@ endmacro() function(resolve_arrow_dependency) prepare_fetchcontent() + # Prevent Arrow from injecting -Werror into CMAKE_CXX_FLAGS_DEBUG via + # arrow_add_werror_if_debug(). PRODUCTION level only adds standard warnings. + set(BUILD_WARNING_LEVEL PRODUCTION) + set(ARROW_BUILD_SHARED OFF) set(ARROW_BUILD_STATIC ON) # Work around undefined symbol: arrow::ipc::ReadSchema(arrow::io::InputStream*, arrow::ipc::DictionaryMemo*) @@ -432,6 +451,61 @@ function(resolve_croaring_dependency) PARENT_SCOPE) endfunction() +# ---------------------------------------------------------------------- +# utf8proc + +function(resolve_utf8proc_dependency) + prepare_fetchcontent() + + # The vendored build needs no install rules; without this, CMake < 3.28 (where + # FetchContent has no EXCLUDE_FROM_ALL) would install utf8proc's headers and + # pkg-config file into the iceberg install prefix. + set(UTF8PROC_INSTALL OFF) + + fetchcontent_declare(utf8proc + ${FC_DECLARE_COMMON_OPTIONS} + URL ${UTF8PROC_SOURCE_URL} + URL_HASH "SHA256=${ICEBERG_UTF8PROC_BUILD_SHA256_CHECKSUM}" + FIND_PACKAGE_ARGS + NAMES + utf8proc + CONFIG) + fetchcontent_makeavailable(utf8proc) + + if(utf8proc_SOURCE_DIR) + if(NOT TARGET utf8proc::utf8proc) + add_library(utf8proc::utf8proc INTERFACE IMPORTED) + target_link_libraries(utf8proc::utf8proc INTERFACE utf8proc) + target_include_directories(utf8proc::utf8proc INTERFACE ${utf8proc_SOURCE_DIR}) + endif() + + set(UTF8PROC_VENDORED TRUE) + # utf8proc's CMake puts a raw build-tree path in INTERFACE_INCLUDE_DIRECTORIES, which + # install(EXPORT) rejects. Wrap it in BUILD_INTERFACE so the export is valid; utf8proc + # is a private dependency, so installed consumers never need its headers. + set_target_properties(utf8proc + PROPERTIES OUTPUT_NAME "iceberg_vendored_utf8proc" + POSITION_INDEPENDENT_CODE ON + INTERFACE_INCLUDE_DIRECTORIES + "$") + install(TARGETS utf8proc + EXPORT iceberg_targets + RUNTIME DESTINATION "${ICEBERG_INSTALL_BINDIR}" + ARCHIVE DESTINATION "${ICEBERG_INSTALL_LIBDIR}" + LIBRARY DESTINATION "${ICEBERG_INSTALL_LIBDIR}") + else() + set(UTF8PROC_VENDORED FALSE) + list(APPEND ICEBERG_SYSTEM_DEPENDENCIES utf8proc) + endif() + + set(ICEBERG_SYSTEM_DEPENDENCIES + ${ICEBERG_SYSTEM_DEPENDENCIES} + PARENT_SCOPE) + set(UTF8PROC_VENDORED + ${UTF8PROC_VENDORED} + PARENT_SCOPE) +endfunction() + # ---------------------------------------------------------------------- # nlohmann-json @@ -730,6 +804,7 @@ endfunction() resolve_zlib_dependency() resolve_nanoarrow_dependency() resolve_croaring_dependency() +resolve_utf8proc_dependency() resolve_nlohmann_json_dependency() resolve_spdlog_dependency() diff --git a/meson.options b/meson.options index c53574889b..0448c61df5 100644 --- a/meson.options +++ b/meson.options @@ -52,3 +52,10 @@ option( ) option('tests', type: 'feature', description: 'Build tests', value: 'enabled') + +option( + 'benchmarks', + type: 'feature', + description: 'Build benchmarks', + value: 'disabled', +) diff --git a/mkdocs/Doxyfile b/mkdocs/Doxyfile index 75e5ee232b..0237eb4795 100644 --- a/mkdocs/Doxyfile +++ b/mkdocs/Doxyfile @@ -966,7 +966,7 @@ WARN_LOGFILE = # spaces. See also FILE_PATTERNS and EXTENSION_MAPPING # Note: If this tag is empty the current directory is searched. -INPUT = ../src ../README.md +INPUT = ../src ../README.md namespace_docs.dox # This tag can be used to specify the character encoding of the source files # that doxygen parses. Internally doxygen uses the UTF-8 encoding. Doxygen uses @@ -1006,57 +1006,9 @@ INPUT_FILE_ENCODING = # be provided as doxygen C comment), *.py, *.pyw, *.f90, *.f95, *.f03, *.f08, # *.f18, *.f, *.for, *.vhd, *.vhdl, *.ucf, *.qsf and *.ice. -FILE_PATTERNS = *.c \ - *.cc \ - *.cxx \ - *.cxxm \ - *.cpp \ - *.cppm \ - *.ccm \ - *.c++ \ - *.c++m \ - *.java \ - *.ii \ - *.ixx \ - *.ipp \ - *.i++ \ - *.inl \ - *.idl \ - *.ddl \ - *.odl \ - *.h \ - *.hh \ - *.hxx \ - *.hpp \ - *.h++ \ - *.ixx \ - *.l \ - *.cs \ - *.d \ - *.php \ - *.php4 \ - *.php5 \ - *.phtml \ - *.inc \ - *.m \ - *.markdown \ +FILE_PATTERNS = *.h \ *.md \ - *.mm \ - *.dox \ - *.py \ - *.pyw \ - *.f90 \ - *.f95 \ - *.f03 \ - *.f08 \ - *.f18 \ - *.f \ - *.for \ - *.vhd \ - *.vhdl \ - *.ucf \ - *.qsf \ - *.ice + *.dox # The RECURSIVE tag can be used to specify whether or not subdirectories should # be searched for input files as well. @@ -1071,7 +1023,8 @@ RECURSIVE = YES # Note that relative paths are relative to the directory from which doxygen is # run. -EXCLUDE = +EXCLUDE = ../src/iceberg/test \ + ../src/iceberg/catalog/hive/gen-cpp # The EXCLUDE_SYMLINKS tag can be used to select whether or not files or # directories that are symbolic links (a Unix file system feature) are excluded @@ -1087,7 +1040,7 @@ EXCLUDE_SYMLINKS = NO # Note that the wildcards are matched against the file with absolute path, so to # exclude all test directories for example use the pattern */test/* -EXCLUDE_PATTERNS = +EXCLUDE_PATTERNS = *_internal.h # The EXCLUDE_SYMBOLS tag can be used to specify one or more symbol names # (namespaces, classes, functions, etc.) that should be excluded from the diff --git a/mkdocs/build-docs.sh b/mkdocs/build-docs.sh index 6917d339ea..70f4bf8f7b 100755 --- a/mkdocs/build-docs.sh +++ b/mkdocs/build-docs.sh @@ -41,6 +41,7 @@ if ! command -v doxygen &> /dev/null; then else echo "Building API documentation with Doxygen..." cd mkdocs + rm -rf docs/api mkdir -p docs/api doxygen Doxyfile cd .. diff --git a/mkdocs/docs/getting-started.md b/mkdocs/docs/getting-started.md index 10a1a5c907..52a04e635c 100644 --- a/mkdocs/docs/getting-started.md +++ b/mkdocs/docs/getting-started.md @@ -143,6 +143,7 @@ If you experience network issues when downloading dependencies, you can override | `ICEBERG_AVRO_GIT_URL` | Apache Avro git repository | | `ICEBERG_NANOARROW_URL` | Nanoarrow tarball | | `ICEBERG_CROARING_URL` | CRoaring tarball | +| `ICEBERG_UTF8PROC_URL` | utf8proc tarball | | `ICEBERG_NLOHMANN_JSON_URL` | nlohmann-json tarball | | `ICEBERG_CPR_URL` | cpr tarball | diff --git a/mkdocs/namespace_docs.dox b/mkdocs/namespace_docs.dox new file mode 100644 index 0000000000..9faa714d68 --- /dev/null +++ b/mkdocs/namespace_docs.dox @@ -0,0 +1,54 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +/// \namespace iceberg +/// \brief Core Apache Iceberg C++ APIs. + +/// \namespace iceberg::arrow +/// \brief Arrow-backed IO and data conversion APIs. + +/// \namespace iceberg::avro +/// \brief Avro reader, writer, and metadata APIs. + +/// \namespace iceberg::hive +/// \brief Hive catalog APIs. + +/// \namespace iceberg::parquet +/// \brief Parquet reader, writer, and registration APIs. + +/// \namespace iceberg::puffin +/// \brief Puffin file metadata, reader, and writer APIs. + +/// \namespace iceberg::rest +/// \brief REST catalog APIs. + +/// \namespace iceberg::rest::auth +/// \brief REST catalog authentication APIs. + +/// \namespace iceberg::retry +/// \brief Retry policy APIs. + +/// \namespace iceberg::sql +/// \brief SQL catalog APIs. + +/// \namespace iceberg::table +/// \brief Table update requirement APIs. + +/// \namespace iceberg::util +/// \brief Utility APIs. diff --git a/src/iceberg/CMakeLists.txt b/src/iceberg/CMakeLists.txt index 999532c1c5..1cd1d44674 100644 --- a/src/iceberg/CMakeLists.txt +++ b/src/iceberg/CMakeLists.txt @@ -24,6 +24,7 @@ set(ICEBERG_SOURCES catalog/memory/in_memory_catalog.cc catalog/session_catalog.cc catalog/session_context.cc + puffin_dv_io.cc delete_file_index.cc expression/aggregate.cc expression/binder.cc @@ -154,23 +155,27 @@ list(APPEND "$,nanoarrow::nanoarrow_static,$,nanoarrow::nanoarrow_static,nanoarrow::nanoarrow_shared>>" nlohmann_json::nlohmann_json spdlog::spdlog + utf8proc::utf8proc ZLIB::ZLIB) list(APPEND ICEBERG_SHARED_BUILD_INTERFACE_LIBS "$,nanoarrow::nanoarrow_static,$,nanoarrow::nanoarrow_shared,nanoarrow::nanoarrow_static>>" nlohmann_json::nlohmann_json spdlog::spdlog + utf8proc::utf8proc ZLIB::ZLIB) list(APPEND ICEBERG_STATIC_INSTALL_INTERFACE_LIBS "$,iceberg::nanoarrow_static,$,nanoarrow::nanoarrow_static,nanoarrow::nanoarrow_shared>>" "$,iceberg::nlohmann_json,$,nlohmann_json::nlohmann_json,nlohmann_json::nlohmann_json>>" - "$,iceberg::spdlog,spdlog::spdlog>") + "$,iceberg::spdlog,spdlog::spdlog>" + "$,iceberg::utf8proc,utf8proc::utf8proc>") list(APPEND ICEBERG_SHARED_INSTALL_INTERFACE_LIBS "$,iceberg::nanoarrow_static,$,nanoarrow::nanoarrow_shared,nanoarrow::nanoarrow_static>>" "$,iceberg::nlohmann_json,$,nlohmann_json::nlohmann_json,nlohmann_json::nlohmann_json>>" - "$,iceberg::spdlog,spdlog::spdlog>") + "$,iceberg::spdlog,spdlog::spdlog>" + "$,iceberg::utf8proc,utf8proc::utf8proc>") add_iceberg_lib(iceberg SOURCES @@ -192,6 +197,9 @@ set(ICEBERG_DATA_SOURCES data/data_writer.cc data/delete_filter.cc data/delete_loader.cc + data/puffin_dv_register.cc + data/deletion_vector_writer.cc + data/dv_util.cc data/equality_delete_writer.cc data/file_scan_task_reader.cc data/position_delete_writer.cc @@ -361,3 +369,7 @@ endif() if(ICEBERG_BUILD_TESTS) add_subdirectory(test) endif() + +if(ICEBERG_BUILD_BENCHMARKS) + add_subdirectory(benchmark) +endif() diff --git a/src/iceberg/arrow/arrow_io_util.h b/src/iceberg/arrow/arrow_io_util.h index e33826dc69..925ee12a27 100644 --- a/src/iceberg/arrow/arrow_io_util.h +++ b/src/iceberg/arrow/arrow_io_util.h @@ -19,6 +19,9 @@ #pragma once +/// \file iceberg/arrow/arrow_io_util.h +/// \brief Provide helpers to create Arrow-backed FileIO implementations. + #include #include #include diff --git a/src/iceberg/arrow/literal_util.cc b/src/iceberg/arrow/literal_util.cc index c13338199b..e8ee4d6c64 100644 --- a/src/iceberg/arrow/literal_util.cc +++ b/src/iceberg/arrow/literal_util.cc @@ -123,9 +123,7 @@ Result> ToArrowScalar(const Literal& literal, return std::make_shared<::arrow::DoubleScalar>(std::get(value)); case TypeId::kDecimal: { const auto& decimal = std::get(value); - ::arrow::Decimal128 arrow_decimal( - static_cast(decimal.value() >> 64), - static_cast(decimal.value() & ~uint64_t{0})); + ::arrow::Decimal128 arrow_decimal(decimal.high(), decimal.low()); return std::make_shared<::arrow::Decimal128Scalar>(arrow_decimal, std::move(arrow_type)); } diff --git a/src/iceberg/arrow/s3/s3_properties.h b/src/iceberg/arrow/s3/s3_properties.h index 7b76968a66..03d1492ade 100644 --- a/src/iceberg/arrow/s3/s3_properties.h +++ b/src/iceberg/arrow/s3/s3_properties.h @@ -19,6 +19,9 @@ #pragma once +/// \file iceberg/arrow/s3/s3_properties.h +/// \brief Define S3 configuration property keys. + #include namespace iceberg::arrow { diff --git a/src/iceberg/arrow_row_builder.cc b/src/iceberg/arrow_row_builder.cc index b53c881bb0..26e7cb4a2d 100644 --- a/src/iceberg/arrow_row_builder.cc +++ b/src/iceberg/arrow_row_builder.cc @@ -17,7 +17,9 @@ * under the License. */ -#include +#include +#include +#include #include @@ -109,12 +111,49 @@ Status AppendInt(ArrowArray* array, int64_t value) { return {}; } +Status AppendUInt(ArrowArray* array, uint64_t value) { + ICEBERG_NANOARROW_RETURN_UNEXPECTED(ArrowArrayAppendUInt(array, value)); + return {}; +} + +Status AppendDouble(ArrowArray* array, double value) { + ICEBERG_NANOARROW_RETURN_UNEXPECTED(ArrowArrayAppendDouble(array, value)); + return {}; +} + Status AppendString(ArrowArray* array, std::string_view value) { ArrowStringView view(value.data(), static_cast(value.size())); ICEBERG_NANOARROW_RETURN_UNEXPECTED(ArrowArrayAppendString(array, view)); return {}; } +Status AppendBytes(ArrowArray* array, std::span value) { + ArrowBufferViewData data; + data.as_char = reinterpret_cast(value.data()); + ArrowBufferView view(data, static_cast(value.size())); + ICEBERG_NANOARROW_RETURN_UNEXPECTED(ArrowArrayAppendBytes(array, view)); + return {}; +} + +Status AppendIntList(ArrowArray* array, const std::vector& values) { + auto list_array = array->children[0]; + for (const auto& value : values) { + ICEBERG_NANOARROW_RETURN_UNEXPECTED( + ArrowArrayAppendInt(list_array, static_cast(value))); + } + ICEBERG_NANOARROW_RETURN_UNEXPECTED(ArrowArrayFinishElement(array)); + return {}; +} + +Status AppendIntList(ArrowArray* array, const std::vector& values) { + auto list_array = array->children[0]; + for (const auto& value : values) { + ICEBERG_NANOARROW_RETURN_UNEXPECTED(ArrowArrayAppendInt(list_array, value)); + } + ICEBERG_NANOARROW_RETURN_UNEXPECTED(ArrowArrayFinishElement(array)); + return {}; +} + Status AppendStringMap(ArrowArray* array, const std::unordered_map& entries) { // A nanoarrow map array is a list of struct. children[0] is the @@ -134,4 +173,37 @@ Status AppendStringMap(ArrowArray* array, return {}; } +Status AppendIntMap(ArrowArray* array, const std::map& entries) { + auto map_array = array->children[0]; + if (map_array->n_children != 2) { + return InvalidArrowData("Map array must have exactly 2 children."); + } + for (const auto& [key, value] : entries) { + auto key_array = map_array->children[0]; + auto value_array = map_array->children[1]; + ICEBERG_RETURN_UNEXPECTED(AppendInt(key_array, static_cast(key))); + ICEBERG_RETURN_UNEXPECTED(AppendInt(value_array, value)); + ICEBERG_NANOARROW_RETURN_UNEXPECTED(ArrowArrayFinishElement(map_array)); + } + ICEBERG_NANOARROW_RETURN_UNEXPECTED(ArrowArrayFinishElement(array)); + return {}; +} + +Status AppendBinaryMap(ArrowArray* array, + const std::map>& entries) { + auto map_array = array->children[0]; + if (map_array->n_children != 2) { + return InvalidArrowData("Map array must have exactly 2 children."); + } + for (const auto& [key, value] : entries) { + auto key_array = map_array->children[0]; + auto value_array = map_array->children[1]; + ICEBERG_RETURN_UNEXPECTED(AppendInt(key_array, static_cast(key))); + ICEBERG_RETURN_UNEXPECTED(AppendBytes(value_array, value)); + ICEBERG_NANOARROW_RETURN_UNEXPECTED(ArrowArrayFinishElement(map_array)); + } + ICEBERG_NANOARROW_RETURN_UNEXPECTED(ArrowArrayFinishElement(array)); + return {}; +} + } // namespace iceberg diff --git a/src/iceberg/arrow_row_builder_internal.h b/src/iceberg/arrow_row_builder_internal.h index f3fd2b499d..db1b66f633 100644 --- a/src/iceberg/arrow_row_builder_internal.h +++ b/src/iceberg/arrow_row_builder_internal.h @@ -30,8 +30,11 @@ /// boilerplate. #include +#include +#include #include #include +#include #include "iceberg/arrow_c_data.h" #include "iceberg/iceberg_export.h" @@ -113,9 +116,26 @@ ICEBERG_EXPORT Status AppendBoolean(ArrowArray* array, bool value); /// Works for int32/int64/timestamp columns, which nanoarrow stores as int64. ICEBERG_EXPORT Status AppendInt(ArrowArray* array, int64_t value); +/// \brief Append an unsigned integer value to a nanoarrow array builder. +ICEBERG_EXPORT Status AppendUInt(ArrowArray* array, uint64_t value); + +/// \brief Append a double value to a nanoarrow array builder. +ICEBERG_EXPORT Status AppendDouble(ArrowArray* array, double value); + /// \brief Append a string value to a nanoarrow array builder. ICEBERG_EXPORT Status AppendString(ArrowArray* array, std::string_view value); +/// \brief Append a binary value to a nanoarrow array builder. +ICEBERG_EXPORT Status AppendBytes(ArrowArray* array, std::span value); + +/// \brief Append a list of int32 values to a nanoarrow list array builder. +ICEBERG_EXPORT Status AppendIntList(ArrowArray* array, + const std::vector& values); + +/// \brief Append a list of int64 values to a nanoarrow list array builder. +ICEBERG_EXPORT Status AppendIntList(ArrowArray* array, + const std::vector& values); + /// \brief Append a map value to a nanoarrow map array builder. /// /// Appends one (possibly empty) map element. The iteration order of the @@ -123,4 +143,12 @@ ICEBERG_EXPORT Status AppendString(ArrowArray* array, std::string_view value); ICEBERG_EXPORT Status AppendStringMap( ArrowArray* array, const std::unordered_map& entries); +/// \brief Append a map to a nanoarrow map array builder. +ICEBERG_EXPORT Status AppendIntMap(ArrowArray* array, + const std::map& entries); + +/// \brief Append a map to a nanoarrow map array builder. +ICEBERG_EXPORT Status AppendBinaryMap( + ArrowArray* array, const std::map>& entries); + } // namespace iceberg diff --git a/src/iceberg/avro/avro_constants.h b/src/iceberg/avro/avro_constants.h index 6fdfdc5edb..f3f175c1e2 100644 --- a/src/iceberg/avro/avro_constants.h +++ b/src/iceberg/avro/avro_constants.h @@ -19,6 +19,9 @@ #pragma once +/// \file iceberg/avro/avro_constants.h +/// \brief Define Avro metadata constants used by Iceberg. + #include namespace iceberg::avro { diff --git a/src/iceberg/avro/avro_metrics.h b/src/iceberg/avro/avro_metrics.h index 799c00ee39..3f055c23ba 100644 --- a/src/iceberg/avro/avro_metrics.h +++ b/src/iceberg/avro/avro_metrics.h @@ -19,6 +19,9 @@ #pragma once +/// \file iceberg/avro/avro_metrics.h +/// \brief Compute Iceberg metrics for Avro data files. + #include "iceberg/iceberg_bundle_export.h" #include "iceberg/metrics.h" diff --git a/src/iceberg/avro/avro_reader.h b/src/iceberg/avro/avro_reader.h index 24f95f5d43..c193d00c24 100644 --- a/src/iceberg/avro/avro_reader.h +++ b/src/iceberg/avro/avro_reader.h @@ -19,6 +19,9 @@ #pragma once +/// \file iceberg/avro/avro_reader.h +/// \brief Provide the Avro file reader interface. + #include "iceberg/file_reader.h" #include "iceberg/iceberg_bundle_export.h" diff --git a/src/iceberg/avro/avro_writer.h b/src/iceberg/avro/avro_writer.h index 3bd8110d93..af68fac440 100644 --- a/src/iceberg/avro/avro_writer.h +++ b/src/iceberg/avro/avro_writer.h @@ -19,6 +19,9 @@ #pragma once +/// \file iceberg/avro/avro_writer.h +/// \brief Provide the Avro file writer interface. + #include "iceberg/file_writer.h" #include "iceberg/iceberg_bundle_export.h" diff --git a/src/iceberg/benchmark/CMakeLists.txt b/src/iceberg/benchmark/CMakeLists.txt new file mode 100644 index 0000000000..0227c38713 --- /dev/null +++ b/src/iceberg/benchmark/CMakeLists.txt @@ -0,0 +1,41 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +set(BENCHMARK_ENABLE_GTEST_TESTS + OFF + CACHE BOOL "" FORCE) +set(BENCHMARK_ENABLE_INSTALL + OFF + CACHE BOOL "" FORCE) +set(BENCHMARK_ENABLE_TESTING + OFF + CACHE BOOL "" FORCE) + +fetchcontent_declare(google_benchmark + GIT_REPOSITORY https://github.com/google/benchmark.git + GIT_TAG a4cf155615c63e019ae549e31703bf367df5b471 # v1.8.4 + FIND_PACKAGE_ARGS + NAMES + benchmark + CONFIG) +fetchcontent_makeavailable(google_benchmark) + +add_executable(benchmark_smoke benchmark_smoke.cc) +target_link_libraries(benchmark_smoke + PRIVATE "$,iceberg_static,iceberg_shared>" + "$,benchmark::benchmark,benchmark>" +) diff --git a/src/iceberg/benchmark/benchmark_smoke.cc b/src/iceberg/benchmark/benchmark_smoke.cc new file mode 100644 index 0000000000..abcc355506 --- /dev/null +++ b/src/iceberg/benchmark/benchmark_smoke.cc @@ -0,0 +1,46 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +#include + +namespace iceberg { +namespace { + +void BM_BenchmarkSmoke(benchmark::State& state) { + while (state.KeepRunning()) { + benchmark::DoNotOptimize(state.iterations()); + } + + state.SetItemsProcessed(state.iterations()); +} + +BENCHMARK(BM_BenchmarkSmoke); + +} // namespace +} // namespace iceberg + +int main(int argc, char** argv) { + benchmark::Initialize(&argc, argv); + if (benchmark::ReportUnrecognizedArguments(argc, argv)) { + return 1; + } + benchmark::RunSpecifiedBenchmarks(); + benchmark::Shutdown(); + return 0; +} diff --git a/src/iceberg/benchmark/meson.build b/src/iceberg/benchmark/meson.build new file mode 100644 index 0000000000..affc34e077 --- /dev/null +++ b/src/iceberg/benchmark/meson.build @@ -0,0 +1,26 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +benchmark_dep = dependency('benchmark', required: true) + +benchmark_smoke = executable( + 'benchmark_smoke', + sources: files('benchmark_smoke.cc'), + dependencies: [iceberg_dep, benchmark_dep], +) + +benchmark('benchmark_smoke', benchmark_smoke) diff --git a/src/iceberg/catalog.h b/src/iceberg/catalog.h index 81c9ddd112..79c901fd0b 100644 --- a/src/iceberg/catalog.h +++ b/src/iceberg/catalog.h @@ -19,6 +19,9 @@ #pragma once +/// \file iceberg/catalog.h +/// \brief Define the catalog API for table and namespace operations. + #include #include #include diff --git a/src/iceberg/catalog/hive/iceberg_hive_export.h b/src/iceberg/catalog/hive/iceberg_hive_export.h index 2293045535..3863297534 100644 --- a/src/iceberg/catalog/hive/iceberg_hive_export.h +++ b/src/iceberg/catalog/hive/iceberg_hive_export.h @@ -19,6 +19,9 @@ #pragma once +/// \file iceberg/catalog/hive/iceberg_hive_export.h +/// \brief Define symbol visibility macros for the Hive catalog library. + #if defined(_WIN32) || defined(__CYGWIN__) # ifdef ICEBERG_HIVE_STATIC # define ICEBERG_HIVE_EXPORT diff --git a/src/iceberg/catalog/memory/in_memory_catalog.h b/src/iceberg/catalog/memory/in_memory_catalog.h index 22a596c107..548fd7afc5 100644 --- a/src/iceberg/catalog/memory/in_memory_catalog.h +++ b/src/iceberg/catalog/memory/in_memory_catalog.h @@ -19,6 +19,9 @@ #pragma once +/// \file iceberg/catalog/memory/in_memory_catalog.h +/// \brief Provide an in-memory catalog implementation. + #include #include "iceberg/catalog.h" diff --git a/src/iceberg/catalog/rest/http_request.h b/src/iceberg/catalog/rest/http_request.h index 47419c3610..4016df66f7 100644 --- a/src/iceberg/catalog/rest/http_request.h +++ b/src/iceberg/catalog/rest/http_request.h @@ -19,6 +19,9 @@ #pragma once +/// \file iceberg/catalog/rest/http_request.h +/// \brief Define REST HTTP request types. + #include #include #include diff --git a/src/iceberg/catalog/rest/iceberg_rest_export.h b/src/iceberg/catalog/rest/iceberg_rest_export.h index 7e8836f8fb..f8b322660c 100644 --- a/src/iceberg/catalog/rest/iceberg_rest_export.h +++ b/src/iceberg/catalog/rest/iceberg_rest_export.h @@ -19,6 +19,9 @@ #pragma once +/// \file iceberg/catalog/rest/iceberg_rest_export.h +/// \brief Define symbol visibility macros for the REST catalog library. + #if defined(_WIN32) || defined(__CYGWIN__) # ifdef ICEBERG_REST_STATIC # define ICEBERG_REST_EXPORT diff --git a/src/iceberg/catalog/rest/rest_file_io.h b/src/iceberg/catalog/rest/rest_file_io.h index 9e5f7a0a9c..9bd0a826e4 100644 --- a/src/iceberg/catalog/rest/rest_file_io.h +++ b/src/iceberg/catalog/rest/rest_file_io.h @@ -19,6 +19,9 @@ #pragma once +/// \file iceberg/catalog/rest/rest_file_io.h +/// \brief Provide helpers to create FileIO instances for REST catalog responses. + #include #include #include diff --git a/src/iceberg/catalog/rest/rest_util.h b/src/iceberg/catalog/rest/rest_util.h index 51a708994f..556b2557ad 100644 --- a/src/iceberg/catalog/rest/rest_util.h +++ b/src/iceberg/catalog/rest/rest_util.h @@ -19,6 +19,9 @@ #pragma once +/// \file iceberg/catalog/rest/rest_util.h +/// \brief Provide utilities for REST catalog path and config handling. + #include #include #include diff --git a/src/iceberg/catalog/session_catalog.h b/src/iceberg/catalog/session_catalog.h index 2d7aeae255..08ac048a51 100644 --- a/src/iceberg/catalog/session_catalog.h +++ b/src/iceberg/catalog/session_catalog.h @@ -19,6 +19,9 @@ #pragma once +/// \file iceberg/catalog/session_catalog.h +/// \brief Define session catalog routing APIs. + #include #include diff --git a/src/iceberg/catalog/session_context.h b/src/iceberg/catalog/session_context.h index 7dee828e1e..19eb8fc5f8 100644 --- a/src/iceberg/catalog/session_context.h +++ b/src/iceberg/catalog/session_context.h @@ -19,6 +19,9 @@ #pragma once +/// \file iceberg/catalog/session_context.h +/// \brief Define session context metadata. + #include #include diff --git a/src/iceberg/catalog/sql/iceberg_sql_catalog_export.h b/src/iceberg/catalog/sql/iceberg_sql_catalog_export.h index 88921779bd..d609e56752 100644 --- a/src/iceberg/catalog/sql/iceberg_sql_catalog_export.h +++ b/src/iceberg/catalog/sql/iceberg_sql_catalog_export.h @@ -19,6 +19,9 @@ #pragma once +/// \file iceberg/catalog/sql/iceberg_sql_catalog_export.h +/// \brief Define symbol visibility macros for the SQL catalog library. + #if defined(_WIN32) || defined(__CYGWIN__) # ifdef ICEBERG_SQL_CATALOG_STATIC # define ICEBERG_SQL_CATALOG_EXPORT diff --git a/src/iceberg/data/data_writer.cc b/src/iceberg/data/data_writer.cc index 4204712804..512f77cc96 100644 --- a/src/iceberg/data/data_writer.cc +++ b/src/iceberg/data/data_writer.cc @@ -20,7 +20,9 @@ #include "iceberg/data/data_writer.h" #include +#include +#include "iceberg/data/writer.h" // IWYU pragma: keep #include "iceberg/file_writer.h" #include "iceberg/manifest/manifest_entry.h" #include "iceberg/partition_spec.h" @@ -58,7 +60,7 @@ class DataWriter::Impl { return {}; } - Result Metadata() { + Result Metadata() { ICEBERG_CHECK(closed_, "Cannot get metadata before closing the writer"); ICEBERG_ASSIGN_OR_RAISE(auto metrics, writer_->metrics()); @@ -98,7 +100,7 @@ class DataWriter::Impl { options_.spec ? std::make_optional(options_.spec->spec_id()) : std::nullopt, }); - FileWriter::WriteResult result; + WriteResult result; result.data_files.push_back(std::move(data_file)); return result; } @@ -127,6 +129,6 @@ Result DataWriter::Length() const { return impl_->Length(); } Status DataWriter::Close() { return impl_->Close(); } -Result DataWriter::Metadata() { return impl_->Metadata(); } +Result DataWriter::Metadata() { return impl_->Metadata(); } } // namespace iceberg diff --git a/src/iceberg/data/delete_loader.cc b/src/iceberg/data/delete_loader.cc index 9221734012..a95fab7e62 100644 --- a/src/iceberg/data/delete_loader.cc +++ b/src/iceberg/data/delete_loader.cc @@ -19,23 +19,29 @@ #include "iceberg/data/delete_loader.h" +#include #include +#include #include #include +#include #include #include #include "iceberg/arrow/nanoarrow_status_internal.h" #include "iceberg/arrow_c_data_guard_internal.h" +#include "iceberg/data/dv_util_internal.h" #include "iceberg/deletes/position_delete_index.h" #include "iceberg/deletes/position_delete_range_consumer.h" +#include "iceberg/file_io.h" #include "iceberg/file_reader.h" #include "iceberg/manifest/manifest_entry.h" #include "iceberg/metadata_columns.h" #include "iceberg/result.h" #include "iceberg/row/arrow_array_wrapper.h" #include "iceberg/schema.h" +#include "iceberg/util/content_file_util.h" #include "iceberg/util/macros.h" #include "iceberg/util/struct_like_set.h" @@ -83,6 +89,21 @@ bool StringEquals(const ArrowArrayView* view, int64_t row_idx, std::string_view return sv.data != nullptr && std::memcmp(sv.data, target.data(), target.size()) == 0; } +Status ValidateDV(const DataFile& dv, std::string_view data_file_path) { + ICEBERG_PRECHECK(dv.content_offset.has_value(), "Invalid DV, offset cannot be null: {}", + ContentFileUtil::DVDesc(dv)); + ICEBERG_PRECHECK(dv.content_size_in_bytes.has_value(), "Invalid DV, length is null: {}", + ContentFileUtil::DVDesc(dv)); + ICEBERG_PRECHECK( + dv.content_size_in_bytes.value() <= std::numeric_limits::max(), + "Can't read DV larger than 2GB: {}", dv.content_size_in_bytes.value()); + ICEBERG_PRECHECK(dv.referenced_data_file.has_value() && + dv.referenced_data_file.value() == data_file_path, + "DV is expected to reference {}, not {}", data_file_path, + dv.referenced_data_file.value_or("")); + return {}; +} + } // namespace DeleteLoader::DeleteLoader(std::shared_ptr io) : io_(std::move(io)) {} @@ -91,7 +112,7 @@ DeleteLoader::~DeleteLoader() = default; Status DeleteLoader::LoadPositionDelete(const DataFile& file, PositionDeleteIndex& index, std::string_view data_file_path) const { - // TODO(gangwu): push down path filter to open the file. + // TODO(gangwu): Add cache hooks, worker pool, and filter pushdown. ICEBERG_ASSIGN_OR_RAISE(auto reader, OpenDeleteFile(file, PosDeleteSchema(), io_)); ICEBERG_ASSIGN_OR_RAISE(auto arrow_schema, reader->Schema()); @@ -170,30 +191,40 @@ Status DeleteLoader::LoadPositionDelete(const DataFile& file, PositionDeleteInde return reader->Close(); } -Status DeleteLoader::LoadDV(const DataFile& file, PositionDeleteIndex& index) const { - return NotSupported("Loading deletion vectors is not yet supported"); -} - Result DeleteLoader::LoadPositionDeletes( std::span> delete_files, std::string_view data_file_path) const { - PositionDeleteIndex index; + for (const auto& file : delete_files) { + ICEBERG_PRECHECK(file != nullptr, "Delete file must not be null"); + } + + if (ContentFileUtil::ContainsSingleDV(delete_files)) { + const auto& dv = delete_files.front(); + ICEBERG_RETURN_UNEXPECTED(ValidateDV(*dv, data_file_path)); + return DVUtil::ReadDV(dv, io_); + } + + std::vector> position_delete_files; for (const auto& file : delete_files) { + ICEBERG_PRECHECK(!file->IsDeletionVector(), + "Deletion vector cannot be mixed with position delete files: {}", + file->file_path); + ICEBERG_PRECHECK(file->content == DataFile::Content::kPositionDeletes, + "Expected position delete file but got content type {}", + ToString(file->content)); + if (file->referenced_data_file.has_value() && file->referenced_data_file.value() != data_file_path) { continue; } - if (file->IsDeletionVector()) { - ICEBERG_RETURN_UNEXPECTED(LoadDV(*file, index)); - continue; - } + position_delete_files.push_back(file); + } - ICEBERG_PRECHECK(file->content == DataFile::Content::kPositionDeletes, - "Expected position delete file but got content type {}", - ToString(file->content)); + PositionDeleteIndex index(position_delete_files); + for (const auto& file : position_delete_files) { ICEBERG_RETURN_UNEXPECTED(LoadPositionDelete(*file, index, data_file_path)); } diff --git a/src/iceberg/data/delete_loader.h b/src/iceberg/data/delete_loader.h index a422f06833..360914e855 100644 --- a/src/iceberg/data/delete_loader.h +++ b/src/iceberg/data/delete_loader.h @@ -70,9 +70,6 @@ class ICEBERG_DATA_EXPORT DeleteLoader { Status LoadPositionDelete(const DataFile& file, PositionDeleteIndex& index, std::string_view data_file_path) const; - /// \brief Load a single deletion vector file into the index. - Status LoadDV(const DataFile& file, PositionDeleteIndex& index) const; - std::shared_ptr io_; }; diff --git a/src/iceberg/data/deletion_vector_writer.cc b/src/iceberg/data/deletion_vector_writer.cc new file mode 100644 index 0000000000..8ff8906277 --- /dev/null +++ b/src/iceberg/data/deletion_vector_writer.cc @@ -0,0 +1,256 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +#include "iceberg/data/deletion_vector_writer.h" + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "iceberg/data/writer.h" +#include "iceberg/deletes/position_delete_index.h" +#include "iceberg/deletes/roaring_position_bitmap.h" +#include "iceberg/file_format.h" +#include "iceberg/file_io.h" // IWYU pragma: keep +#include "iceberg/manifest/manifest_entry.h" +#include "iceberg/metadata_columns.h" +#include "iceberg/partition_spec.h" +#include "iceberg/puffin/file_metadata.h" +#include "iceberg/puffin/puffin_writer.h" +#include "iceberg/result.h" +#include "iceberg/row/partition_values.h" +#include "iceberg/util/content_file_util.h" +#include "iceberg/util/macros.h" +#include "iceberg/util/string_util.h" +#include "iceberg/version.h" + +namespace iceberg { + +namespace { +constexpr std::string_view kReferencedDataFile = "referenced-data-file"; +constexpr std::string_view kCardinality = "cardinality"; +} // namespace + +class DeletionVectorWriter::Impl { + public: + explicit Impl(DeletionVectorWriterOptions options) : options_(std::move(options)) {} + + // Accumulated positions and metadata for a single referenced data file. + struct Deletes { + PositionDeleteIndex positions; + std::shared_ptr spec; + PartitionValues partition; + }; + + Deletes& DeletesFor(std::string_view referenced_data_file, + const std::shared_ptr& spec, + const PartitionValues& partition) { + auto it = deletes_by_path_.lower_bound(referenced_data_file); + if (it == deletes_by_path_.end() || + deletes_by_path_.key_comp()(referenced_data_file, it->first)) { + it = + deletes_by_path_.emplace_hint(it, std::string(referenced_data_file), Deletes{}); + it->second.spec = spec; + it->second.partition = partition; + } + return it->second; + } + + Status Delete(std::string_view referenced_data_file, int64_t pos, + const std::shared_ptr& spec, + const PartitionValues& partition) { + ICEBERG_CHECK(!closed_, "Cannot delete after writer is closed"); + ICEBERG_PRECHECK(!referenced_data_file.empty(), + "Deletion vector requires a non-empty referenced data file"); + ICEBERG_PRECHECK(spec != nullptr, "Deletion vector requires a partition spec"); + ICEBERG_PRECHECK(pos >= 0 && pos <= RoaringPositionBitmap::kMaxPosition, + "Deletion vector position out of range [0, {}]: {}", + RoaringPositionBitmap::kMaxPosition, pos); + DeletesFor(referenced_data_file, spec, partition).positions.Delete(pos); + return {}; + } + + Status Delete(std::string_view referenced_data_file, + const PositionDeleteIndex& positions, + const std::shared_ptr& spec, + const PartitionValues& partition) { + ICEBERG_CHECK(!closed_, "Cannot delete after writer is closed"); + ICEBERG_PRECHECK(!referenced_data_file.empty(), + "Deletion vector requires a non-empty referenced data file"); + ICEBERG_PRECHECK(spec != nullptr, "Deletion vector requires a partition spec"); + DeletesFor(referenced_data_file, spec, partition).positions.Merge(positions); + return {}; + } + + Status Close() { + if (closed_) { + return {}; + } + + if (deletes_by_path_.empty()) { + closed_ = true; + return {}; + } + + for (auto& [path, deletes] : deletes_by_path_) { + ICEBERG_RETURN_UNEXPECTED(LoadPreviousDeletes(path, deletes)); + } + + ICEBERG_ASSIGN_OR_RAISE(auto output_file, options_.io->NewOutputFile(options_.path)); + const std::string output_path(options_.path); + ICEBERG_ASSIGN_OR_RAISE( + auto writer, puffin::PuffinWriter::Make( + std::move(output_file), + {{std::string(puffin::StandardPuffinProperties::kCreatedBy), + ICEBERG_FULL_VERSION_STRING}})); + + for (auto& [path, deletes] : deletes_by_path_) { + ICEBERG_RETURN_UNEXPECTED(Write(*writer, path, deletes)); + } + + ICEBERG_RETURN_UNEXPECTED(writer->Finish()); + ICEBERG_ASSIGN_OR_RAISE(const int64_t file_size, writer->FileSize()); + + for (const auto& [path, _] : deletes_by_path_) { + result_.referenced_data_files.push_back(path); + ICEBERG_ASSIGN_OR_RAISE(auto data_file, CreateDV(output_path, file_size, path)); + result_.data_files.push_back(std::move(data_file)); + } + + closed_ = true; + return {}; + } + + Result Metadata() { + ICEBERG_CHECK(closed_, "Cannot get metadata before closing the writer"); + return result_; + } + + private: + Status LoadPreviousDeletes(std::string_view path, Deletes& deletes) { + ICEBERG_ASSIGN_OR_RAISE(auto previous, options_.load_previous_deletes(path)); + if (!previous.has_value()) { + return {}; + } + + deletes.positions.Merge(*previous); + for (const auto& delete_file : previous->delete_files()) { + ICEBERG_ASSIGN_OR_RAISE(bool file_scoped, + ContentFileUtil::IsFileScoped(*delete_file)); + if (file_scoped) { + result_.rewritten_delete_files.push_back(delete_file); + } + } + return {}; + } + + Status Write(puffin::PuffinWriter& writer, std::string_view path, Deletes& deletes) { + const int64_t cardinality = deletes.positions.Cardinality(); + ICEBERG_ASSIGN_OR_RAISE(auto data, deletes.positions.Serialize()); + + puffin::Blob blob{ + .type = std::string(puffin::StandardBlobTypes::kDeletionVectorV1), + .input_fields = {MetadataColumns::kFilePositionColumnId}, + // Snapshot ID and sequence number are inherited; the spec requires -1. + .snapshot_id = -1, + .sequence_number = -1, + .data = std::move(data), + .requested_compression = puffin::PuffinCompressionCodec::kNone, + }; + blob.properties.emplace(std::string(kReferencedDataFile), path); + blob.properties.emplace(std::string(kCardinality), std::format("{}", cardinality)); + + ICEBERG_ASSIGN_OR_RAISE(auto blob_metadata, writer.Write(blob)); + blobs_by_path_.insert_or_assign(std::string(path), std::move(blob_metadata)); + return {}; + } + + Result> CreateDV(const std::string& path, int64_t size, + std::string_view referenced_data_file) { + auto deletes = deletes_by_path_.find(referenced_data_file); + ICEBERG_CHECK(deletes != deletes_by_path_.end(), "Missing deletion vector for {}", + referenced_data_file); + auto blob_metadata = blobs_by_path_.find(referenced_data_file); + ICEBERG_CHECK(blob_metadata != blobs_by_path_.end(), + "Missing deletion vector blob for {}", referenced_data_file); + + return std::make_shared(DataFile{ + .content = DataFile::Content::kPositionDeletes, + .file_path = path, + .file_format = FileFormatType::kPuffin, + .partition = deletes->second.partition, + .record_count = deletes->second.positions.Cardinality(), + .file_size_in_bytes = size, + // TODO(gangwu): support encryption key metadata + .referenced_data_file = std::string(referenced_data_file), + .content_offset = blob_metadata->second.offset, + .content_size_in_bytes = blob_metadata->second.length, + .partition_spec_id = deletes->second.spec + ? std::make_optional(deletes->second.spec->spec_id()) + : std::nullopt, + }); + } + + DeletionVectorWriterOptions options_; + std::map deletes_by_path_; + std::map blobs_by_path_; + WriteResult result_; + bool closed_ = false; +}; + +DeletionVectorWriter::DeletionVectorWriter(std::unique_ptr impl) + : impl_(std::move(impl)) {} + +DeletionVectorWriter::~DeletionVectorWriter() = default; + +Result> DeletionVectorWriter::Make( + DeletionVectorWriterOptions options) { + ICEBERG_PRECHECK(!options.path.empty(), "DeletionVectorWriter requires an output path"); + ICEBERG_PRECHECK(options.io != nullptr, "DeletionVectorWriter requires a FileIO"); + ICEBERG_PRECHECK(options.load_previous_deletes != nullptr, + "DeletionVectorWriter requires a load_previous_deletes callback"); + return std::unique_ptr( + new DeletionVectorWriter(std::make_unique(std::move(options)))); +} + +Status DeletionVectorWriter::Delete(std::string_view referenced_data_file, int64_t pos, + const std::shared_ptr& spec, + const PartitionValues& partition) { + return impl_->Delete(referenced_data_file, pos, spec, partition); +} + +Status DeletionVectorWriter::Delete(std::string_view referenced_data_file, + const PositionDeleteIndex& positions, + const std::shared_ptr& spec, + const PartitionValues& partition) { + return impl_->Delete(referenced_data_file, positions, spec, partition); +} + +Status DeletionVectorWriter::Close() { return impl_->Close(); } + +Result DeletionVectorWriter::Metadata() { return impl_->Metadata(); } + +} // namespace iceberg diff --git a/src/iceberg/data/deletion_vector_writer.h b/src/iceberg/data/deletion_vector_writer.h new file mode 100644 index 0000000000..bd5bb66a5f --- /dev/null +++ b/src/iceberg/data/deletion_vector_writer.h @@ -0,0 +1,83 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +#pragma once + +/// \file iceberg/data/deletion_vector_writer.h +/// Writer that emits deletion vectors as `deletion-vector-v1` blobs in a Puffin file. + +#include +#include +#include +#include +#include +#include + +#include "iceberg/data/writer.h" +#include "iceberg/deletes/position_delete_index.h" +#include "iceberg/iceberg_data_export.h" +#include "iceberg/result.h" +#include "iceberg/row/partition_values.h" +#include "iceberg/type_fwd.h" + +namespace iceberg { + +/// \brief Options for creating a DeletionVectorWriter. +struct ICEBERG_DATA_EXPORT DeletionVectorWriterOptions { + std::string path; + std::shared_ptr io; + /// Loads existing deletes for a data file to merge, if any. + std::function>(std::string_view)> + load_previous_deletes; +}; + +/// \brief A deletion vector file writer. +class ICEBERG_DATA_EXPORT DeletionVectorWriter { + public: + ~DeletionVectorWriter(); + + /// \brief Create a new DeletionVectorWriter. + static Result> Make( + DeletionVectorWriterOptions options); + + /// \brief Mark a row position as deleted for the given data file. + Status Delete(std::string_view referenced_data_file, int64_t pos, + const std::shared_ptr& spec, + const PartitionValues& partition); + + /// \brief Mark all positions in the given index as deleted for a data file. + Status Delete(std::string_view referenced_data_file, + const PositionDeleteIndex& positions, + const std::shared_ptr& spec, + const PartitionValues& partition); + + /// \brief Write all accumulated deletion vectors to the Puffin file and close. + Status Close(); + + /// \brief The result of writing; valid only after Close(). + Result Metadata(); + + private: + class Impl; + std::unique_ptr impl_; + + explicit DeletionVectorWriter(std::unique_ptr impl); +}; + +} // namespace iceberg diff --git a/src/iceberg/data/dv_util.cc b/src/iceberg/data/dv_util.cc new file mode 100644 index 0000000000..175644841b --- /dev/null +++ b/src/iceberg/data/dv_util.cc @@ -0,0 +1,69 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +#include +#include +#include +#include +#include +#include + +#include "iceberg/data/dv_util_internal.h" +#include "iceberg/deletes/position_delete_index.h" +#include "iceberg/file_io.h" +#include "iceberg/manifest/manifest_entry.h" +#include "iceberg/result.h" +#include "iceberg/util/content_file_util.h" +#include "iceberg/util/macros.h" + +namespace iceberg { + +Result DVUtil::ReadDV(const std::shared_ptr& delete_file, + const std::shared_ptr& io) { + ICEBERG_PRECHECK(delete_file != nullptr, "Delete file must not be null"); + ICEBERG_PRECHECK(io != nullptr, "DV read requires a FileIO"); + ICEBERG_PRECHECK(ContentFileUtil::IsDV(*delete_file), + "Cannot read, not a deletion vector: {}", delete_file->file_path); + ICEBERG_PRECHECK( + delete_file->content_offset.has_value() && + delete_file->content_size_in_bytes.has_value(), + "Deletion vector requires content_offset and content_size_in_bytes: {}", + delete_file->file_path); + + const int64_t offset = delete_file->content_offset.value(); + const int64_t length = delete_file->content_size_in_bytes.value(); + ICEBERG_PRECHECK(offset >= 0 && length >= 0, + "Invalid deletion vector offset/length: offset={}, length={}", offset, + length); + ICEBERG_PRECHECK(length <= std::numeric_limits::max(), + "Cannot read deletion vector larger than 2GB: {}", length); + + ICEBERG_ASSIGN_OR_RAISE(auto input_file, io->NewInputFile(delete_file->file_path)); + ICEBERG_ASSIGN_OR_RAISE(auto stream, input_file->Open()); + + std::vector bytes(static_cast(length)); + ICEBERG_RETURN_UNEXPECTED(stream->ReadFully(offset, bytes)); + ICEBERG_RETURN_UNEXPECTED(stream->Close()); + + std::span blob(reinterpret_cast(bytes.data()), + bytes.size()); + return PositionDeleteIndex::Deserialize(blob, delete_file); +} + +} // namespace iceberg diff --git a/src/iceberg/data/dv_util_internal.h b/src/iceberg/data/dv_util_internal.h new file mode 100644 index 0000000000..4738b55bbf --- /dev/null +++ b/src/iceberg/data/dv_util_internal.h @@ -0,0 +1,40 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +#pragma once + +/// \file iceberg/data/dv_util_internal.h +/// Internal deletion vector helpers. + +#include + +#include "iceberg/deletes/position_delete_index.h" +#include "iceberg/iceberg_data_export.h" +#include "iceberg/result.h" +#include "iceberg/type_fwd.h" + +namespace iceberg { + +class ICEBERG_DATA_EXPORT DVUtil { + public: + static Result ReadDV(const std::shared_ptr& delete_file, + const std::shared_ptr& io); +}; + +} // namespace iceberg diff --git a/src/iceberg/data/equality_delete_writer.cc b/src/iceberg/data/equality_delete_writer.cc index 6604c03e48..86293addb7 100644 --- a/src/iceberg/data/equality_delete_writer.cc +++ b/src/iceberg/data/equality_delete_writer.cc @@ -20,7 +20,9 @@ #include "iceberg/data/equality_delete_writer.h" #include +#include +#include "iceberg/data/writer.h" // IWYU pragma: keep #include "iceberg/file_writer.h" #include "iceberg/manifest/manifest_entry.h" #include "iceberg/partition_spec.h" @@ -57,7 +59,7 @@ class EqualityDeleteWriter::Impl { return {}; } - Result Metadata() { + Result Metadata() { ICEBERG_CHECK(closed_, "Cannot get metadata before closing the writer"); ICEBERG_ASSIGN_OR_RAISE(auto metrics, writer_->metrics()); @@ -99,7 +101,7 @@ class EqualityDeleteWriter::Impl { options_.spec ? std::make_optional(options_.spec->spec_id()) : std::nullopt, }); - FileWriter::WriteResult result; + WriteResult result; result.data_files.push_back(std::move(data_file)); return result; } @@ -134,9 +136,7 @@ Result EqualityDeleteWriter::Length() const { return impl_->Length(); } Status EqualityDeleteWriter::Close() { return impl_->Close(); } -Result EqualityDeleteWriter::Metadata() { - return impl_->Metadata(); -} +Result EqualityDeleteWriter::Metadata() { return impl_->Metadata(); } std::span EqualityDeleteWriter::equality_field_ids() const { return impl_->equality_field_ids(); diff --git a/src/iceberg/data/meson.build b/src/iceberg/data/meson.build index bbb26db27d..2ddac93df6 100644 --- a/src/iceberg/data/meson.build +++ b/src/iceberg/data/meson.build @@ -20,6 +20,8 @@ install_headers( 'data_writer.h', 'delete_filter.h', 'delete_loader.h', + 'puffin_dv_register.h', + 'deletion_vector_writer.h', 'equality_delete_writer.h', 'file_scan_task_reader.h', 'position_delete_writer.h', diff --git a/src/iceberg/data/position_delete_writer.cc b/src/iceberg/data/position_delete_writer.cc index 3238dc506d..e487e0e97c 100644 --- a/src/iceberg/data/position_delete_writer.cc +++ b/src/iceberg/data/position_delete_writer.cc @@ -21,12 +21,14 @@ #include #include +#include #include #include #include "iceberg/arrow/nanoarrow_status_internal.h" #include "iceberg/arrow_c_data_guard_internal.h" +#include "iceberg/data/writer.h" // IWYU pragma: keep #include "iceberg/file_writer.h" #include "iceberg/manifest/manifest_entry.h" #include "iceberg/metadata_columns.h" @@ -92,7 +94,7 @@ class PositionDeleteWriter::Impl { return {}; } - Result Metadata() { + Result Metadata() { ICEBERG_CHECK(closed_, "Cannot get metadata before closing the writer"); ICEBERG_ASSIGN_OR_RAISE(auto metrics, writer_->metrics()); @@ -159,7 +161,7 @@ class PositionDeleteWriter::Impl { options_.spec ? std::make_optional(options_.spec->spec_id()) : std::nullopt, }); - FileWriter::WriteResult result; + WriteResult result; result.data_files.push_back(std::move(data_file)); return result; } @@ -233,8 +235,6 @@ Result PositionDeleteWriter::Length() const { return impl_->Length(); } Status PositionDeleteWriter::Close() { return impl_->Close(); } -Result PositionDeleteWriter::Metadata() { - return impl_->Metadata(); -} +Result PositionDeleteWriter::Metadata() { return impl_->Metadata(); } } // namespace iceberg diff --git a/src/iceberg/data/puffin_dv_register.cc b/src/iceberg/data/puffin_dv_register.cc new file mode 100644 index 0000000000..5d68bd0d7b --- /dev/null +++ b/src/iceberg/data/puffin_dv_register.cc @@ -0,0 +1,79 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +#include "iceberg/data/puffin_dv_register.h" + +#include +#include +#include + +#include "iceberg/data/deletion_vector_writer.h" +#include "iceberg/data/dv_util_internal.h" +#include "iceberg/deletes/position_delete_index.h" +#include "iceberg/manifest/manifest_entry.h" +#include "iceberg/util/macros.h" + +namespace iceberg { +namespace { + +class DataPuffinDVIO final : public PuffinDVIO { + public: + Result>> MergeAndWriteDVs( + std::span groups, std::string_view output_path, + const std::shared_ptr& io) override { + if (groups.empty()) { + return std::vector>{}; + } + + ICEBERG_ASSIGN_OR_RAISE( + auto writer, + DeletionVectorWriter::Make(DeletionVectorWriterOptions{ + .path = std::string(output_path), + .io = io, + .load_previous_deletes = [](std::string_view) + -> Result> { return std::nullopt; }})); + + for (const auto& group : groups) { + PositionDeleteIndex merged; + for (const auto& delete_file : group.delete_files) { + ICEBERG_ASSIGN_OR_RAISE(auto index, DVUtil::ReadDV(delete_file, io)); + merged.Merge(index); + } + ICEBERG_RETURN_UNEXPECTED(writer->Delete(group.referenced_data_file, merged, + group.spec, group.partition)); + } + + ICEBERG_RETURN_UNEXPECTED(writer->Close()); + ICEBERG_ASSIGN_OR_RAISE(auto metadata, writer->Metadata()); + return std::move(metadata.data_files); + } +}; + +} // namespace + +std::shared_ptr MakePuffinDVIO() { + return std::make_shared(); +} + +void RegisterPuffinDVIO() { + static PuffinDVIORegistry registry( + []() -> Result> { return MakePuffinDVIO(); }); +} + +} // namespace iceberg diff --git a/src/iceberg/data/puffin_dv_register.h b/src/iceberg/data/puffin_dv_register.h new file mode 100644 index 0000000000..fa942819da --- /dev/null +++ b/src/iceberg/data/puffin_dv_register.h @@ -0,0 +1,36 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +#pragma once + +/// \file iceberg/data/puffin_dv_register.h +/// Puffin deletion vector I/O registration. + +#include + +#include "iceberg/iceberg_data_export.h" +#include "iceberg/puffin_dv_io.h" + +namespace iceberg { + +ICEBERG_DATA_EXPORT std::shared_ptr MakePuffinDVIO(); + +ICEBERG_DATA_EXPORT void RegisterPuffinDVIO(); + +} // namespace iceberg diff --git a/src/iceberg/data/writer.h b/src/iceberg/data/writer.h index a6f5f063d9..0efcdc2745 100644 --- a/src/iceberg/data/writer.h +++ b/src/iceberg/data/writer.h @@ -24,6 +24,7 @@ #include #include +#include #include #include "iceberg/arrow_c_data.h" @@ -33,6 +34,18 @@ namespace iceberg { +/// \brief File metadata for files produced by a writer. +struct ICEBERG_DATA_EXPORT WriteResult { + /// Usually a writer produces a single data or delete file. + /// Position delete writer may produce multiple file-scoped delete files. + /// In the future, multiple files can be produced if file rolling is supported. + std::vector> data_files; + /// Data files referenced by the produced delete files. + std::vector referenced_data_files; + /// Previously written delete files that were merged and should be removed. + std::vector> rewritten_delete_files; +}; + /// \brief Base interface for data file writers. class ICEBERG_DATA_EXPORT FileWriter { public: @@ -48,14 +61,6 @@ class ICEBERG_DATA_EXPORT FileWriter { /// \brief Close the writer and finalize the file. virtual Status Close() = 0; - /// \brief File metadata for all files produced by this writer. - struct ICEBERG_DATA_EXPORT WriteResult { - /// Usually a writer produces a single data or delete file. - /// Position delete writer may produce multiple file-scoped delete files. - /// In the future, multiple files can be produced if file rolling is supported. - std::vector> data_files; - }; - /// \brief Get file metadata for all files produced by this writer. /// \note This method should be called after Close(). virtual Result Metadata() = 0; diff --git a/src/iceberg/deletes/position_delete_index.cc b/src/iceberg/deletes/position_delete_index.cc index f09d0b4809..53e33e635c 100644 --- a/src/iceberg/deletes/position_delete_index.cc +++ b/src/iceberg/deletes/position_delete_index.cc @@ -19,8 +19,109 @@ #include "iceberg/deletes/position_delete_index.h" +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "iceberg/deletes/roaring_position_bitmap.h" +#include "iceberg/manifest/manifest_entry.h" +#include "iceberg/result.h" +#include "iceberg/util/endian.h" +#include "iceberg/util/macros.h" + namespace iceberg { +namespace { + +// `deletion-vector-v1` blob framing. See: +// https://iceberg.apache.org/puffin-spec/#deletion-vector-v1-blob-type +constexpr std::array kMagic = {0xD1, 0xD3, 0x39, 0x64}; +constexpr int32_t kLengthPrefixBytes = 4; +constexpr int32_t kMagicBytes = 4; +constexpr int32_t kCrcBytes = 4; + +uint32_t ComputeCrc32(std::span bytes) { + uLong crc = crc32(0L, Z_NULL, 0); + crc = crc32(crc, reinterpret_cast(bytes.data()), + static_cast(bytes.size())); + return static_cast(crc); +} + +Result ReadBitmapDataLength(std::span blob, + const DataFile& delete_file) { + ICEBERG_PRECHECK(delete_file.content_size_in_bytes.has_value(), + "Deletion vector requires content_size_in_bytes: {}", + delete_file.file_path); + ICEBERG_PRECHECK( + std::cmp_equal(blob.size(), *delete_file.content_size_in_bytes), + "Deletion vector blob size {} does not match content_size_in_bytes {}: {}", + blob.size(), *delete_file.content_size_in_bytes, delete_file.file_path); + ICEBERG_PRECHECK(blob.size() >= kLengthPrefixBytes + kMagicBytes + kCrcBytes, + "Deletion vector blob too small: {} bytes", blob.size()); + + const auto length = ReadBigEndian(blob.data()); + const size_t expected_size = + kLengthPrefixBytes + static_cast(length) + kCrcBytes; + ICEBERG_PRECHECK(length >= kMagicBytes, "Invalid deletion vector length prefix: {}", + length); + ICEBERG_PRECHECK(blob.size() == expected_size, + "Deletion vector blob size mismatch: {} bytes, expected {}", + blob.size(), expected_size); + return length; +} + +Result DeserializeBitmap(std::span blob, + int32_t length, + const DataFile& delete_file) { + const uint8_t* bitmap_data = blob.data() + kLengthPrefixBytes; + ICEBERG_PRECHECK(std::memcmp(bitmap_data, kMagic.data(), kMagic.size()) == 0, + "Invalid deletion vector magic"); + + std::string_view vector_bytes(reinterpret_cast(bitmap_data + kMagicBytes), + static_cast(length) - kMagicBytes); + ICEBERG_ASSIGN_OR_RAISE(auto bitmap, RoaringPositionBitmap::Deserialize(vector_bytes)); + ICEBERG_PRECHECK(std::cmp_equal(bitmap.Cardinality(), delete_file.record_count), + "Deletion vector cardinality {} does not match record count {}: {}", + bitmap.Cardinality(), delete_file.record_count, delete_file.file_path); + return bitmap; +} + +Status ValidateChecksum(std::span blob, int32_t length) { + const uint8_t* bitmap_data = blob.data() + kLengthPrefixBytes; + const auto stored_crc = ReadBigEndian(bitmap_data + length); + const auto actual_crc = + ComputeCrc32(std::span(bitmap_data, static_cast(length))); + ICEBERG_PRECHECK(stored_crc == actual_crc, + "Deletion vector CRC mismatch: stored {:#010x}, computed {:#010x}", + stored_crc, actual_crc); + return {}; +} + +} // namespace + +PositionDeleteIndex::PositionDeleteIndex(RoaringPositionBitmap bitmap) + : bitmap_(std::move(bitmap)) {} + +PositionDeleteIndex::PositionDeleteIndex(std::shared_ptr delete_file) { + if (delete_file != nullptr) { + delete_files_.push_back(std::move(delete_file)); + } +} + +PositionDeleteIndex::PositionDeleteIndex( + std::vector> delete_files) + : delete_files_(std::move(delete_files)) {} + void PositionDeleteIndex::Delete(int64_t pos) { bitmap_.Add(pos); } void PositionDeleteIndex::Delete(int64_t pos_start, int64_t pos_end) { @@ -37,6 +138,40 @@ int64_t PositionDeleteIndex::Cardinality() const { void PositionDeleteIndex::Merge(const PositionDeleteIndex& other) { bitmap_.Or(other.bitmap_); + delete_files_.insert(delete_files_.end(), other.delete_files_.begin(), + other.delete_files_.end()); +} + +Result> PositionDeleteIndex::Serialize() { + bitmap_.Optimize(); // run-length encode before serializing + std::vector blob(kLengthPrefixBytes); + blob.insert(blob.end(), kMagic.begin(), kMagic.end()); + ICEBERG_ASSIGN_OR_RAISE(const auto vector_size, bitmap_.SerializeTo(blob)); + + const size_t magic_and_vector_size = kMagicBytes + vector_size; + ICEBERG_PRECHECK(magic_and_vector_size <= std::numeric_limits::max(), + "Deletion vector is too large to serialize: {} bytes", + magic_and_vector_size); + + WriteBigEndian(static_cast(magic_and_vector_size), blob.data()); + const auto crc_offset = blob.size(); + blob.resize(crc_offset + kCrcBytes); + WriteBigEndian(ComputeCrc32(std::span(blob.data() + kLengthPrefixBytes, + magic_and_vector_size)), + blob.data() + crc_offset); + return blob; +} + +Result PositionDeleteIndex::Deserialize( + std::span blob, std::shared_ptr delete_file) { + ICEBERG_PRECHECK(delete_file != nullptr, + "Deletion vector requires a source delete file"); + ICEBERG_ASSIGN_OR_RAISE(const auto length, ReadBitmapDataLength(blob, *delete_file)); + ICEBERG_ASSIGN_OR_RAISE(auto bitmap, DeserializeBitmap(blob, length, *delete_file)); + ICEBERG_RETURN_UNEXPECTED(ValidateChecksum(blob, length)); + PositionDeleteIndex index(std::move(bitmap)); + index.delete_files_.push_back(std::move(delete_file)); + return index; } void PositionDeleteIndex::BulkAddForKey(int32_t key, diff --git a/src/iceberg/deletes/position_delete_index.h b/src/iceberg/deletes/position_delete_index.h index 592216ed60..cbf2db478a 100644 --- a/src/iceberg/deletes/position_delete_index.h +++ b/src/iceberg/deletes/position_delete_index.h @@ -29,6 +29,8 @@ #include "iceberg/deletes/roaring_position_bitmap.h" #include "iceberg/iceberg_data_export.h" +#include "iceberg/result.h" +#include "iceberg/type_fwd.h" namespace iceberg { @@ -40,6 +42,8 @@ namespace iceberg { class ICEBERG_DATA_EXPORT PositionDeleteIndex { public: PositionDeleteIndex() = default; + explicit PositionDeleteIndex(std::shared_ptr delete_file); + explicit PositionDeleteIndex(std::vector> delete_files); ~PositionDeleteIndex() = default; /// \brief Mark a position as deleted. @@ -66,7 +70,33 @@ class ICEBERG_DATA_EXPORT PositionDeleteIndex { /// \param other The index to merge (union operation) void Merge(const PositionDeleteIndex& other); + /// \brief The delete files whose positions were merged into this index. + /// + /// Populated by constructors and Deserialize, and preserved across Merge. + /// Callers use these to report the delete files that were rewritten when + /// replacing them with a new deletion vector. + const std::vector>& delete_files() const { + return delete_files_; + } + + /// \brief Serialize the index into a `deletion-vector-v1` blob. + /// + /// The positions are run-length encoded, then framed per the Puffin spec: + /// https://iceberg.apache.org/puffin-spec/#deletion-vector-v1-blob-type + Result> Serialize(); + + /// \brief Deserialize a `deletion-vector-v1` blob into an index. + /// + /// Validates the blob framing (length prefix, magic sequence, CRC-32) and, + /// against the source delete file, that the blob length matches + /// `content_size_in_bytes` and the bitmap cardinality matches `record_count`. + /// The source delete file is retained and exposed via delete_files(). + static Result Deserialize(std::span blob, + std::shared_ptr delete_file); + private: + explicit PositionDeleteIndex(RoaringPositionBitmap bitmap); + // Bulk-add positions sharing high-32-bit `key`. Private hook for // `ForEachPositionDelete`'s bulk path; keeps `Delete` the sole public // mutation surface. @@ -77,6 +107,7 @@ class ICEBERG_DATA_EXPORT PositionDeleteIndex { std::vector& scratch); RoaringPositionBitmap bitmap_; + std::vector> delete_files_; }; } // namespace iceberg diff --git a/src/iceberg/deletes/position_delete_range_consumer.h b/src/iceberg/deletes/position_delete_range_consumer.h index 69d1c33d7c..37f6caeac5 100644 --- a/src/iceberg/deletes/position_delete_range_consumer.h +++ b/src/iceberg/deletes/position_delete_range_consumer.h @@ -19,6 +19,9 @@ #pragma once +/// \file iceberg/deletes/position_delete_range_consumer.h +/// \brief Provide bulk application of position deletes. + #include #include #include diff --git a/src/iceberg/deletes/roaring_position_bitmap.cc b/src/iceberg/deletes/roaring_position_bitmap.cc index e31efeec18..a2827d4bb6 100644 --- a/src/iceberg/deletes/roaring_position_bitmap.cc +++ b/src/iceberg/deletes/roaring_position_bitmap.cc @@ -19,9 +19,11 @@ #include "iceberg/deletes/roaring_position_bitmap.h" +#include #include #include #include +#include #include #include @@ -57,6 +59,16 @@ Status ValidatePosition(int64_t pos) { return {}; } +void WriteBitmaps(const std::vector& bitmaps, uint8_t* buf) { + WriteLittleEndian(static_cast(bitmaps.size()), buf); + buf += kBitmapCountSizeBytes; + for (int32_t key = 0; std::cmp_less(key, bitmaps.size()); ++key) { + WriteLittleEndian(key, buf); + buf += kBitmapKeySizeBytes; + buf += bitmaps[key].write(reinterpret_cast(buf), /*portable=*/true); + } +} + } // namespace struct RoaringPositionBitmap::Impl { @@ -186,23 +198,18 @@ size_t RoaringPositionBitmap::SerializedSizeInBytes() const { Result RoaringPositionBitmap::Serialize() const { size_t size = SerializedSizeInBytes(); std::string result(size, '\0'); - char* buf = result.data(); - - // Write bitmap count (array length including empties) - WriteLittleEndian(static_cast(impl_->bitmaps.size()), buf); - buf += kBitmapCountSizeBytes; - - // Write each bitmap with its key - for (int32_t key = 0; std::cmp_less(key, impl_->bitmaps.size()); ++key) { - WriteLittleEndian(key, buf); - buf += kBitmapKeySizeBytes; - size_t written = impl_->bitmaps[key].write(buf, /*portable=*/true); - buf += written; - } - + WriteBitmaps(impl_->bitmaps, reinterpret_cast(result.data())); return result; } +Result RoaringPositionBitmap::SerializeTo(std::vector& output) const { + const size_t offset = output.size(); + const size_t size = SerializedSizeInBytes(); + output.resize(offset + size); + WriteBitmaps(impl_->bitmaps, output.data() + offset); + return size; +} + Result RoaringPositionBitmap::Deserialize(std::string_view bytes) { const char* buf = bytes.data(); size_t remaining = bytes.size(); diff --git a/src/iceberg/deletes/roaring_position_bitmap.h b/src/iceberg/deletes/roaring_position_bitmap.h index bfb7d7c9eb..50807dc6aa 100644 --- a/src/iceberg/deletes/roaring_position_bitmap.h +++ b/src/iceberg/deletes/roaring_position_bitmap.h @@ -28,6 +28,7 @@ #include #include #include +#include #include "iceberg/iceberg_data_export.h" #include "iceberg/result.h" @@ -118,6 +119,8 @@ class ICEBERG_DATA_EXPORT RoaringPositionBitmap { // `PositionDeleteIndex::BulkAddForKey`; per-key grouping is the caller's // job, keeping this a thin wrapper around CRoaring's `addMany`. void AddManyForKey(int32_t key, std::span positions); + // Appends the portable serialized bitmap and returns bytes appended. + Result SerializeTo(std::vector& output) const; friend class PositionDeleteIndex; }; diff --git a/src/iceberg/expression/literal.h b/src/iceberg/expression/literal.h index dbcabb5210..d88ea9248a 100644 --- a/src/iceberg/expression/literal.h +++ b/src/iceberg/expression/literal.h @@ -19,6 +19,9 @@ #pragma once +/// \file iceberg/expression/literal.h +/// \brief Define typed literal values used by expressions. + #include #include #include diff --git a/src/iceberg/file_io.h b/src/iceberg/file_io.h index 7d7b31113f..3ea4afa499 100644 --- a/src/iceberg/file_io.h +++ b/src/iceberg/file_io.h @@ -19,6 +19,9 @@ #pragma once +/// \file iceberg/file_io.h +/// \brief Define the FileIO abstraction and file stream interfaces. + #include #include #include diff --git a/src/iceberg/file_io_registry.h b/src/iceberg/file_io_registry.h index a0be1b5632..1643bf457e 100644 --- a/src/iceberg/file_io_registry.h +++ b/src/iceberg/file_io_registry.h @@ -19,6 +19,9 @@ #pragma once +/// \file iceberg/file_io_registry.h +/// \brief Define the FileIO registry. + #include #include #include diff --git a/src/iceberg/iceberg_bundle_export.h b/src/iceberg/iceberg_bundle_export.h index c63f8811dc..49407a34ae 100644 --- a/src/iceberg/iceberg_bundle_export.h +++ b/src/iceberg/iceberg_bundle_export.h @@ -19,6 +19,9 @@ #pragma once +/// \file iceberg/iceberg_bundle_export.h +/// \brief Define symbol visibility macros for bundled format integrations. + #if defined(_WIN32) || defined(__CYGWIN__) # ifdef ICEBERG_BUNDLE_STATIC # define ICEBERG_BUNDLE_EXPORT diff --git a/src/iceberg/iceberg_data_export.h b/src/iceberg/iceberg_data_export.h index 652de97b6d..5b710e9cc8 100644 --- a/src/iceberg/iceberg_data_export.h +++ b/src/iceberg/iceberg_data_export.h @@ -19,6 +19,9 @@ #pragma once +/// \file iceberg/iceberg_data_export.h +/// \brief Define symbol visibility macros for data APIs. + #if defined(_WIN32) || defined(__CYGWIN__) # ifdef ICEBERG_DATA_STATIC # define ICEBERG_DATA_EXPORT diff --git a/src/iceberg/iceberg_export.h b/src/iceberg/iceberg_export.h index 2463fbeaf5..5dcfa2f668 100644 --- a/src/iceberg/iceberg_export.h +++ b/src/iceberg/iceberg_export.h @@ -19,6 +19,9 @@ #pragma once +/// \file iceberg/iceberg_export.h +/// \brief Define symbol visibility macros for core Iceberg APIs. + #if defined(_WIN32) || defined(__CYGWIN__) # ifdef ICEBERG_STATIC # define ICEBERG_EXPORT diff --git a/src/iceberg/inspect/history_table.h b/src/iceberg/inspect/history_table.h index 94a26368ff..21f1f80027 100644 --- a/src/iceberg/inspect/history_table.h +++ b/src/iceberg/inspect/history_table.h @@ -19,6 +19,9 @@ #pragma once +/// \file iceberg/inspect/history_table.h +/// \brief Define the history metadata table. + #include #include "iceberg/iceberg_export.h" diff --git a/src/iceberg/inspect/metadata_table.h b/src/iceberg/inspect/metadata_table.h index 7d0ac22dad..51c5f7920e 100644 --- a/src/iceberg/inspect/metadata_table.h +++ b/src/iceberg/inspect/metadata_table.h @@ -19,6 +19,9 @@ #pragma once +/// \file iceberg/inspect/metadata_table.h +/// \brief Define base APIs for metadata tables. + #include #include "iceberg/iceberg_export.h" diff --git a/src/iceberg/inspect/snapshots_table.h b/src/iceberg/inspect/snapshots_table.h index 50017796fa..9af1bcacb4 100644 --- a/src/iceberg/inspect/snapshots_table.h +++ b/src/iceberg/inspect/snapshots_table.h @@ -19,6 +19,9 @@ #pragma once +/// \file iceberg/inspect/snapshots_table.h +/// \brief Define the snapshots metadata table. + #include #include "iceberg/iceberg_export.h" diff --git a/src/iceberg/json_serde.cc b/src/iceberg/json_serde.cc index 297265435a..f322996ec7 100644 --- a/src/iceberg/json_serde.cc +++ b/src/iceberg/json_serde.cc @@ -1377,9 +1377,9 @@ Result> TableMetadataFromJson(const nlohmann::jso TimePointMs{std::chrono::milliseconds(last_updated_ms)}; if (json.contains(kRefs)) { - ICEBERG_ASSIGN_OR_RAISE( - table_metadata->refs, - FromJsonMap>(json, kRefs, SnapshotRefFromJson)); + ICEBERG_ASSIGN_OR_RAISE(auto refs, FromJsonMap>( + json, kRefs, SnapshotRefFromJson)); + table_metadata->refs = std::move(refs); } else if (table_metadata->current_snapshot_id != kInvalidSnapshotId) { table_metadata->refs["main"] = std::make_unique(SnapshotRef{ .snapshot_id = table_metadata->current_snapshot_id, diff --git a/src/iceberg/location_provider.h b/src/iceberg/location_provider.h index 6b6fa4475a..8541b7c117 100644 --- a/src/iceberg/location_provider.h +++ b/src/iceberg/location_provider.h @@ -19,6 +19,9 @@ #pragma once +/// \file iceberg/location_provider.h +/// \brief Define table data location providers. + #include #include #include diff --git a/src/iceberg/manifest/manifest_adapter.cc b/src/iceberg/manifest/manifest_adapter.cc index 526d312716..4b6846ae3b 100644 --- a/src/iceberg/manifest/manifest_adapter.cc +++ b/src/iceberg/manifest/manifest_adapter.cc @@ -23,6 +23,7 @@ #include #include "iceberg/arrow/nanoarrow_status_internal.h" +#include "iceberg/arrow_row_builder_internal.h" #include "iceberg/manifest/manifest_adapter_internal.h" #include "iceberg/manifest/manifest_entry.h" #include "iceberg/manifest/manifest_list.h" @@ -38,87 +39,6 @@ namespace { constexpr int64_t kBlockSizeInBytesV1 = 64 * 1024 * 1024L; constexpr int32_t kBlockSizeInBytesFieldId = 105; -Status AppendField(ArrowArray* array, int64_t value) { - ICEBERG_NANOARROW_RETURN_UNEXPECTED(ArrowArrayAppendInt(array, value)); - return {}; -} - -Status AppendField(ArrowArray* array, uint64_t value) { - ICEBERG_NANOARROW_RETURN_UNEXPECTED(ArrowArrayAppendUInt(array, value)); - return {}; -} - -Status AppendField(ArrowArray* array, double value) { - ICEBERG_NANOARROW_RETURN_UNEXPECTED(ArrowArrayAppendDouble(array, value)); - return {}; -} - -Status AppendField(ArrowArray* array, std::string_view value) { - ArrowStringView view(value.data(), value.size()); - ICEBERG_NANOARROW_RETURN_UNEXPECTED(ArrowArrayAppendString(array, view)); - return {}; -} - -Status AppendField(ArrowArray* array, std::span value) { - ArrowBufferViewData data; - data.as_char = reinterpret_cast(value.data()); - ArrowBufferView view(data, value.size()); - ICEBERG_NANOARROW_RETURN_UNEXPECTED(ArrowArrayAppendBytes(array, view)); - return {}; -} - -Status AppendList(ArrowArray* array, const std::vector& list_value) { - auto list_array = array->children[0]; - for (const auto& value : list_value) { - ICEBERG_NANOARROW_RETURN_UNEXPECTED( - ArrowArrayAppendInt(list_array, static_cast(value))); - } - ICEBERG_NANOARROW_RETURN_UNEXPECTED(ArrowArrayFinishElement(array)); - return {}; -} - -Status AppendList(ArrowArray* array, const std::vector& list_value) { - auto list_array = array->children[0]; - for (const auto& value : list_value) { - ICEBERG_NANOARROW_RETURN_UNEXPECTED(ArrowArrayAppendInt(list_array, value)); - } - ICEBERG_NANOARROW_RETURN_UNEXPECTED(ArrowArrayFinishElement(array)); - return {}; -} - -Status AppendMap(ArrowArray* array, const std::map& map_value) { - auto map_array = array->children[0]; - if (map_array->n_children != 2) { - return InvalidArrowData("Map array must have exactly 2 children."); - } - for (const auto& [key, value] : map_value) { - auto key_array = map_array->children[0]; - auto value_array = map_array->children[1]; - ICEBERG_RETURN_UNEXPECTED(AppendField(key_array, static_cast(key))); - ICEBERG_RETURN_UNEXPECTED(AppendField(value_array, value)); - ICEBERG_NANOARROW_RETURN_UNEXPECTED(ArrowArrayFinishElement(map_array)); - } - ICEBERG_NANOARROW_RETURN_UNEXPECTED(ArrowArrayFinishElement(array)); - return {}; -} - -Status AppendMap(ArrowArray* array, - const std::map>& map_value) { - auto map_array = array->children[0]; - if (map_array->n_children != 2) { - return InvalidArrowData("Map array must have exactly 2 children."); - } - for (const auto& [key, value] : map_value) { - auto key_array = map_array->children[0]; - auto value_array = map_array->children[1]; - ICEBERG_RETURN_UNEXPECTED(AppendField(key_array, static_cast(key))); - ICEBERG_RETURN_UNEXPECTED(AppendField(value_array, value)); - ICEBERG_NANOARROW_RETURN_UNEXPECTED(ArrowArrayFinishElement(map_array)); - } - ICEBERG_NANOARROW_RETURN_UNEXPECTED(ArrowArrayFinishElement(array)); - return {}; -} - } // namespace Status ManifestAdapter::StartAppending() { @@ -179,45 +99,44 @@ Status ManifestEntryAdapter::AppendPartitionValues( const auto& partition_field = fields[i]; auto child_array = array->children[i]; if (partition_value.IsNull()) { - ICEBERG_NANOARROW_RETURN_UNEXPECTED(ArrowArrayAppendNull(child_array, 1)); + ICEBERG_RETURN_UNEXPECTED(AppendNull(child_array)); continue; } switch (partition_field.type()->type_id()) { case TypeId::kBoolean: - ICEBERG_RETURN_UNEXPECTED(AppendField( - child_array, static_cast( - std::get(partition_value.value()) == true ? 1L : 0L))); + ICEBERG_RETURN_UNEXPECTED( + AppendBoolean(child_array, std::get(partition_value.value()))); break; case TypeId::kInt: - ICEBERG_RETURN_UNEXPECTED(AppendField( - child_array, - static_cast(std::get(partition_value.value())))); + ICEBERG_RETURN_UNEXPECTED( + AppendInt(child_array, + static_cast(std::get(partition_value.value())))); break; case TypeId::kLong: ICEBERG_RETURN_UNEXPECTED( - AppendField(child_array, std::get(partition_value.value()))); + AppendInt(child_array, std::get(partition_value.value()))); break; case TypeId::kFloat: - ICEBERG_RETURN_UNEXPECTED(AppendField( + ICEBERG_RETURN_UNEXPECTED(AppendDouble( child_array, static_cast(std::get(partition_value.value())))); break; case TypeId::kDouble: ICEBERG_RETURN_UNEXPECTED( - AppendField(child_array, std::get(partition_value.value()))); + AppendDouble(child_array, std::get(partition_value.value()))); break; case TypeId::kString: ICEBERG_RETURN_UNEXPECTED( - AppendField(child_array, std::get(partition_value.value()))); + AppendString(child_array, std::get(partition_value.value()))); break; case TypeId::kFixed: case TypeId::kBinary: - ICEBERG_RETURN_UNEXPECTED(AppendField( + ICEBERG_RETURN_UNEXPECTED(AppendBytes( child_array, std::get>(partition_value.value()))); break; case TypeId::kDate: - ICEBERG_RETURN_UNEXPECTED(AppendField( - child_array, - static_cast(std::get(partition_value.value())))); + ICEBERG_RETURN_UNEXPECTED( + AppendInt(child_array, + static_cast(std::get(partition_value.value())))); break; case TypeId::kTime: case TypeId::kTimestamp: @@ -225,15 +144,15 @@ Status ManifestEntryAdapter::AppendPartitionValues( case TypeId::kTimestampNs: case TypeId::kTimestampTzNs: ICEBERG_RETURN_UNEXPECTED( - AppendField(child_array, std::get(partition_value.value()))); + AppendInt(child_array, std::get(partition_value.value()))); break; case TypeId::kDecimal: - ICEBERG_RETURN_UNEXPECTED(AppendField( + ICEBERG_RETURN_UNEXPECTED(AppendBytes( child_array, std::get(partition_value.value()).ToBytes())); break; case TypeId::kUuid: ICEBERG_RETURN_UNEXPECTED( - AppendField(child_array, std::get(partition_value.value()).bytes())); + AppendBytes(child_array, std::get(partition_value.value()).bytes())); break; case TypeId::kStruct: case TypeId::kList: @@ -259,13 +178,13 @@ Status ManifestEntryAdapter::AppendDataFile( switch (field.field_id()) { case DataFile::kContentFieldId: // optional int32 ICEBERG_RETURN_UNEXPECTED( - AppendField(child_array, static_cast(file.content))); + AppendInt(child_array, static_cast(file.content))); break; case DataFile::kFilePathFieldId: // required string - ICEBERG_RETURN_UNEXPECTED(AppendField(child_array, file.file_path)); + ICEBERG_RETURN_UNEXPECTED(AppendString(child_array, file.file_path)); break; case DataFile::kFileFormatFieldId: // required string - ICEBERG_RETURN_UNEXPECTED(AppendField(child_array, ToString(file.file_format))); + ICEBERG_RETURN_UNEXPECTED(AppendString(child_array, ToString(file.file_format))); break; case DataFile::kPartitionFieldId: { // required struct auto partition_type = internal::checked_pointer_cast(field.type()); @@ -273,85 +192,84 @@ Status ManifestEntryAdapter::AppendDataFile( AppendPartitionValues(child_array, partition_type, file.partition)); } break; case DataFile::kRecordCountFieldId: // required int64 - ICEBERG_RETURN_UNEXPECTED(AppendField(child_array, file.record_count)); + ICEBERG_RETURN_UNEXPECTED(AppendInt(child_array, file.record_count)); break; case DataFile::kFileSizeFieldId: // required int64 - ICEBERG_RETURN_UNEXPECTED(AppendField(child_array, file.file_size_in_bytes)); + ICEBERG_RETURN_UNEXPECTED(AppendInt(child_array, file.file_size_in_bytes)); break; case kBlockSizeInBytesFieldId: // compatible with v1 // always 64MB for v1 - ICEBERG_RETURN_UNEXPECTED(AppendField(child_array, kBlockSizeInBytesV1)); + ICEBERG_RETURN_UNEXPECTED(AppendInt(child_array, kBlockSizeInBytesV1)); break; case DataFile::kColumnSizesFieldId: // optional map - ICEBERG_RETURN_UNEXPECTED(AppendMap(child_array, file.column_sizes)); + ICEBERG_RETURN_UNEXPECTED(AppendIntMap(child_array, file.column_sizes)); break; case DataFile::kValueCountsFieldId: // optional map - ICEBERG_RETURN_UNEXPECTED(AppendMap(child_array, file.value_counts)); + ICEBERG_RETURN_UNEXPECTED(AppendIntMap(child_array, file.value_counts)); break; case DataFile::kNullValueCountsFieldId: // optional map - ICEBERG_RETURN_UNEXPECTED(AppendMap(child_array, file.null_value_counts)); + ICEBERG_RETURN_UNEXPECTED(AppendIntMap(child_array, file.null_value_counts)); break; case DataFile::kNanValueCountsFieldId: // optional map - ICEBERG_RETURN_UNEXPECTED(AppendMap(child_array, file.nan_value_counts)); + ICEBERG_RETURN_UNEXPECTED(AppendIntMap(child_array, file.nan_value_counts)); break; case DataFile::kLowerBoundsFieldId: // optional map - ICEBERG_RETURN_UNEXPECTED(AppendMap(child_array, file.lower_bounds)); + ICEBERG_RETURN_UNEXPECTED(AppendBinaryMap(child_array, file.lower_bounds)); break; case DataFile::kUpperBoundsFieldId: // optional map - ICEBERG_RETURN_UNEXPECTED(AppendMap(child_array, file.upper_bounds)); + ICEBERG_RETURN_UNEXPECTED(AppendBinaryMap(child_array, file.upper_bounds)); break; case DataFile::kKeyMetadataFieldId: // optional binary if (!file.key_metadata.empty()) { - ICEBERG_RETURN_UNEXPECTED(AppendField(child_array, file.key_metadata)); + ICEBERG_RETURN_UNEXPECTED(AppendBytes(child_array, file.key_metadata)); } else { - ICEBERG_NANOARROW_RETURN_UNEXPECTED(ArrowArrayAppendNull(child_array, 1)); + ICEBERG_RETURN_UNEXPECTED(AppendNull(child_array)); } break; case DataFile::kSplitOffsetsFieldId: // optional list - ICEBERG_RETURN_UNEXPECTED(AppendList(child_array, file.split_offsets)); + ICEBERG_RETURN_UNEXPECTED(AppendIntList(child_array, file.split_offsets)); break; case DataFile::kEqualityIdsFieldId: // optional list - ICEBERG_RETURN_UNEXPECTED(AppendList(child_array, file.equality_ids)); + ICEBERG_RETURN_UNEXPECTED(AppendIntList(child_array, file.equality_ids)); break; case DataFile::kSortOrderIdFieldId: // optional int32 if (file.sort_order_id.has_value()) { ICEBERG_RETURN_UNEXPECTED( - AppendField(child_array, static_cast(file.sort_order_id.value()))); + AppendInt(child_array, static_cast(file.sort_order_id.value()))); } else { - ICEBERG_NANOARROW_RETURN_UNEXPECTED(ArrowArrayAppendNull(child_array, 1)); + ICEBERG_RETURN_UNEXPECTED(AppendNull(child_array)); } break; case DataFile::kFirstRowIdFieldId: // optional int64 if (file.first_row_id.has_value()) { - ICEBERG_RETURN_UNEXPECTED(AppendField(child_array, file.first_row_id.value())); + ICEBERG_RETURN_UNEXPECTED(AppendInt(child_array, file.first_row_id.value())); } else { - ICEBERG_NANOARROW_RETURN_UNEXPECTED(ArrowArrayAppendNull(child_array, 1)); + ICEBERG_RETURN_UNEXPECTED(AppendNull(child_array)); } break; case DataFile::kReferencedDataFileFieldId: { // optional string ICEBERG_ASSIGN_OR_RAISE(auto referenced_data_file, GetReferenceDataFile(file)); if (referenced_data_file.has_value()) { ICEBERG_RETURN_UNEXPECTED( - AppendField(child_array, referenced_data_file.value())); + AppendString(child_array, referenced_data_file.value())); } else { - ICEBERG_NANOARROW_RETURN_UNEXPECTED(ArrowArrayAppendNull(child_array, 1)); + ICEBERG_RETURN_UNEXPECTED(AppendNull(child_array)); } break; } case DataFile::kContentOffsetFieldId: // optional int64 if (file.content_offset.has_value()) { - ICEBERG_RETURN_UNEXPECTED( - AppendField(child_array, file.content_offset.value())); + ICEBERG_RETURN_UNEXPECTED(AppendInt(child_array, file.content_offset.value())); } else { - ICEBERG_NANOARROW_RETURN_UNEXPECTED(ArrowArrayAppendNull(child_array, 1)); + ICEBERG_RETURN_UNEXPECTED(AppendNull(child_array)); } break; case DataFile::kContentSizeFieldId: // optional int64 if (file.content_size_in_bytes.has_value()) { ICEBERG_RETURN_UNEXPECTED( - AppendField(child_array, file.content_size_in_bytes.value())); + AppendInt(child_array, file.content_size_in_bytes.value())); } else { - ICEBERG_NANOARROW_RETURN_UNEXPECTED(ArrowArrayAppendNull(child_array, 1)); + ICEBERG_RETURN_UNEXPECTED(AppendNull(child_array)); } break; default: @@ -400,13 +318,13 @@ Status ManifestEntryAdapter::AppendInternal(const ManifestEntry& entry) { switch (field.field_id()) { case ManifestEntry::kStatusFieldId: // required int32 ICEBERG_RETURN_UNEXPECTED( - AppendField(array, static_cast(static_cast(entry.status)))); + AppendInt(array, static_cast(static_cast(entry.status)))); break; case ManifestEntry::kSnapshotIdFieldId: // optional int64 if (entry.snapshot_id.has_value()) { - ICEBERG_RETURN_UNEXPECTED(AppendField(array, entry.snapshot_id.value())); + ICEBERG_RETURN_UNEXPECTED(AppendInt(array, entry.snapshot_id.value())); } else { - ICEBERG_NANOARROW_RETURN_UNEXPECTED(ArrowArrayAppendNull(array, 1)); + ICEBERG_RETURN_UNEXPECTED(AppendNull(array)); } break; case ManifestEntry::kDataFileFieldId: // required struct @@ -422,18 +340,17 @@ Status ManifestEntryAdapter::AppendInternal(const ManifestEntry& entry) { case ManifestEntry::kSequenceNumberFieldId: { // optional int64 ICEBERG_ASSIGN_OR_RAISE(auto sequence_num, GetSequenceNumber(entry)); if (sequence_num.has_value()) { - ICEBERG_RETURN_UNEXPECTED(AppendField(array, sequence_num.value())); + ICEBERG_RETURN_UNEXPECTED(AppendInt(array, sequence_num.value())); } else { - ICEBERG_NANOARROW_RETURN_UNEXPECTED(ArrowArrayAppendNull(array, 1)); + ICEBERG_RETURN_UNEXPECTED(AppendNull(array)); } break; } case ManifestEntry::kFileSequenceNumberFieldId: // optional int64 if (entry.file_sequence_number.has_value()) { - ICEBERG_RETURN_UNEXPECTED( - AppendField(array, entry.file_sequence_number.value())); + ICEBERG_RETURN_UNEXPECTED(AppendInt(array, entry.file_sequence_number.value())); } else { - ICEBERG_NANOARROW_RETURN_UNEXPECTED(ArrowArrayAppendNull(array, 1)); + ICEBERG_RETURN_UNEXPECTED(AppendNull(array)); } break; default: @@ -470,18 +387,16 @@ Status ManifestFileAdapter::AppendPartitionSummary( switch (summary_field.field_id()) { case 509: // contains_null (required bool) ICEBERG_RETURN_UNEXPECTED( - AppendField(summary_array->children[0], - static_cast(summary.contains_null ? 1 : 0))); + AppendBoolean(summary_array->children[0], summary.contains_null)); break; case 518: { // contains_nan (optional bool) auto field_array = summary_array->children[1]; if (summary.contains_nan.has_value()) { ICEBERG_RETURN_UNEXPECTED( - AppendField(field_array, - static_cast(summary.contains_nan.value() ? 1 : 0))); + AppendBoolean(field_array, summary.contains_nan.value())); } else { - ICEBERG_NANOARROW_RETURN_UNEXPECTED(ArrowArrayAppendNull(field_array, 1)); + ICEBERG_RETURN_UNEXPECTED(AppendNull(field_array)); } break; } @@ -490,9 +405,9 @@ Status ManifestFileAdapter::AppendPartitionSummary( auto field_array = summary_array->children[2]; if (summary.lower_bound.has_value() && !summary.lower_bound->empty()) { ICEBERG_RETURN_UNEXPECTED( - AppendField(field_array, summary.lower_bound.value())); + AppendBytes(field_array, summary.lower_bound.value())); } else { - ICEBERG_NANOARROW_RETURN_UNEXPECTED(ArrowArrayAppendNull(field_array, 1)); + ICEBERG_RETURN_UNEXPECTED(AppendNull(field_array)); } break; } @@ -501,9 +416,9 @@ Status ManifestFileAdapter::AppendPartitionSummary( auto field_array = summary_array->children[3]; if (summary.upper_bound.has_value() && !summary.upper_bound->empty()) { ICEBERG_RETURN_UNEXPECTED( - AppendField(field_array, summary.upper_bound.value())); + AppendBytes(field_array, summary.upper_bound.value())); } else { - ICEBERG_NANOARROW_RETURN_UNEXPECTED(ArrowArrayAppendNull(field_array, 1)); + ICEBERG_RETURN_UNEXPECTED(AppendNull(field_array)); } break; } @@ -539,81 +454,81 @@ Status ManifestFileAdapter::AppendInternal(const ManifestFile& file) { auto array = array_.children[i]; switch (field.field_id()) { case ManifestFile::kManifestPathFieldId: - ICEBERG_RETURN_UNEXPECTED(AppendField(array, file.manifest_path)); + ICEBERG_RETURN_UNEXPECTED(AppendString(array, file.manifest_path)); break; case ManifestFile::kManifestLengthFieldId: - ICEBERG_RETURN_UNEXPECTED(AppendField(array, file.manifest_length)); + ICEBERG_RETURN_UNEXPECTED(AppendInt(array, file.manifest_length)); break; case ManifestFile::kPartitionSpecIdFieldId: ICEBERG_RETURN_UNEXPECTED( - AppendField(array, static_cast(file.partition_spec_id))); + AppendInt(array, static_cast(file.partition_spec_id))); break; case ManifestFile::kContentFieldId: ICEBERG_RETURN_UNEXPECTED( - AppendField(array, static_cast(static_cast(file.content)))); + AppendInt(array, static_cast(static_cast(file.content)))); break; case ManifestFile::kSequenceNumberFieldId: { ICEBERG_ASSIGN_OR_RAISE(auto sequence_num, GetSequenceNumber(file)); - ICEBERG_RETURN_UNEXPECTED(AppendField(array, sequence_num)); + ICEBERG_RETURN_UNEXPECTED(AppendInt(array, sequence_num)); break; } case ManifestFile::kMinSequenceNumberFieldId: { ICEBERG_ASSIGN_OR_RAISE(auto min_sequence_num, GetMinSequenceNumber(file)); - ICEBERG_RETURN_UNEXPECTED(AppendField(array, min_sequence_num)); + ICEBERG_RETURN_UNEXPECTED(AppendInt(array, min_sequence_num)); break; } case ManifestFile::kAddedSnapshotIdFieldId: - ICEBERG_RETURN_UNEXPECTED(AppendField(array, file.added_snapshot_id)); + ICEBERG_RETURN_UNEXPECTED(AppendInt(array, file.added_snapshot_id)); break; case ManifestFile::kAddedFilesCountFieldId: if (file.added_files_count.has_value()) { ICEBERG_RETURN_UNEXPECTED( - AppendField(array, static_cast(file.added_files_count.value()))); + AppendInt(array, static_cast(file.added_files_count.value()))); } else { // Append null for optional field - ICEBERG_NANOARROW_RETURN_UNEXPECTED(ArrowArrayAppendNull(array, 1)); + ICEBERG_RETURN_UNEXPECTED(AppendNull(array)); } break; case ManifestFile::kExistingFilesCountFieldId: if (file.existing_files_count.has_value()) { - ICEBERG_RETURN_UNEXPECTED(AppendField( - array, static_cast(file.existing_files_count.value()))); + ICEBERG_RETURN_UNEXPECTED( + AppendInt(array, static_cast(file.existing_files_count.value()))); } else { // Append null for optional field - ICEBERG_NANOARROW_RETURN_UNEXPECTED(ArrowArrayAppendNull(array, 1)); + ICEBERG_RETURN_UNEXPECTED(AppendNull(array)); } break; case ManifestFile::kDeletedFilesCountFieldId: if (file.deleted_files_count.has_value()) { ICEBERG_RETURN_UNEXPECTED( - AppendField(array, static_cast(file.deleted_files_count.value()))); + AppendInt(array, static_cast(file.deleted_files_count.value()))); } else { // Append null for optional field - ICEBERG_NANOARROW_RETURN_UNEXPECTED(ArrowArrayAppendNull(array, 1)); + ICEBERG_RETURN_UNEXPECTED(AppendNull(array)); } break; case ManifestFile::kAddedRowsCountFieldId: if (file.added_rows_count.has_value()) { - ICEBERG_RETURN_UNEXPECTED(AppendField(array, file.added_rows_count.value())); + ICEBERG_RETURN_UNEXPECTED(AppendInt(array, file.added_rows_count.value())); } else { // Append null for optional field - ICEBERG_NANOARROW_RETURN_UNEXPECTED(ArrowArrayAppendNull(array, 1)); + ICEBERG_RETURN_UNEXPECTED(AppendNull(array)); } break; case ManifestFile::kExistingRowsCountFieldId: if (file.existing_rows_count.has_value()) { - ICEBERG_RETURN_UNEXPECTED(AppendField(array, file.existing_rows_count.value())); + ICEBERG_RETURN_UNEXPECTED(AppendInt(array, file.existing_rows_count.value())); } else { // Append null for optional field - ICEBERG_NANOARROW_RETURN_UNEXPECTED(ArrowArrayAppendNull(array, 1)); + ICEBERG_RETURN_UNEXPECTED(AppendNull(array)); } break; case ManifestFile::kDeletedRowsCountFieldId: if (file.deleted_rows_count.has_value()) { - ICEBERG_RETURN_UNEXPECTED(AppendField(array, file.deleted_rows_count.value())); + ICEBERG_RETURN_UNEXPECTED(AppendInt(array, file.deleted_rows_count.value())); } else { // Append null for optional field - ICEBERG_NANOARROW_RETURN_UNEXPECTED(ArrowArrayAppendNull(array, 1)); + ICEBERG_RETURN_UNEXPECTED(AppendNull(array)); } break; case ManifestFile::kPartitionSummaryFieldId: @@ -623,18 +538,18 @@ Status ManifestFileAdapter::AppendInternal(const ManifestFile& file) { break; case ManifestFile::kKeyMetadataFieldId: if (!file.key_metadata.empty()) { - ICEBERG_RETURN_UNEXPECTED(AppendField(array, file.key_metadata)); + ICEBERG_RETURN_UNEXPECTED(AppendBytes(array, file.key_metadata)); } else { - ICEBERG_NANOARROW_RETURN_UNEXPECTED(ArrowArrayAppendNull(array, 1)); + ICEBERG_RETURN_UNEXPECTED(AppendNull(array)); } break; case ManifestFile::kFirstRowIdFieldId: { ICEBERG_ASSIGN_OR_RAISE(auto first_row_id, GetFirstRowId(file)); if (first_row_id.has_value()) { - ICEBERG_RETURN_UNEXPECTED(AppendField(array, first_row_id.value())); + ICEBERG_RETURN_UNEXPECTED(AppendInt(array, first_row_id.value())); } else { // Append null for optional field - ICEBERG_NANOARROW_RETURN_UNEXPECTED(ArrowArrayAppendNull(array, 1)); + ICEBERG_RETURN_UNEXPECTED(AppendNull(array)); } break; } diff --git a/src/iceberg/meson.build b/src/iceberg/meson.build index 4c7960ae63..1de293cb59 100644 --- a/src/iceberg/meson.build +++ b/src/iceberg/meson.build @@ -18,6 +18,7 @@ conf_data = configuration_data() version = meson.project_version() version_suffix = '-SNAPSHOT' +version_string = version + version_suffix components = version.split('.') assert( components.length() >= 3, @@ -29,7 +30,30 @@ ver_patch = components[2] conf_data.set('PROJECT_VERSION_MAJOR', ver_major) conf_data.set('PROJECT_VERSION_MINOR', ver_minor) conf_data.set('PROJECT_VERSION_PATCH', ver_patch) -conf_data.set('ICEBERG_VERSION_STRING', version + version_suffix) +conf_data.set('ICEBERG_VERSION_STRING', version_string) +git = find_program('git', required: false) +git_commit_id = 'unknown' +if git.found() + git_result = run_command( + git, + '-C', + meson.project_source_root(), + 'rev-parse', + 'HEAD', + check: false, + ) + if git_result.returncode() == 0 + git_commit_id_candidate = git_result.stdout().strip() + if git_commit_id_candidate != '' + git_commit_id = git_commit_id_candidate + endif + endif +endif +conf_data.set('ICEBERG_GIT_COMMIT_ID', git_commit_id) +conf_data.set( + 'ICEBERG_FULL_VERSION_STRING', + 'Apache Iceberg-CPP ' + version_string + ' (commit ' + git_commit_id + ')', +) conf_data.set('PROJECT_NAME', meson.project_name()) configure_file( input: 'version.h.in', @@ -103,6 +127,7 @@ iceberg_sources = files( 'partition_field.cc', 'partition_spec.cc', 'partition_summary.cc', + 'puffin_dv_io.cc', 'row/arrow_array_wrapper.cc', 'row/manifest_wrapper.cc', 'row/partition_values.cc', @@ -174,9 +199,12 @@ iceberg_data_sources = files( 'data/data_writer.cc', 'data/delete_filter.cc', 'data/delete_loader.cc', + 'data/deletion_vector_writer.cc', + 'data/dv_util.cc', 'data/equality_delete_writer.cc', 'data/file_scan_task_reader.cc', 'data/position_delete_writer.cc', + 'data/puffin_dv_register.cc', 'data/writer.cc', 'deletes/position_delete_index.cc', 'deletes/position_delete_range_consumer.cc', @@ -198,9 +226,27 @@ croaring_dep = dependency('croaring', static: croaring_needs_static) nanoarrow_dep = dependency('nanoarrow') nlohmann_json_dep = dependency('nlohmann_json') spdlog_dep = dependency('spdlog') +# utf8proc's header declares its functions __declspec(dllimport) on Windows unless +# UTF8PROC_STATIC is defined, and the wrap does not propagate that define to consumers. +# Define it whenever utf8proc is linked statically, so the header's declarations match +# how it is linked. Harmless on other platforms. +utf8proc_needs_static = get_option('default_library') == 'static' +utf8proc_dep = dependency('libutf8proc', static: utf8proc_needs_static) +if utf8proc_needs_static + utf8proc_dep = declare_dependency( + compile_args: ['-DUTF8PROC_STATIC'], + dependencies: utf8proc_dep, + ) +endif zlib_dep = dependency('zlib') -iceberg_deps = [nanoarrow_dep, nlohmann_json_dep, spdlog_dep, zlib_dep] +iceberg_deps = [ + nanoarrow_dep, + nlohmann_json_dep, + spdlog_dep, + utf8proc_dep, + zlib_dep, +] iceberg_lib = library( 'iceberg', @@ -319,3 +365,7 @@ subdir('inspect') if get_option('tests').enabled() subdir('test') endif + +if get_option('benchmarks').enabled() + subdir('benchmark') +endif diff --git a/src/iceberg/metrics/commit_report.h b/src/iceberg/metrics/commit_report.h index 63867c8e0c..a644d0b07e 100644 --- a/src/iceberg/metrics/commit_report.h +++ b/src/iceberg/metrics/commit_report.h @@ -19,6 +19,9 @@ #pragma once +/// \file iceberg/metrics/commit_report.h +/// \brief Define commit metrics and reports. + #include #include #include diff --git a/src/iceberg/metrics/counter.h b/src/iceberg/metrics/counter.h index 4542b90f0b..b44cf844b8 100644 --- a/src/iceberg/metrics/counter.h +++ b/src/iceberg/metrics/counter.h @@ -19,6 +19,9 @@ #pragma once +/// \file iceberg/metrics/counter.h +/// \brief Define metrics counters. + #include #include #include diff --git a/src/iceberg/metrics/metrics_reporter.h b/src/iceberg/metrics/metrics_reporter.h index b255a95e81..a45754dc0c 100644 --- a/src/iceberg/metrics/metrics_reporter.h +++ b/src/iceberg/metrics/metrics_reporter.h @@ -19,6 +19,9 @@ #pragma once +/// \file iceberg/metrics/metrics_reporter.h +/// \brief Define metrics report dispatch APIs. + #include #include #include diff --git a/src/iceberg/metrics/scan_report.h b/src/iceberg/metrics/scan_report.h index 2d81e7e8fe..1ac74c05fb 100644 --- a/src/iceberg/metrics/scan_report.h +++ b/src/iceberg/metrics/scan_report.h @@ -19,6 +19,9 @@ #pragma once +/// \file iceberg/metrics/scan_report.h +/// \brief Define scan metrics and reports. + #include #include #include diff --git a/src/iceberg/metrics/timer.h b/src/iceberg/metrics/timer.h index 08bbf8e34c..e15c623f34 100644 --- a/src/iceberg/metrics/timer.h +++ b/src/iceberg/metrics/timer.h @@ -19,6 +19,9 @@ #pragma once +/// \file iceberg/metrics/timer.h +/// \brief Define metrics timers. + #include #include #include diff --git a/src/iceberg/name_mapping.h b/src/iceberg/name_mapping.h index 392b573e34..7fd140c0d7 100644 --- a/src/iceberg/name_mapping.h +++ b/src/iceberg/name_mapping.h @@ -19,6 +19,9 @@ #pragma once +/// \file iceberg/name_mapping.h +/// \brief Define schema name mapping APIs. + #include #include #include diff --git a/src/iceberg/parquet/parquet_reader.h b/src/iceberg/parquet/parquet_reader.h index 0604230c8f..3a8e57cf3a 100644 --- a/src/iceberg/parquet/parquet_reader.h +++ b/src/iceberg/parquet/parquet_reader.h @@ -19,6 +19,9 @@ #pragma once +/// \file iceberg/parquet/parquet_reader.h +/// \brief Provide the Parquet file reader interface. + #include "iceberg/file_reader.h" #include "iceberg/iceberg_bundle_export.h" diff --git a/src/iceberg/parquet/parquet_writer.h b/src/iceberg/parquet/parquet_writer.h index 5abe45db95..562ffb8013 100644 --- a/src/iceberg/parquet/parquet_writer.h +++ b/src/iceberg/parquet/parquet_writer.h @@ -19,6 +19,9 @@ #pragma once +/// \file iceberg/parquet/parquet_writer.h +/// \brief Provide the Parquet file writer interface. + #include "iceberg/file_writer.h" #include "iceberg/iceberg_bundle_export.h" diff --git a/src/iceberg/puffin_dv_io.cc b/src/iceberg/puffin_dv_io.cc new file mode 100644 index 0000000000..477de630e0 --- /dev/null +++ b/src/iceberg/puffin_dv_io.cc @@ -0,0 +1,56 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +#include "iceberg/puffin_dv_io.h" + +#include + +#include "iceberg/util/macros.h" + +namespace iceberg { +namespace { + +PuffinDVIOFactory GetNotImplementedFactory() { + return []() -> Result> { + return NotImplemented("Missing Puffin DV I/O factory"); + }; +} + +} // namespace + +PuffinDVIO::~PuffinDVIO() = default; + +PuffinDVIORegistry::PuffinDVIORegistry(PuffinDVIOFactory factory) { + GetFactory() = std::move(factory); +} + +PuffinDVIOFactory& PuffinDVIORegistry::GetFactory() { + static auto* factory = new PuffinDVIOFactory(GetNotImplementedFactory()); + return *factory; +} + +Result>> PuffinDVIORegistry::MergeAndWriteDVs( + std::span groups, std::string_view output_path, + const std::shared_ptr& io) { + ICEBERG_ASSIGN_OR_RAISE(auto dv_io, GetFactory()()); + ICEBERG_PRECHECK(dv_io != nullptr, "Puffin DV I/O factory returned null"); + return dv_io->MergeAndWriteDVs(groups, output_path, io); +} + +} // namespace iceberg diff --git a/src/iceberg/puffin_dv_io.h b/src/iceberg/puffin_dv_io.h new file mode 100644 index 0000000000..9c42cae8e2 --- /dev/null +++ b/src/iceberg/puffin_dv_io.h @@ -0,0 +1,67 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +#pragma once + +/// \file iceberg/puffin_dv_io.h +/// Deletion vector merge hooks. + +#include +#include +#include +#include +#include +#include + +#include "iceberg/iceberg_export.h" +#include "iceberg/result.h" +#include "iceberg/row/partition_values.h" +#include "iceberg/type_fwd.h" + +namespace iceberg { + +struct ICEBERG_EXPORT DeletionVectorMergeGroup { + std::string referenced_data_file; + std::vector> delete_files; + std::shared_ptr spec; + PartitionValues partition; +}; + +class ICEBERG_EXPORT PuffinDVIO { + public: + virtual ~PuffinDVIO(); + + virtual Result>> MergeAndWriteDVs( + std::span groups, std::string_view output_path, + const std::shared_ptr& io) = 0; +}; + +using PuffinDVIOFactory = std::function>()>; + +struct ICEBERG_EXPORT PuffinDVIORegistry { + explicit PuffinDVIORegistry(PuffinDVIOFactory factory); + + static PuffinDVIOFactory& GetFactory(); + + static Result>> MergeAndWriteDVs( + std::span groups, std::string_view output_path, + const std::shared_ptr& io); +}; + +} // namespace iceberg diff --git a/src/iceberg/result.h b/src/iceberg/result.h index 4b4d0e7fb7..88b13c51ea 100644 --- a/src/iceberg/result.h +++ b/src/iceberg/result.h @@ -19,6 +19,9 @@ #pragma once +/// \file iceberg/result.h +/// \brief Define Result, Status, and error helpers. + #include #include #include diff --git a/src/iceberg/row/manifest_wrapper.h b/src/iceberg/row/manifest_wrapper.h index 20c2165b27..bd4fc4df68 100644 --- a/src/iceberg/row/manifest_wrapper.h +++ b/src/iceberg/row/manifest_wrapper.h @@ -19,7 +19,7 @@ #pragma once -/// \file iceberg/manifest_wrapper.h +/// \file iceberg/row/manifest_wrapper.h /// Wrapper classes for manifest-related data structures that implement /// StructLike, ArrayLike, and MapLike interfaces for unified data access. diff --git a/src/iceberg/schema_field.cc b/src/iceberg/schema_field.cc index 6c8d10d970..80ea61bce9 100644 --- a/src/iceberg/schema_field.cc +++ b/src/iceberg/schema_field.cc @@ -22,9 +22,11 @@ #include #include #include +#include #include "iceberg/expression/literal.h" #include "iceberg/type.h" +#include "iceberg/util/checked_cast.h" #include "iceberg/util/formatter.h" // IWYU pragma: keep #include "iceberg/util/macros.h" @@ -32,8 +34,7 @@ namespace iceberg { namespace { -// A null default value is modeled as the absence of a default (matching Java), so it is -// not stored. +// Treat null defaults as absent. std::shared_ptr DropNullDefault(std::shared_ptr value) { if (value != nullptr && value->IsNull()) { return nullptr; @@ -65,6 +66,55 @@ SchemaField SchemaField::MakeRequired(int32_t field_id, std::string_view name, return {field_id, name, std::move(type), false, doc}; } +SchemaField SchemaField::WithName(std::string_view name) const { + return {field_id_, name, type_, optional_, doc_, initial_default_, write_default_}; +} + +SchemaField SchemaField::WithType(std::shared_ptr type) const { + return {field_id_, name_, std::move(type), optional_, doc_, + initial_default_, write_default_}; +} + +SchemaField SchemaField::WithDoc(std::string_view doc) const { + return {field_id_, name_, type_, optional_, doc, initial_default_, write_default_}; +} + +SchemaField SchemaField::WithInitialDefault( + std::shared_ptr initial_default) const { + return {field_id_, name_, type_, optional_, doc_, std::move(initial_default), + write_default_}; +} + +SchemaField SchemaField::WithWriteDefault( + std::shared_ptr write_default) const { + return {field_id_, + name_, + type_, + optional_, + doc_, + initial_default_, + std::move(write_default)}; +} + +Result SchemaField::CastDefaultValue( + const Literal& value, const std::shared_ptr& target_type) { + if (target_type->type_id() == TypeId::kDecimal && + std::holds_alternative(value.value())) { + const auto& source_type = internal::checked_cast(*value.type()); + const auto& decimal_type = internal::checked_cast(*target_type); + if (source_type.scale() == decimal_type.scale()) { + const auto& decimal_value = std::get(value.value()); + if (!decimal_value.FitsInPrecision(decimal_type.precision())) { + return InvalidArgument("Cannot cast default value to {}: {}", *target_type, + value); + } + return Literal::Decimal(decimal_value.value(), decimal_type.precision(), + decimal_type.scale()); + } + } + return value.CastTo(target_type); +} + int32_t SchemaField::field_id() const { return field_id_; } std::string_view SchemaField::name() const { return name_; } @@ -87,8 +137,7 @@ namespace { Status ValidateDefault(const SchemaField& field, const Literal& value, std::string_view kind) { - // A null default is modeled as absence and dropped at construction, so it never reaches - // here; only the out-of-range cast sentinels need rejecting. + // Null defaults are dropped at construction. if (value.IsAboveMax() || value.IsBelowMin()) { return InvalidSchema("Invalid {} value for {}: value is out of range", kind, field.name()); @@ -97,8 +146,7 @@ Status ValidateDefault(const SchemaField& field, const Literal& value, return InvalidSchema("Invalid {} value for {}: field has no type", kind, field.name()); } - // The spec requires unknown/variant/geometry/geography columns to default to null, so a - // non-null default on them is invalid (a null default was already dropped as absence). + // These types can only use null defaults. switch (field.type()->type_id()) { case TypeId::kUnknown: case TypeId::kVariant: @@ -109,17 +157,13 @@ Status ValidateDefault(const SchemaField& field, const Literal& value, default: break; } - // Defaults are otherwise only supported on primitive fields. The spec also permits JSON - // single-value defaults for struct/list/map (e.g. an empty struct `{}` whose sub-field - // defaults live in field metadata); that matches the current Java model's gap and is - // left as a follow-up. + // Nested defaults are not supported yet. if (!field.type()->is_primitive()) { return InvalidSchema( "Invalid {} value for {}: default values are only supported for primitive types", kind, field.name()); } - // Defaults are stored verbatim (no implicit cast), so a default whose literal type does - // not match the field type is invalid. + // Stored defaults must already match the field type. if (*value.type() != *field.type()) { return InvalidSchema("{} of field {} has type {} but expected {}", kind, field.name(), *value.type(), *field.type()); diff --git a/src/iceberg/schema_field.h b/src/iceberg/schema_field.h index 8066a64065..86bfb92b63 100644 --- a/src/iceberg/schema_field.h +++ b/src/iceberg/schema_field.h @@ -46,10 +46,8 @@ class ICEBERG_EXPORT SchemaField : public iceberg::util::Formattable { /// \param[in] type The field type. /// \param[in] optional Whether values of this field are required or nullable. /// \param[in] doc Optional documentation string for the field. - /// \param[in] initial_default The v3 `initial-default` value, or null if absent. The - /// field shares ownership of the (immutable) value. - /// \param[in] write_default The v3 `write-default` value, or null if absent. The field - /// shares ownership of the (immutable) value. + /// \param[in] initial_default The v3 `initial-default`, or null if absent. + /// \param[in] write_default The v3 `write-default`, or null if absent. SchemaField(int32_t field_id, std::string_view name, std::shared_ptr type, bool optional, std::string_view doc = {}, std::shared_ptr initial_default = nullptr, @@ -62,14 +60,33 @@ class ICEBERG_EXPORT SchemaField : public iceberg::util::Formattable { static SchemaField MakeRequired(int32_t field_id, std::string_view name, std::shared_ptr type, std::string_view doc = {}); + /// \brief Return a copy with a new name. + SchemaField WithName(std::string_view name) const; + + /// \brief Return a copy with a new type. + SchemaField WithType(std::shared_ptr type) const; + + /// \brief Return a copy with new documentation. + SchemaField WithDoc(std::string_view doc) const; + + /// \brief Return a copy with a new `initial-default`. + SchemaField WithInitialDefault(std::shared_ptr initial_default) const; + + /// \brief Return a copy with a new `write-default`. + SchemaField WithWriteDefault(std::shared_ptr write_default) const; + + /// \brief Cast a default literal to a field type. + static Result CastDefaultValue( + const Literal& value, const std::shared_ptr& target_type); + /// \brief Get the field ID. - [[nodiscard]] int32_t field_id() const; + int32_t field_id() const; /// \brief Get the field name. - [[nodiscard]] std::string_view name() const; + std::string_view name() const; /// \brief Get the field type. - [[nodiscard]] const std::shared_ptr& type() const; + const std::shared_ptr& type() const; /// \brief Get whether the field is optional. [[nodiscard]] bool optional() const; @@ -77,15 +94,13 @@ class ICEBERG_EXPORT SchemaField : public iceberg::util::Formattable { /// \brief Get the field documentation. std::string_view doc() const; - /// \brief Get the owning pointer to the default value for this field used when reading - /// rows written before the field existed (v3 `initial-default`), or null if absent. + /// \brief Get the v3 `initial-default`, or null if absent. const std::shared_ptr& initial_default() const; - /// \brief Get the owning pointer to the default value for this field used when a writer - /// does not supply a value (v3 `write-default`), or null if absent. + /// \brief Get the v3 `write-default`, or null if absent. const std::shared_ptr& write_default() const; - [[nodiscard]] std::string ToString() const override; + std::string ToString() const override; Status Validate() const; @@ -107,14 +122,13 @@ class ICEBERG_EXPORT SchemaField : public iceberg::util::Formattable { private: /// \brief Compare two fields for equality. - [[nodiscard]] bool Equals(const SchemaField& other) const; + bool Equals(const SchemaField& other) const; int32_t field_id_; std::string name_; std::shared_ptr type_; bool optional_; std::string doc_; - // Immutable default values, shared (not deep-copied) across field copies, like `type_`. std::shared_ptr initial_default_; std::shared_ptr write_default_; }; diff --git a/src/iceberg/schema_util.h b/src/iceberg/schema_util.h index 9df4023365..421d00e0ad 100644 --- a/src/iceberg/schema_util.h +++ b/src/iceberg/schema_util.h @@ -19,6 +19,9 @@ #pragma once +/// \file iceberg/schema_util.h +/// \brief Provide schema projection utilities. + #include #include #include diff --git a/src/iceberg/snapshot.h b/src/iceberg/snapshot.h index c2fd9b8a29..e32f3a715e 100644 --- a/src/iceberg/snapshot.h +++ b/src/iceberg/snapshot.h @@ -19,6 +19,9 @@ #pragma once +/// \file iceberg/snapshot.h +/// \brief Define snapshot metadata types. + #include #include #include diff --git a/src/iceberg/sort_order.h b/src/iceberg/sort_order.h index 7c9b799fbe..75cabeca7e 100644 --- a/src/iceberg/sort_order.h +++ b/src/iceberg/sort_order.h @@ -19,6 +19,9 @@ #pragma once +/// \file iceberg/sort_order.h +/// \brief Define table sort order metadata. + #include #include #include diff --git a/src/iceberg/storage_credential.h b/src/iceberg/storage_credential.h index 604e8ddce4..746bad086c 100644 --- a/src/iceberg/storage_credential.h +++ b/src/iceberg/storage_credential.h @@ -19,6 +19,9 @@ #pragma once +/// \file iceberg/storage_credential.h +/// \brief Define storage credential metadata. + #include #include diff --git a/src/iceberg/table.h b/src/iceberg/table.h index b52b5d29d0..f9b72302a2 100644 --- a/src/iceberg/table.h +++ b/src/iceberg/table.h @@ -19,6 +19,9 @@ #pragma once +/// \file iceberg/table.h +/// \brief Define table abstractions and update entry points. + #include #include #include diff --git a/src/iceberg/table_metadata.cc b/src/iceberg/table_metadata.cc index 83ae47351f..94a154501b 100644 --- a/src/iceberg/table_metadata.cc +++ b/src/iceberg/table_metadata.cc @@ -71,7 +71,7 @@ Result> FreshPartitionSpec(int32_t spec_id, const Schema& fresh_schema) { std::vector partition_fields; partition_fields.reserve(spec.fields().size()); - int32_t last_partition_field_id = PartitionSpec::kInvalidPartitionFieldId; + int32_t last_partition_field_id = PartitionSpec::kLegacyPartitionDataIdStart - 1; for (auto& field : spec.fields()) { ICEBERG_ASSIGN_OR_RAISE(auto source_name, base_schema.FindColumnNameById(field.source_id())); @@ -556,7 +556,7 @@ class TableMetadataBuilder::Impl { metadata_.last_updated_ms = kInvalidLastUpdatedMs; metadata_.last_column_id = Schema::kInvalidColumnId; metadata_.default_spec_id = PartitionSpec::kInitialSpecId; - metadata_.last_partition_id = PartitionSpec::kInvalidPartitionFieldId; + metadata_.last_partition_id = PartitionSpec::kLegacyPartitionDataIdStart - 1; metadata_.current_snapshot_id = kInvalidSnapshotId; metadata_.default_sort_order_id = SortOrder::kInitialSortOrderId; metadata_.next_row_id = TableMetadata::kInitialRowId; diff --git a/src/iceberg/table_properties.h b/src/iceberg/table_properties.h index 6b92ea1fff..f334272e65 100644 --- a/src/iceberg/table_properties.h +++ b/src/iceberg/table_properties.h @@ -19,6 +19,9 @@ #pragma once +/// \file iceberg/table_properties.h +/// \brief Define table property helpers. + #include #include #include diff --git a/src/iceberg/table_scan.h b/src/iceberg/table_scan.h index e55330f307..f3e2f244d6 100644 --- a/src/iceberg/table_scan.h +++ b/src/iceberg/table_scan.h @@ -19,6 +19,9 @@ #pragma once +/// \file iceberg/table_scan.h +/// \brief Define table scan APIs and scan task types. + #include #include #include diff --git a/src/iceberg/test/CMakeLists.txt b/src/iceberg/test/CMakeLists.txt index b3592d8e2c..98129a6d2b 100644 --- a/src/iceberg/test/CMakeLists.txt +++ b/src/iceberg/test/CMakeLists.txt @@ -15,8 +15,6 @@ # specific language governing permissions and limitations # under the License. -set(CMAKE_COMPILE_WARNING_AS_ERROR FALSE) - set(INSTALL_GTEST OFF CACHE BOOL "" FORCE) @@ -256,6 +254,7 @@ if(ICEBERG_BUILD_BUNDLE) data_writer_test.cc delete_filter_test.cc delete_loader_test.cc + deletion_vector_writer_test.cc file_scan_task_reader_test.cc literal_util_test.cc) diff --git a/src/iceberg/test/delete_filter_test.cc b/src/iceberg/test/delete_filter_test.cc index 89d1b6b859..4b5e888e51 100644 --- a/src/iceberg/test/delete_filter_test.cc +++ b/src/iceberg/test/delete_filter_test.cc @@ -25,6 +25,7 @@ #include #include #include +#include #include #include @@ -34,9 +35,12 @@ #include #include "iceberg/arrow/arrow_io_internal.h" +#include "iceberg/data/deletion_vector_writer.h" #include "iceberg/data/equality_delete_writer.h" #include "iceberg/data/position_delete_writer.h" +#include "iceberg/deletes/position_delete_index.h" #include "iceberg/file_format.h" +#include "iceberg/file_io.h" #include "iceberg/file_reader.h" #include "iceberg/manifest/manifest_entry.h" #include "iceberg/metadata_columns.h" @@ -249,6 +253,25 @@ class DeleteFilterTest : public ::testing::Test { static constexpr std::string_view kDataPath = "data.parquet"; + Result> DeletionVectorFile( + const std::string& path, const std::vector& positions, + const std::string& data_path = std::string(kDataPath)) { + DeletionVectorWriterOptions options{ + .path = path, + .io = file_io_, + .load_previous_deletes = [](std::string_view) + -> Result> { return std::nullopt; }, + }; + ICEBERG_ASSIGN_OR_RAISE(auto writer, DeletionVectorWriter::Make(std::move(options))); + for (int64_t pos : positions) { + ICEBERG_RETURN_UNEXPECTED( + writer->Delete(data_path, pos, partition_spec_, PartitionValues{})); + } + ICEBERG_RETURN_UNEXPECTED(writer->Close()); + ICEBERG_ASSIGN_OR_RAISE(auto metadata, writer->Metadata()); + return metadata.data_files[0]; + } + std::shared_ptr file_io_; std::shared_ptr table_schema_; std::shared_ptr partition_spec_; @@ -1156,7 +1179,7 @@ TEST_F(DeleteFilterTest, DeletionVectorErrorPropagatesFromCompute) { ICEBERG_UNWRAP_OR_FAIL(auto batch, MakeBatch(*filter.value()->RequiredSchema(), R"([[1, 0]])")); auto alive = filter.value()->ComputeAliveRows(batch.schema, batch.array); - ASSERT_THAT(alive, IsError(ErrorKind::kNotSupported)); + ASSERT_THAT(alive, IsError(ErrorKind::kInvalidArgument)); } TEST_F(DeleteFilterTest, EmptyBatchPropagatesDeleteLoadErrors) { @@ -1175,7 +1198,27 @@ TEST_F(DeleteFilterTest, EmptyBatchPropagatesDeleteLoadErrors) { auto alive = filter.value()->ComputeAliveRows(batch.schema, batch.array); - ASSERT_THAT(alive, IsError(ErrorKind::kNotSupported)); + ASSERT_THAT(alive, IsError(ErrorKind::kInvalidArgument)); +} + +TEST_F(DeleteFilterTest, DeletionVectorComputeAliveRows) { + // Write a real deletion vector with DeletionVectorWriter, then load it through + // DeleteFilter (DeleteLoader::LoadDV) and verify deleted positions are filtered. + ICEBERG_UNWRAP_OR_FAIL(auto dv, DeletionVectorFile("dv-alive.puffin", {1, 3})); + std::vector> delete_files = {dv}; + auto requested_schema = Project({1}); + auto filter = DeleteFilter::Make(std::string(kDataPath), delete_files, table_schema_, + requested_schema, file_io_, + /*need_row_pos_col=*/true); + ASSERT_THAT(filter, IsOk()); + + ICEBERG_UNWRAP_OR_FAIL(auto batch, + MakeBatch(*filter.value()->RequiredSchema(), + R"([[10, 0], [20, 1], [30, 2], [40, 3], [50, 4]])")); + auto alive = filter.value()->ComputeAliveRows(batch.schema, batch.array); + + ASSERT_THAT(alive, IsOk()); + ExpectAliveRows(alive.value(), {0, 2, 4}); } TEST_F(DeleteFilterTest, CounterAccumulatesAcrossBatches) { diff --git a/src/iceberg/test/delete_loader_test.cc b/src/iceberg/test/delete_loader_test.cc index b392065cf0..a5dda22e99 100644 --- a/src/iceberg/test/delete_loader_test.cc +++ b/src/iceberg/test/delete_loader_test.cc @@ -19,6 +19,9 @@ #include "iceberg/data/delete_loader.h" +#include +#include + #include #include #include @@ -26,19 +29,23 @@ #include #include "iceberg/arrow/arrow_io_internal.h" +#include "iceberg/data/deletion_vector_writer.h" #include "iceberg/data/equality_delete_writer.h" #include "iceberg/data/position_delete_writer.h" #include "iceberg/deletes/position_delete_index.h" #include "iceberg/file_format.h" +#include "iceberg/file_io.h" #include "iceberg/manifest/manifest_entry.h" #include "iceberg/parquet/parquet_register.h" #include "iceberg/partition_spec.h" +#include "iceberg/result.h" #include "iceberg/row/partition_values.h" #include "iceberg/schema.h" #include "iceberg/schema_field.h" #include "iceberg/schema_internal.h" #include "iceberg/test/matchers.h" #include "iceberg/type.h" +#include "iceberg/util/macros.h" #include "iceberg/util/struct_like_set.h" namespace iceberg { @@ -110,6 +117,27 @@ class DeleteLoaderTest : public ::testing::Test { return writer->Metadata().value().data_files[0]; } + /// Write a deletion vector to a Puffin file and return the DataFile metadata. + std::shared_ptr WriteDeletionVector(const std::string& path, + const std::string& referenced_data_file, + const std::vector& positions) { + auto writer = + DeletionVectorWriter::Make(DeletionVectorWriterOptions{ + .path = path, + .io = file_io_, + .load_previous_deletes = [](std::string_view) + -> Result> { + return std::nullopt; + }}) + .value(); + for (int64_t pos : positions) { + ICEBERG_THROW_NOT_OK( + writer->Delete(referenced_data_file, pos, partition_spec_, PartitionValues{})); + } + ICEBERG_THROW_NOT_OK(writer->Close()); + return writer->Metadata().value().data_files[0]; + } + std::shared_ptr file_io_; std::shared_ptr schema_; std::shared_ptr partition_spec_; @@ -169,8 +197,7 @@ TEST_F(DeleteLoaderTest, LoadPositionDeletesFiltersByDataFilePath) { {"data_b.parquet", 10}, {"data_b.parquet", 20}}); - // Mixed paths -> writer must NOT set the hint, forcing the loader's - // per-row filter path. Locks the routing in case the writer behavior changes. + // Mixed paths keep the loader on the per-row filter path. ASSERT_FALSE(delete_file->referenced_data_file.has_value()); std::vector> files = {delete_file}; @@ -197,6 +224,46 @@ TEST_F(DeleteLoaderTest, LoadPositionDeletesFiltersByDataFilePath) { ASSERT_TRUE(result_none.value().IsEmpty()); } +// The loaded index keeps the source Parquet position delete file, so a DV +// writer can report file-scoped deletes as rewritten. +TEST_F(DeleteLoaderTest, LoadPositionDeletesTracksSourceDeleteFile) { + // Single referenced data file -> writer sets the hint -> fast path. + auto file_scoped = WritePositionDeletes("pos_src_scoped.parquet", + {{"data.parquet", 0}, {"data.parquet", 5}}); + ASSERT_TRUE(file_scoped->referenced_data_file.has_value()); + + std::vector> files = {file_scoped}; + auto result = loader_->LoadPositionDeletes(files, "data.parquet"); + ASSERT_THAT(result, IsOk()); + ASSERT_EQ(result.value().delete_files().size(), 1u); + EXPECT_EQ(result.value().delete_files()[0], file_scoped); +} + +TEST_F(DeleteLoaderTest, LoadPositionDeletesTracksSourceForFilterPath) { + auto mixed = WritePositionDeletes("pos_src_mixed.parquet", + {{"data_a.parquet", 0}, {"data_b.parquet", 10}}); + ASSERT_FALSE(mixed->referenced_data_file.has_value()); + + std::vector> files = {mixed}; + + auto matched = loader_->LoadPositionDeletes(files, "data_a.parquet"); + ASSERT_THAT(matched, IsOk()); + ASSERT_EQ(matched.value().delete_files().size(), 1u); + EXPECT_EQ(matched.value().delete_files()[0], mixed); + + auto unmatched = loader_->LoadPositionDeletes(files, "data_c.parquet"); + ASSERT_THAT(unmatched, IsOk()); + ASSERT_TRUE(unmatched.value().IsEmpty()); + ASSERT_EQ(unmatched.value().delete_files().size(), 1u); + EXPECT_EQ(unmatched.value().delete_files()[0], mixed); +} + +TEST_F(DeleteLoaderTest, LoadPositionDeletesRejectsNullFile) { + std::vector> files = {nullptr}; + auto result = loader_->LoadPositionDeletes(files, "data.parquet"); + ASSERT_THAT(result, IsError(ErrorKind::kInvalidArgument)); +} + TEST_F(DeleteLoaderTest, LoadPositionDeletesSkipsMismatchedReferencedDataFile) { auto delete_file = std::make_shared(DataFile{ .content = DataFile::Content::kPositionDeletes, @@ -212,10 +279,7 @@ TEST_F(DeleteLoaderTest, LoadPositionDeletesSkipsMismatchedReferencedDataFile) { } TEST_F(DeleteLoaderTest, LoadPositionDeletesFastPathHonorsReferencedDataFile) { - // Single-file writes -> writer sets referenced_data_file -> loader takes - // the zero-copy fast path. Sized above the consumer's 64-element sniff - // threshold so the dispatcher's real coalesce/bulk logic runs end-to-end, - // not just the small-input shortcut covered by LoadPositionDeletesSingleFile. + // Large enough to exercise the bulk path. constexpr int64_t kRowCount = 128; std::vector> deletes; deletes.reserve(kRowCount); @@ -239,15 +303,91 @@ TEST_F(DeleteLoaderTest, LoadPositionDeletesFastPathHonorsReferencedDataFile) { ASSERT_FALSE(index.IsDeleted(kRowCount)); } -TEST_F(DeleteLoaderTest, LoadPositionDeletesRejectsDV) { +TEST_F(DeleteLoaderTest, LoadDeletionVector) { + auto dv_file = + WriteDeletionVector("dv-a.puffin", "data.parquet", {0, 5, 10, 4'000'000'000LL}); + + std::vector> files = {dv_file}; + auto result = loader_->LoadPositionDeletes(files, "data.parquet"); + ASSERT_THAT(result, IsOk()); + + auto& index = result.value(); + ASSERT_EQ(index.Cardinality(), 4); + ASSERT_TRUE(index.IsDeleted(0)); + ASSERT_TRUE(index.IsDeleted(5)); + ASSERT_TRUE(index.IsDeleted(10)); + ASSERT_TRUE(index.IsDeleted(4'000'000'000LL)); + ASSERT_FALSE(index.IsDeleted(1)); +} + +TEST_F(DeleteLoaderTest, LoadDeletionVectorRejectsMismatchedReferencedDataFile) { + // A DV referencing another data file must not be silently skipped: the single + // DV applies to its referenced file, so a mismatch is bad grouping/metadata. + auto dv_file = WriteDeletionVector("dv-b.puffin", "other-data.parquet", {1, 2, 3}); + + std::vector> files = {dv_file}; + auto result = loader_->LoadPositionDeletes(files, "data.parquet"); + ASSERT_THAT(result, IsError(ErrorKind::kInvalidArgument)); +} + +TEST_F(DeleteLoaderTest, LoadDeletionVectorRequiresContentOffsetAndSize) { auto dv_file = std::make_shared(DataFile{ .content = DataFile::Content::kPositionDeletes, .file_path = "dv.puffin", .file_format = FileFormatType::kPuffin, + .referenced_data_file = "data.parquet", }); std::vector> files = {dv_file}; auto result = loader_->LoadPositionDeletes(files, "data.parquet"); - ASSERT_THAT(result, IsError(ErrorKind::kNotSupported)); + ASSERT_THAT(result, IsError(ErrorKind::kInvalidArgument)); +} + +TEST_F(DeleteLoaderTest, LoadDeletionVectorRejectsCardinalityMismatch) { + auto dv_file = WriteDeletionVector("dv-card.puffin", "data.parquet", {0, 1, 2}); + // Corrupt the recorded cardinality so it no longer matches the bitmap. + dv_file->record_count = 99; + + std::vector> files = {dv_file}; + auto result = loader_->LoadPositionDeletes(files, "data.parquet"); + ASSERT_THAT(result, IsError(ErrorKind::kInvalidArgument)); +} + +// End to end: a previously written Parquet position delete is loaded, merged +// into a new DV, and reported in rewritten_delete_files when it is file-scoped. +TEST_F(DeleteLoaderTest, DVWriterReportsFileScopedPositionDeleteAsRewritten) { + // A file-scoped position delete (single referenced data file). + auto previous = WritePositionDeletes("prev_pos.parquet", + {{"data.parquet", 100}, {"data.parquet", 200}}); + ASSERT_TRUE(previous->referenced_data_file.has_value()); + + auto* loader = loader_.get(); + ICEBERG_UNWRAP_OR_FAIL( + auto writer, + DeletionVectorWriter::Make(DeletionVectorWriterOptions{ + .path = "dv-replace.puffin", + .io = file_io_, + .load_previous_deletes = + [&](std::string_view path) -> Result> { + std::vector> files = {previous}; + ICEBERG_ASSIGN_OR_RAISE(auto index, loader->LoadPositionDeletes(files, path)); + if (index.IsEmpty()) { + return std::nullopt; + } + return std::optional(std::move(index)); + }, + })); + + ICEBERG_THROW_NOT_OK( + writer->Delete("data.parquet", 0, partition_spec_, PartitionValues{})); + ICEBERG_THROW_NOT_OK(writer->Close()); + + ICEBERG_UNWRAP_OR_FAIL(auto result, writer->Metadata()); + ASSERT_EQ(result.data_files.size(), 1u); + // New position plus the two previous positions. + EXPECT_EQ(result.data_files[0]->record_count, 3); + // The previous Parquet position delete is file-scoped and must be reported. + ASSERT_EQ(result.rewritten_delete_files.size(), 1u); + EXPECT_EQ(result.rewritten_delete_files[0], previous); } TEST_F(DeleteLoaderTest, LoadPositionDeletesRejectsWrongContent) { diff --git a/src/iceberg/test/deletion_vector_writer_test.cc b/src/iceberg/test/deletion_vector_writer_test.cc new file mode 100644 index 0000000000..605b2d4e1a --- /dev/null +++ b/src/iceberg/test/deletion_vector_writer_test.cc @@ -0,0 +1,396 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +#include "iceberg/data/deletion_vector_writer.h" + +#include +#include +#include +#include +#include +#include +#include +#include + +#include + +#include "iceberg/data/delete_loader.h" +#include "iceberg/deletes/position_delete_index.h" +#include "iceberg/deletes/roaring_position_bitmap.h" +#include "iceberg/file_format.h" +#include "iceberg/manifest/manifest_entry.h" +#include "iceberg/metadata_columns.h" +#include "iceberg/partition_spec.h" +#include "iceberg/puffin/file_metadata.h" +#include "iceberg/puffin/puffin_reader.h" +#include "iceberg/result.h" +#include "iceberg/row/partition_values.h" +#include "iceberg/test/matchers.h" +#include "iceberg/test/mock_io.h" +#include "iceberg/util/macros.h" +#include "iceberg/version.h" + +namespace iceberg { + +namespace { + +std::shared_ptr FindByReferencedFile( + const std::vector>& files, const std::string& ref) { + for (const auto& file : files) { + if (file->referenced_data_file == ref) { + return file; + } + } + return nullptr; +} + +std::shared_ptr UnpartitionedSpec() { + return PartitionSpec::Unpartitioned(); +} + +const puffin::BlobMetadata* FindBlobByReferencedFile( + const std::vector& blobs, const std::string& ref) { + for (const auto& blob : blobs) { + auto it = blob.properties.find("referenced-data-file"); + if (it != blob.properties.end() && it->second == ref) { + return &blob; + } + } + return nullptr; +} + +// A load_previous_deletes callback for data files that have no existing deletes. +Result> NoPreviousDeletes(std::string_view) { + return std::nullopt; +} + +DeletionVectorWriterOptions MakeDVWriterOptions( + std::shared_ptr io, std::string path, + std::function>(std::string_view)> + load_previous_deletes = NoPreviousDeletes) { + return DeletionVectorWriterOptions{ + .path = std::move(path), + .io = std::move(io), + .load_previous_deletes = std::move(load_previous_deletes), + }; +} + +} // namespace + +TEST(DeletionVectorWriterTest, WriteThenLoadEndToEnd) { + auto io = std::make_shared(); + auto spec = UnpartitionedSpec(); + + std::vector> delete_files; + { + ICEBERG_UNWRAP_OR_FAIL(auto writer, DeletionVectorWriter::Make(MakeDVWriterOptions( + io, "memory://deletes.puffin"))); + + ASSERT_THAT(writer->Delete("data-a.parquet", 0, spec, PartitionValues{}), IsOk()); + ASSERT_THAT(writer->Delete("data-a.parquet", 5, spec, PartitionValues{}), IsOk()); + ASSERT_THAT(writer->Delete("data-a.parquet", 10, spec, PartitionValues{}), IsOk()); + ASSERT_THAT(writer->Delete("data-b.parquet", 1, spec, PartitionValues{}), IsOk()); + ASSERT_THAT(writer->Delete("data-b.parquet", 2, spec, PartitionValues{}), IsOk()); + ASSERT_THAT(writer->Close(), IsOk()); + + ICEBERG_UNWRAP_OR_FAIL(auto result, writer->Metadata()); + delete_files = result.data_files; + // Each referenced data file is reported once. + EXPECT_EQ(result.referenced_data_files.size(), 2u); + // No previous deletes were loaded, so nothing was rewritten. + EXPECT_TRUE(result.rewritten_delete_files.empty()); + } + + // One DataFile per referenced data file. + ASSERT_EQ(delete_files.size(), 2u); + + auto dv_a = FindByReferencedFile(delete_files, "data-a.parquet"); + auto dv_b = FindByReferencedFile(delete_files, "data-b.parquet"); + ASSERT_NE(dv_a, nullptr); + ASSERT_NE(dv_b, nullptr); + + EXPECT_EQ(dv_a->content, DataFile::Content::kPositionDeletes); + EXPECT_EQ(dv_a->file_format, FileFormatType::kPuffin); + EXPECT_TRUE(dv_a->IsDeletionVector()); + EXPECT_EQ(dv_a->file_path, "memory://deletes.puffin"); + EXPECT_EQ(dv_a->record_count, 3); + EXPECT_TRUE(dv_a->content_offset.has_value()); + EXPECT_TRUE(dv_a->content_size_in_bytes.has_value()); + EXPECT_GT(dv_a->file_size_in_bytes, 0); + EXPECT_EQ(dv_a->partition_spec_id, spec->spec_id()); + EXPECT_EQ(dv_b->record_count, 2); + + // Both blobs live in the same Puffin file but at different offsets. + EXPECT_EQ(dv_a->file_path, dv_b->file_path); + EXPECT_NE(dv_a->content_offset.value(), dv_b->content_offset.value()); + + ICEBERG_UNWRAP_OR_FAIL(auto input_file, io->NewInputFile("memory://deletes.puffin")); + ICEBERG_UNWRAP_OR_FAIL(auto puffin_reader, + puffin::PuffinReader::Make(std::move(input_file))); + ICEBERG_UNWRAP_OR_FAIL(auto puffin_metadata, puffin_reader->ReadFileMetadata()); + ASSERT_EQ(puffin_metadata.blobs.size(), 2u); + EXPECT_EQ(puffin_metadata.properties.at("created-by"), ICEBERG_FULL_VERSION_STRING); + + auto assert_blob = [](const puffin::BlobMetadata& blob, const DataFile& file, + const std::string& ref, const std::string& cardinality) { + EXPECT_EQ(blob.type, puffin::StandardBlobTypes::kDeletionVectorV1); + EXPECT_EQ(blob.input_fields, + std::vector{MetadataColumns::kFilePositionColumnId}); + EXPECT_EQ(blob.snapshot_id, -1); + EXPECT_EQ(blob.sequence_number, -1); + EXPECT_TRUE(blob.compression_codec.empty()); + EXPECT_EQ(blob.offset, file.content_offset.value()); + EXPECT_EQ(blob.length, file.content_size_in_bytes.value()); + EXPECT_EQ(blob.properties.at("referenced-data-file"), ref); + EXPECT_EQ(blob.properties.at("cardinality"), cardinality); + }; + ASSERT_NE(FindBlobByReferencedFile(puffin_metadata.blobs, "data-a.parquet"), nullptr); + ASSERT_NE(FindBlobByReferencedFile(puffin_metadata.blobs, "data-b.parquet"), nullptr); + assert_blob(*FindBlobByReferencedFile(puffin_metadata.blobs, "data-a.parquet"), *dv_a, + "data-a.parquet", "3"); + assert_blob(*FindBlobByReferencedFile(puffin_metadata.blobs, "data-b.parquet"), *dv_b, + "data-b.parquet", "2"); + + // Load each data file's DV separately. + DeleteLoader loader(io); + { + auto result = loader.LoadPositionDeletes({&dv_a, 1}, "data-a.parquet"); + ASSERT_THAT(result, IsOk()); + auto& index = result.value(); + EXPECT_EQ(index.Cardinality(), 3); + EXPECT_TRUE(index.IsDeleted(0)); + EXPECT_TRUE(index.IsDeleted(5)); + EXPECT_TRUE(index.IsDeleted(10)); + EXPECT_FALSE(index.IsDeleted(1)); + } + + { + auto result = loader.LoadPositionDeletes({&dv_b, 1}, "data-b.parquet"); + ASSERT_THAT(result, IsOk()); + auto& index = result.value(); + EXPECT_EQ(index.Cardinality(), 2); + EXPECT_TRUE(index.IsDeleted(1)); + EXPECT_TRUE(index.IsDeleted(2)); + EXPECT_FALSE(index.IsDeleted(0)); + } +} + +// The PositionDeleteIndex overload bulk-adds positions for a data file. +TEST(DeletionVectorWriterTest, DeleteFromIndex) { + auto io = std::make_shared(); + auto spec = UnpartitionedSpec(); + + PositionDeleteIndex positions; + positions.Delete(0); + positions.Delete(3, 6); // [3, 6) -> 3, 4, 5 + + ICEBERG_UNWRAP_OR_FAIL(auto writer, DeletionVectorWriter::Make(MakeDVWriterOptions( + io, "memory://from-index.puffin"))); + ASSERT_THAT(writer->Delete("data.parquet", positions, spec, PartitionValues{}), IsOk()); + ASSERT_THAT(writer->Close(), IsOk()); + + ICEBERG_UNWRAP_OR_FAIL(auto result, writer->Metadata()); + ASSERT_EQ(result.data_files.size(), 1u); + EXPECT_EQ(result.data_files[0]->record_count, 4); + + DeleteLoader loader(io); + auto loaded = loader.LoadPositionDeletes(result.data_files, "data.parquet"); + ASSERT_THAT(loaded, IsOk()); + EXPECT_EQ(loaded.value().Cardinality(), 4); + EXPECT_TRUE(loaded.value().IsDeleted(0)); + EXPECT_TRUE(loaded.value().IsDeleted(5)); + EXPECT_FALSE(loaded.value().IsDeleted(6)); +} + +// Previously written deletes are merged into the new vector, and the file-scoped +// delete files they came from are reported as rewritten. +TEST(DeletionVectorWriterTest, LoadPreviousDeletesMergesAndReportsRewritten) { + auto io = std::make_shared(); + auto spec = UnpartitionedSpec(); + + // Build a previous DV index carrying its source (file-scoped) delete file, + // the same way DeleteLoader would produce it. + PositionDeleteIndex previous_positions; + previous_positions.Delete(100); + previous_positions.Delete(200); + ICEBERG_UNWRAP_OR_FAIL(auto previous_blob, previous_positions.Serialize()); + auto previous_dv = std::make_shared(DataFile{ + .content = DataFile::Content::kPositionDeletes, + .file_path = "memory://old.puffin", + .file_format = FileFormatType::kPuffin, + .record_count = 2, + .referenced_data_file = "data.parquet", + .content_offset = 0, + .content_size_in_bytes = static_cast(previous_blob.size()), + }); + + ICEBERG_UNWRAP_OR_FAIL( + auto writer, + DeletionVectorWriter::Make(MakeDVWriterOptions( + io, "memory://merged.puffin", + [&](std::string_view path) -> Result> { + if (path != "data.parquet") { + return std::nullopt; + } + ICEBERG_ASSIGN_OR_RAISE( + auto index, PositionDeleteIndex::Deserialize(previous_blob, previous_dv)); + return std::optional(std::move(index)); + }))); + + ASSERT_THAT(writer->Delete("data.parquet", 0, spec, PartitionValues{}), IsOk()); + ASSERT_THAT(writer->Close(), IsOk()); + + ICEBERG_UNWRAP_OR_FAIL(auto result, writer->Metadata()); + ASSERT_EQ(result.data_files.size(), 1u); + // New position plus the two previous positions. + EXPECT_EQ(result.data_files[0]->record_count, 3); + // The previous DV is file-scoped, so it is reported for removal. + ASSERT_EQ(result.rewritten_delete_files.size(), 1u); + EXPECT_EQ(result.rewritten_delete_files[0]->file_path, "memory://old.puffin"); + + DeleteLoader loader(io); + auto loaded = loader.LoadPositionDeletes(result.data_files, "data.parquet"); + ASSERT_THAT(loaded, IsOk()); + EXPECT_EQ(loaded.value().Cardinality(), 3); + EXPECT_TRUE(loaded.value().IsDeleted(0)); + EXPECT_TRUE(loaded.value().IsDeleted(100)); + EXPECT_TRUE(loaded.value().IsDeleted(200)); +} + +TEST(DeletionVectorWriterTest, PreviousDeletesWithoutSourceFilesAreNotRewritten) { + auto io = std::make_shared(); + auto spec = UnpartitionedSpec(); + + ICEBERG_UNWRAP_OR_FAIL( + auto writer, + DeletionVectorWriter::Make(MakeDVWriterOptions( + io, "memory://merged-partition.puffin", + [&](std::string_view path) -> Result> { + PositionDeleteIndex index; + index.Delete(50); + return std::optional(std::move(index)); + }))); + + ASSERT_THAT(writer->Delete("data.parquet", 0, spec, PartitionValues{}), IsOk()); + ASSERT_THAT(writer->Close(), IsOk()); + + ICEBERG_UNWRAP_OR_FAIL(auto result, writer->Metadata()); + ASSERT_EQ(result.data_files.size(), 1u); + // The previous position was merged in. + EXPECT_EQ(result.data_files[0]->record_count, 2); + // The previous delete is partition-scoped, so it is not rewritten. + EXPECT_TRUE(result.rewritten_delete_files.empty()); + + DeleteLoader loader(io); + auto loaded = loader.LoadPositionDeletes(result.data_files, "data.parquet"); + ASSERT_THAT(loaded, IsOk()); + EXPECT_EQ(loaded.value().Cardinality(), 2); + EXPECT_TRUE(loaded.value().IsDeleted(0)); + EXPECT_TRUE(loaded.value().IsDeleted(50)); +} + +TEST(DeletionVectorWriterTest, EmptyWriterProducesNoDataFiles) { + auto io = std::make_shared(); + ICEBERG_UNWRAP_OR_FAIL(auto writer, + DeletionVectorWriter::Make(DeletionVectorWriterOptions{ + .path = "memory://empty.puffin", + .io = io, + .load_previous_deletes = NoPreviousDeletes})); + ASSERT_THAT(writer->Close(), IsOk()); + ICEBERG_UNWRAP_OR_FAIL(auto result, writer->Metadata()); + EXPECT_TRUE(result.data_files.empty()); + EXPECT_THAT(io->NewInputFile("memory://empty.puffin"), IsError(ErrorKind::kNotFound)); +} + +TEST(DeletionVectorWriterTest, DeleteRejectsEmptyReferencedFile) { + auto io = std::make_shared(); + auto spec = UnpartitionedSpec(); + ICEBERG_UNWRAP_OR_FAIL(auto writer, DeletionVectorWriter::Make(MakeDVWriterOptions( + io, "memory://invalid.puffin"))); + EXPECT_THAT(writer->Delete("", 0, spec, PartitionValues{}), + IsError(ErrorKind::kInvalidArgument)); + EXPECT_THAT(writer->Delete("data.parquet", 0, nullptr, PartitionValues{}), + IsError(ErrorKind::kInvalidArgument)); + PositionDeleteIndex positions; + positions.Delete(0); + EXPECT_THAT(writer->Delete("data.parquet", positions, nullptr, PartitionValues{}), + IsError(ErrorKind::kInvalidArgument)); +} + +TEST(DeletionVectorWriterTest, DeleteRejectsOutOfRangePosition) { + auto io = std::make_shared(); + auto spec = UnpartitionedSpec(); + ICEBERG_UNWRAP_OR_FAIL(auto writer, DeletionVectorWriter::Make(MakeDVWriterOptions( + io, "memory://invalid.puffin"))); + // Negative and out-of-range positions are rejected rather than silently + // dropped by the underlying bitmap. + EXPECT_THAT(writer->Delete("data.parquet", -1, spec, PartitionValues{}), + IsError(ErrorKind::kInvalidArgument)); + EXPECT_THAT(writer->Delete("data.parquet", RoaringPositionBitmap::kMaxPosition + 1, + spec, PartitionValues{}), + IsError(ErrorKind::kInvalidArgument)); +} + +// Close propagates a load_previous_deletes failure and returns no metadata. +TEST(DeletionVectorWriterTest, ClosePropagatesLoadPreviousDeletesError) { + auto io = std::make_shared(); + auto spec = UnpartitionedSpec(); + ICEBERG_UNWRAP_OR_FAIL( + auto writer, + DeletionVectorWriter::Make(MakeDVWriterOptions( + io, "memory://err.puffin", + [](std::string_view) -> Result> { + return IOError("boom"); + }))); + + ASSERT_THAT(writer->Delete("data.parquet", 0, spec, PartitionValues{}), IsOk()); + EXPECT_THAT(writer->Close(), IsError(ErrorKind::kIOError)); + // No partial delete metadata is exposed after a failed Close. + EXPECT_THAT(writer->Metadata(), IsError(ErrorKind::kValidationFailed)); + // No orphan Puffin file was created. + EXPECT_THAT(io->NewInputFile("memory://err.puffin"), IsError(ErrorKind::kNotFound)); +} + +TEST(DeletionVectorWriterTest, MakeRejectsMissingOutputPath) { + EXPECT_THAT(DeletionVectorWriter::Make(DeletionVectorWriterOptions{ + .io = std::make_shared(), + .load_previous_deletes = NoPreviousDeletes}), + IsError(ErrorKind::kInvalidArgument)); +} + +TEST(DeletionVectorWriterTest, MakeRejectsMissingLoadPreviousDeletes) { + auto io = std::make_shared(); + EXPECT_THAT(DeletionVectorWriter::Make(DeletionVectorWriterOptions{ + .path = "x.puffin", + .io = io, + }), + IsError(ErrorKind::kInvalidArgument)); +} + +TEST(DeletionVectorWriterTest, DeleteAfterCloseFails) { + auto io = std::make_shared(); + auto spec = UnpartitionedSpec(); + ICEBERG_UNWRAP_OR_FAIL(auto writer, DeletionVectorWriter::Make(MakeDVWriterOptions( + io, "memory://closed.puffin"))); + ASSERT_THAT(writer->Close(), IsOk()); + EXPECT_THAT(writer->Delete("data-a.parquet", 0, spec, PartitionValues{}), + IsError(ErrorKind::kValidationFailed)); +} + +} // namespace iceberg diff --git a/src/iceberg/test/endian_test.cc b/src/iceberg/test/endian_test.cc index 717baeafee..93aadde4e9 100644 --- a/src/iceberg/test/endian_test.cc +++ b/src/iceberg/test/endian_test.cc @@ -94,6 +94,9 @@ TEST(EndianTest, BufferReadWriteRoundTrip) { WriteLittleEndian(int32_t{-1}, buf.data()); EXPECT_EQ(ReadLittleEndian(buf.data()), -1); + + WriteBigEndian(int32_t{0x12345678}, buf.data()); + EXPECT_EQ(ReadBigEndian(buf.data()), 0x12345678); } } // namespace iceberg diff --git a/src/iceberg/test/file_scan_task_reader_test.cc b/src/iceberg/test/file_scan_task_reader_test.cc index 1630a108ab..10e9c8e5f6 100644 --- a/src/iceberg/test/file_scan_task_reader_test.cc +++ b/src/iceberg/test/file_scan_task_reader_test.cc @@ -36,11 +36,15 @@ #include "iceberg/arrow/arrow_io_internal.h" #include "iceberg/arrow/arrow_register.h" +#include "iceberg/arrow_c_data.h" #include "iceberg/arrow_c_data_guard_internal.h" #include "iceberg/arrow_c_data_util_internal.h" +#include "iceberg/data/deletion_vector_writer.h" #include "iceberg/data/equality_delete_writer.h" #include "iceberg/data/position_delete_writer.h" +#include "iceberg/deletes/position_delete_index.h" #include "iceberg/file_format.h" +#include "iceberg/file_io.h" #include "iceberg/file_reader.h" #include "iceberg/manifest/manifest_entry.h" #include "iceberg/parquet/parquet_register.h" @@ -298,6 +302,25 @@ class FileScanTaskReaderTest : public TempFileTestBase { return metadata.data_files[0]; } + Result> MakeDeletionVectorFile( + const std::string& path, const std::vector& positions, + const std::string& data_path) { + DeletionVectorWriterOptions options{ + .path = path, + .io = file_io_, + .load_previous_deletes = [](std::string_view) + -> Result> { return std::nullopt; }, + }; + ICEBERG_ASSIGN_OR_RAISE(auto writer, DeletionVectorWriter::Make(std::move(options))); + for (int64_t pos : positions) { + ICEBERG_RETURN_UNEXPECTED( + writer->Delete(data_path, pos, partition_spec_, PartitionValues{})); + } + ICEBERG_RETURN_UNEXPECTED(writer->Close()); + ICEBERG_ASSIGN_OR_RAISE(auto metadata, writer->Metadata()); + return metadata.data_files[0]; + } + void VerifyStream(struct ArrowArrayStream* stream, std::string_view expected_json) { auto record_batch_reader = ::arrow::ImportRecordBatchReader(stream).ValueOrDie(); @@ -401,6 +424,30 @@ TEST_F(FileScanTaskReaderTest, OpenWithPositionDeletesFiltersRowsAndPrunesPos) { ASSERT_NO_FATAL_FAILURE(VerifyStream(&stream, R"([[1, "Foo"], [3, "Baz"]])")); } +TEST_F(FileScanTaskReaderTest, OpenWithDeletionVectorFiltersRows) { + ICEBERG_UNWRAP_OR_FAIL( + auto data_file, + MakeDataFile(table_schema_, + R"([[1, "Foo", "blue"], [2, "Bar", "red"], [3, "Baz", "green"]])")); + ICEBERG_UNWRAP_OR_FAIL( + auto dv, MakeDeletionVectorFile(CreateNewTempFilePathWithSuffix(".puffin"), {1}, + data_file->file_path)); + FileScanTask task(data_file, {dv}); + + FileScanTaskReader::Options options{ + .io = file_io_, + .table_schema = table_schema_, + .schemas = {table_schema_}, + .projected_schema = projected_schema_, + }; + ICEBERG_UNWRAP_OR_FAIL(auto reader, FileScanTaskReader::Make(std::move(options))); + auto stream_result = reader->Open(task); + ASSERT_THAT(stream_result, IsOk()); + auto stream = std::move(stream_result.value()); + + ASSERT_NO_FATAL_FAILURE(VerifyStream(&stream, R"([[1, "Foo"], [3, "Baz"]])")); +} + TEST_F(FileScanTaskReaderTest, OpenWithEqualityDeletesAddsAndPrunesDeleteOnlyColumns) { ICEBERG_UNWRAP_OR_FAIL( auto data_file, diff --git a/src/iceberg/test/merging_snapshot_update_test.cc b/src/iceberg/test/merging_snapshot_update_test.cc index 5306ba1b38..6684cf221c 100644 --- a/src/iceberg/test/merging_snapshot_update_test.cc +++ b/src/iceberg/test/merging_snapshot_update_test.cc @@ -38,6 +38,7 @@ #include "iceberg/manifest/manifest_reader.h" #include "iceberg/manifest/manifest_writer.h" #include "iceberg/partition_spec.h" +#include "iceberg/puffin_dv_io.h" #include "iceberg/row/partition_values.h" #include "iceberg/schema.h" #include "iceberg/snapshot.h" @@ -57,6 +58,55 @@ namespace iceberg { +class RecordingPuffinDVIO final : public PuffinDVIO { + public: + struct Call { + std::string output_path; + std::vector groups; + }; + + Result>> MergeAndWriteDVs( + std::span groups, std::string_view output_path, + const std::shared_ptr& /*io*/) override { + calls.push_back(Call{.output_path = std::string(output_path), + .groups = {groups.begin(), groups.end()}}); + + std::vector> result; + result.reserve(groups.size()); + for (const auto& group : groups) { + auto file = std::make_shared(*group.delete_files.front()); + file->file_path = std::string(output_path); + file->file_format = FileFormatType::kPuffin; + file->referenced_data_file = group.referenced_data_file; + file->record_count = 0; + for (const auto& delete_file : group.delete_files) { + file->record_count += delete_file->record_count; + } + file->content_offset = static_cast(result.size()) * 100; + file->content_size_in_bytes = 100; + result.push_back(std::move(file)); + } + return result; + } + + std::vector calls; +}; + +class ScopedPuffinDVIORegistry { + public: + explicit ScopedPuffinDVIORegistry(PuffinDVIOFactory factory) + : previous_factory_(std::move(PuffinDVIORegistry::GetFactory())) { + PuffinDVIORegistry::GetFactory() = std::move(factory); + } + + ~ScopedPuffinDVIORegistry() { + PuffinDVIORegistry::GetFactory() = std::move(previous_factory_); + } + + private: + PuffinDVIOFactory previous_factory_; +}; + /// \brief Concrete subclass of MergingSnapshotUpdate for testing. class TestMergeAppend : public MergingSnapshotUpdate { public: @@ -264,6 +314,20 @@ class MergingSnapshotUpdateTest : public MinimalUpdateTestBase { return f; } + std::shared_ptr MakeDeletionVector(const std::string& path, + const std::shared_ptr& data_file, + int64_t record_count) { + auto f = MakeDeleteFile(path, 0); + f->file_format = FileFormatType::kPuffin; + f->partition = data_file->partition; + f->partition_spec_id = data_file->partition_spec_id; + f->referenced_data_file = data_file->file_path; + f->record_count = record_count; + f->content_offset = 0; + f->content_size_in_bytes = 100; + return f; + } + std::shared_ptr MakeEqualityDeleteFile(const std::string& path, int64_t partition_x) { auto f = MakeDeleteFile(path, partition_x); @@ -978,6 +1042,78 @@ TEST_F(MergingSnapshotUpdateTest, ValidateNewDeleteFileV3AllowsDeletionVector) { EXPECT_THAT(op->AddDelete(del_file), IsOk()); } +TEST_F(MergingSnapshotUpdateTest, ApplyMergesDuplicateDeletionVectors) { + SetTableFormatVersion(3); + + auto dv_io = std::make_shared(); + ScopedPuffinDVIORegistry registry( + [dv_io]() -> Result> { return dv_io; }); + ICEBERG_UNWRAP_OR_FAIL(auto op, NewMergeAppend()); + + auto dv_a1 = MakeDeletionVector("/delete/dv_a1.puffin", file_a_, 1); + auto dv_a2 = MakeDeletionVector("/delete/dv_a2.puffin", file_a_, 2); + auto dv_b = MakeDeletionVector("/delete/dv_b.puffin", file_b_, 3); + + EXPECT_THAT(op->AddDelete(dv_a1, 7), IsOk()); + EXPECT_THAT(op->AddDelete(dv_a2, 7), IsOk()); + EXPECT_THAT(op->AddDelete(dv_b, 8), IsOk()); + + ICEBERG_UNWRAP_OR_FAIL(auto manifests, op->Apply(*table_->metadata(), nullptr)); + auto delete_manifest_it = + std::ranges::find_if(manifests, [](const ManifestFile& manifest) { + return manifest.content == ManifestContent::kDeletes; + }); + ASSERT_NE(delete_manifest_it, manifests.end()); + ICEBERG_UNWRAP_OR_FAIL(auto entries, + ReadAllEntries(std::vector{*delete_manifest_it}, + *table_->metadata())); + + ASSERT_EQ(dv_io->calls.size(), 1U); + EXPECT_THAT(dv_io->calls[0].output_path, ::testing::HasSubstr("/data/merged-dvs-")); + ASSERT_EQ(dv_io->calls[0].groups.size(), 1U); + EXPECT_EQ(dv_io->calls[0].groups[0].referenced_data_file, file_a_->file_path); + EXPECT_THAT(dv_io->calls[0].groups[0].delete_files, ::testing::SizeIs(2)); + + ASSERT_EQ(entries.size(), 2U); + auto merged_it = std::ranges::find_if(entries, [&](const ManifestEntry& entry) { + return entry.data_file->referenced_data_file == file_a_->file_path; + }); + ASSERT_NE(merged_it, entries.end()); + EXPECT_THAT(merged_it->data_file->file_path, ::testing::HasSubstr("/data/merged-dvs-")); + EXPECT_EQ(merged_it->data_file->record_count, 3); + ASSERT_TRUE(merged_it->sequence_number.has_value()); + EXPECT_EQ(*merged_it->sequence_number, 7); + + auto single_it = std::ranges::find_if(entries, [&](const ManifestEntry& entry) { + return entry.data_file->referenced_data_file == file_b_->file_path; + }); + ASSERT_NE(single_it, entries.end()); + EXPECT_EQ(single_it->data_file->file_path, dv_b->file_path); + ASSERT_TRUE(single_it->sequence_number.has_value()); + EXPECT_EQ(*single_it->sequence_number, 8); +} + +TEST_F(MergingSnapshotUpdateTest, ApplyMergesDuplicateDeletionVectorsWithNullPartition) { + SetTableFormatVersion(3); + + auto dv_io = std::make_shared(); + ScopedPuffinDVIORegistry registry( + [dv_io]() -> Result> { return dv_io; }); + ICEBERG_UNWRAP_OR_FAIL(auto op, NewMergeAppend()); + + file_a_->partition = PartitionValues({Literal::Null(int64())}); + auto dv_a1 = MakeDeletionVector("/delete/dv_a1.puffin", file_a_, 1); + auto dv_a2 = MakeDeletionVector("/delete/dv_a2.puffin", file_a_, 2); + + EXPECT_THAT(op->AddDelete(dv_a1, 7), IsOk()); + EXPECT_THAT(op->AddDelete(dv_a2, 7), IsOk()); + + EXPECT_THAT(op->Apply(*table_->metadata(), nullptr), IsOk()); + ASSERT_EQ(dv_io->calls.size(), 1U); + ASSERT_EQ(dv_io->calls[0].groups.size(), 1U); + EXPECT_EQ(dv_io->calls[0].groups[0].referenced_data_file, file_a_->file_path); +} + TEST_F(MergingSnapshotUpdateTest, ValidateNewDeleteFileRejectsUnsupportedVersion) { SetTableFormatVersion(TableMetadata::kSupportedTableFormatVersion + 1); diff --git a/src/iceberg/test/meson.build b/src/iceberg/test/meson.build index ba0f6230d0..0e53993473 100644 --- a/src/iceberg/test/meson.build +++ b/src/iceberg/test/meson.build @@ -59,6 +59,7 @@ iceberg_tests = { 'table_requirements_test.cc', 'table_test.cc', 'table_update_test.cc', + 'update_schema_test.cc', ), }, 'logging_test': { @@ -123,6 +124,7 @@ iceberg_tests = { }, 'puffin_test': { 'sources': files( + 'deletion_vector_writer_test.cc', 'puffin_format_test.cc', 'puffin_json_test.cc', 'puffin_reader_writer_test.cc', diff --git a/src/iceberg/test/position_delete_index_test.cc b/src/iceberg/test/position_delete_index_test.cc index 0c24a587b5..1075ee6fa2 100644 --- a/src/iceberg/test/position_delete_index_test.cc +++ b/src/iceberg/test/position_delete_index_test.cc @@ -19,10 +19,38 @@ #include "iceberg/deletes/position_delete_index.h" +#include +#include +#include + #include +#include "iceberg/file_format.h" +#include "iceberg/manifest/manifest_entry.h" +#include "iceberg/result.h" +#include "iceberg/test/matchers.h" + namespace iceberg { +namespace { + +// A source delete file matching a serialized blob: content_size_in_bytes and +// record_count are what Deserialize validates against. +std::shared_ptr DeleteFileFor(const std::vector& blob, + int64_t record_count) { + return std::make_shared(DataFile{ + .content = DataFile::Content::kPositionDeletes, + .file_path = "memory://dv.puffin", + .file_format = FileFormatType::kPuffin, + .record_count = record_count, + .referenced_data_file = "data.parquet", + .content_offset = 0, + .content_size_in_bytes = static_cast(blob.size()), + }); +} + +} // namespace + TEST(PositionDeleteIndexTest, TestEmptyIndex) { PositionDeleteIndex index; ASSERT_TRUE(index.IsEmpty()); @@ -200,4 +228,141 @@ TEST(PositionDeleteIndexTest, TestMergeIdempotence) { ASSERT_TRUE(index1.IsDeleted(20)); } +// ==================== deletion-vector-v1 serialization ==================== + +TEST(PositionDeleteIndexTest, SerializeRoundTrip) { + PositionDeleteIndex index; + for (int64_t pos : + {int64_t{0}, int64_t{1}, int64_t{5}, int64_t{100}, int64_t{4'000'000'000}}) { + index.Delete(pos); + } + + ICEBERG_UNWRAP_OR_FAIL(auto blob, index.Serialize()); + ICEBERG_UNWRAP_OR_FAIL(auto restored, + PositionDeleteIndex::Deserialize(blob, DeleteFileFor(blob, 5))); + + EXPECT_EQ(restored.Cardinality(), 5); + for (int64_t pos : + {int64_t{0}, int64_t{1}, int64_t{5}, int64_t{100}, int64_t{4'000'000'000}}) { + EXPECT_TRUE(restored.IsDeleted(pos)); + } +} + +TEST(PositionDeleteIndexTest, SerializeEmptyRoundTrip) { + PositionDeleteIndex index; + ICEBERG_UNWRAP_OR_FAIL(auto blob, index.Serialize()); + ICEBERG_UNWRAP_OR_FAIL(auto restored, + PositionDeleteIndex::Deserialize(blob, DeleteFileFor(blob, 0))); + EXPECT_TRUE(restored.IsEmpty()); +} + +// Spans two high-32-bit keys and exercises all Roaring container types +// (sparse "array", dense "bitset", and run containers after optimization). +TEST(PositionDeleteIndexTest, SerializeAllContainerTypesAcrossKeys) { + constexpr int64_t kKeyStride = 0x100000000LL; // 2^32: high-32-bit key + constexpr int64_t kContainerStride = 1 << 16; // 2^16: Roaring container + auto pos = [](int64_t key, int64_t container, int64_t value) { + return key * kKeyStride + container * kContainerStride + value; + }; + + PositionDeleteIndex index; + int64_t expected = 0; + for (int64_t key : {int64_t{0}, int64_t{1}}) { + index.Delete(pos(key, 0, 5)); + index.Delete(pos(key, 0, 7)); + expected += 2; + index.Delete(pos(key, 1, 1), pos(key, 1, 1000)); + expected += 999; + index.Delete(pos(key, 2, 1), pos(key, 2, kContainerStride)); + expected += kContainerStride - 1; + } + + ICEBERG_UNWRAP_OR_FAIL(auto blob, index.Serialize()); + ICEBERG_UNWRAP_OR_FAIL(auto restored, PositionDeleteIndex::Deserialize( + blob, DeleteFileFor(blob, expected))); + + EXPECT_EQ(restored.Cardinality(), expected); + EXPECT_TRUE(restored.IsDeleted(pos(0, 0, 5))); + EXPECT_TRUE(restored.IsDeleted(pos(1, 2, kContainerStride - 1))); + EXPECT_TRUE(restored.IsDeleted(pos(0, 1, 999))); + EXPECT_FALSE(restored.IsDeleted(pos(0, 0, 6))); + EXPECT_FALSE(restored.IsDeleted(pos(1, 1, 1000))); // range end is exclusive +} + +TEST(PositionDeleteIndexTest, DeserializeRejectsCorruptedCrc) { + PositionDeleteIndex index; + index.Delete(1); + index.Delete(2); + ICEBERG_UNWRAP_OR_FAIL(auto blob, index.Serialize()); + + blob.back() ^= 0xFF; + EXPECT_THAT(PositionDeleteIndex::Deserialize(blob, DeleteFileFor(blob, 2)), + IsError(ErrorKind::kInvalidArgument)); +} + +TEST(PositionDeleteIndexTest, DeserializeRejectsBadMagic) { + PositionDeleteIndex index; + index.Delete(1); + ICEBERG_UNWRAP_OR_FAIL(auto blob, index.Serialize()); + + blob[4] = 0x00; + EXPECT_THAT(PositionDeleteIndex::Deserialize(blob, DeleteFileFor(blob, 1)), + IsError(ErrorKind::kInvalidArgument)); +} + +TEST(PositionDeleteIndexTest, DeserializeRejectsTruncatedBlob) { + std::vector blob = {0x00, 0x00}; + EXPECT_THAT(PositionDeleteIndex::Deserialize(blob, DeleteFileFor(blob, 0)), + IsError(ErrorKind::kInvalidArgument)); +} + +TEST(PositionDeleteIndexTest, DeserializeRejectsCardinalityMismatch) { + PositionDeleteIndex index; + index.Delete(1); + index.Delete(2); + ICEBERG_UNWRAP_OR_FAIL(auto blob, index.Serialize()); + + // record_count in metadata disagrees with the bitmap cardinality. + EXPECT_THAT(PositionDeleteIndex::Deserialize(blob, DeleteFileFor(blob, 99)), + IsError(ErrorKind::kInvalidArgument)); +} + +TEST(PositionDeleteIndexTest, DeserializeRejectsContentSizeMismatch) { + PositionDeleteIndex index; + index.Delete(1); + ICEBERG_UNWRAP_OR_FAIL(auto blob, index.Serialize()); + + auto delete_file = DeleteFileFor(blob, 1); + delete_file->content_size_in_bytes = static_cast(blob.size()) + 1; + EXPECT_THAT(PositionDeleteIndex::Deserialize(blob, delete_file), + IsError(ErrorKind::kInvalidArgument)); +} + +TEST(PositionDeleteIndexTest, DeserializePreservesSourceDeleteFile) { + PositionDeleteIndex index; + index.Delete(1); + index.Delete(2); + ICEBERG_UNWRAP_OR_FAIL(auto blob, index.Serialize()); + + auto delete_file = DeleteFileFor(blob, 2); + ICEBERG_UNWRAP_OR_FAIL(auto restored, + PositionDeleteIndex::Deserialize(blob, delete_file)); + ASSERT_EQ(restored.delete_files().size(), 1u); + EXPECT_EQ(restored.delete_files()[0], delete_file); +} + +TEST(PositionDeleteIndexTest, ConstructorsPreserveSourceDeleteFiles) { + auto file_a = std::make_shared(DataFile{.file_path = "a.parquet"}); + auto file_b = std::make_shared(DataFile{.file_path = "b.parquet"}); + + PositionDeleteIndex single(file_a); + ASSERT_EQ(single.delete_files().size(), 1u); + EXPECT_EQ(single.delete_files()[0], file_a); + + PositionDeleteIndex multiple(std::vector>{file_a, file_b}); + ASSERT_EQ(multiple.delete_files().size(), 2u); + EXPECT_EQ(multiple.delete_files()[0], file_a); + EXPECT_EQ(multiple.delete_files()[1], file_b); +} + } // namespace iceberg diff --git a/src/iceberg/test/scan_test_base.h b/src/iceberg/test/scan_test_base.h index 5bd1222e02..4e32febe28 100644 --- a/src/iceberg/test/scan_test_base.h +++ b/src/iceberg/test/scan_test_base.h @@ -221,7 +221,7 @@ class ScanTestBase : public testing::TestWithParam { std::shared_ptr spec = nullptr) { std::vector> files_with_partitions; for (const auto& path : added_files) { - files_with_partitions.emplace_back(path, kEmptyPartition); + files_with_partitions.emplace_back(path, PartitionValues{}); } return MakeAppendSnapshotWithPartitionValues(format_version, snapshot_id, parent_snapshot_id, sequence_number, @@ -350,7 +350,6 @@ class ScanTestBase : public testing::TestWithParam { private: int manifest_counter_ = 0; int manifest_list_counter_ = 0; - constexpr static PartitionValues kEmptyPartition{}; }; } // namespace iceberg diff --git a/src/iceberg/test/schema_field_test.cc b/src/iceberg/test/schema_field_test.cc index c2cd6a7584..d757cb0b20 100644 --- a/src/iceberg/test/schema_field_test.cc +++ b/src/iceberg/test/schema_field_test.cc @@ -24,6 +24,7 @@ #include +#include "iceberg/expression/literal.h" #include "iceberg/type.h" #include "iceberg/util/formatter.h" // IWYU pragma: keep @@ -107,4 +108,66 @@ TEST(SchemaFieldTest, WithDoc) { } } +TEST(SchemaFieldTest, WithFieldMetadata) { + auto initial_default = std::make_shared(Literal::Int(1)); + auto write_default = std::make_shared(Literal::Int(2)); + SchemaField field(1, "foo", int32(), false, "doc", initial_default, write_default); + + auto renamed = field.WithName("bar"); + EXPECT_EQ(renamed.name(), "bar"); + EXPECT_EQ(renamed.field_id(), field.field_id()); + EXPECT_EQ(renamed.type(), field.type()); + EXPECT_EQ(renamed.doc(), field.doc()); + EXPECT_EQ(renamed.initial_default(), field.initial_default()); + EXPECT_EQ(renamed.write_default(), field.write_default()); + + auto retyped = field.WithType(int64()); + EXPECT_EQ(retyped.type(), int64()); + EXPECT_EQ(retyped.name(), field.name()); + EXPECT_EQ(retyped.initial_default(), field.initial_default()); + EXPECT_EQ(retyped.write_default(), field.write_default()); + + auto documented = field.WithDoc("new doc"); + EXPECT_EQ(documented.doc(), "new doc"); + EXPECT_EQ(documented.name(), field.name()); + EXPECT_EQ(documented.initial_default(), field.initial_default()); + EXPECT_EQ(documented.write_default(), field.write_default()); + + auto new_initial = std::make_shared(Literal::Int(3)); + auto with_initial = field.WithInitialDefault(new_initial); + EXPECT_EQ(with_initial.initial_default(), new_initial); + EXPECT_EQ(with_initial.write_default(), field.write_default()); + + auto new_write = std::make_shared(Literal::Int(4)); + auto with_write = field.WithWriteDefault(new_write); + EXPECT_EQ(with_write.initial_default(), field.initial_default()); + EXPECT_EQ(with_write.write_default(), new_write); + EXPECT_EQ(field.name(), "foo"); + EXPECT_EQ(*field.write_default(), Literal::Int(2)); +} + +TEST(SchemaFieldTest, CastDefaultValue) { + { + auto result = SchemaField::CastDefaultValue(Literal::Int(5), int64()); + ASSERT_TRUE(result.has_value()); + EXPECT_EQ(result.value(), Literal::Long(5)); + } + { + auto result = + SchemaField::CastDefaultValue(Literal::Decimal(1234, 9, 2), decimal(18, 2)); + ASSERT_TRUE(result.has_value()); + EXPECT_EQ(result.value(), Literal::Decimal(1234, 18, 2)); + } + { + auto result = + SchemaField::CastDefaultValue(Literal::Decimal(1234, 9, 3), decimal(18, 2)); + EXPECT_FALSE(result.has_value()); + } + { + auto result = + SchemaField::CastDefaultValue(Literal::Decimal(1234567, 9, 2), decimal(4, 2)); + EXPECT_FALSE(result.has_value()); + } +} + } // namespace iceberg diff --git a/src/iceberg/test/string_util_test.cc b/src/iceberg/test/string_util_test.cc index a3fd03760e..65d5c23cc5 100644 --- a/src/iceberg/test/string_util_test.cc +++ b/src/iceberg/test/string_util_test.cc @@ -41,19 +41,53 @@ TEST(StringUtilsTest, ToUpper) { ASSERT_EQ(StringUtils::ToUpper("123"), "123"); } -// Non-ASCII (multibyte UTF-8) bytes have the high bit set, i.e. are negative when stored -// in a signed char. Only ASCII letters are converted; multibyte bytes pass through -// unchanged. The non-ASCII strings are written as explicit UTF-8 byte escapes so the test -// does not depend on the source-file encoding. See -// https://github.com/apache/iceberg-cpp/issues/613. -TEST(StringUtilsTest, NonAsciiPassThrough) { - // "Naïve" -> "naïve" (ï = U+00EF = 0xC3 0xAF; only the ASCII letters change). - ASSERT_EQ(StringUtils::ToLower("Na\xC3\xAFve"), "na\xC3\xAFve"); - // "café" -> "CAFé" (é = U+00E9 = 0xC3 0xA9 stays unchanged). - ASSERT_EQ(StringUtils::ToUpper("caf\xC3\xA9"), "CAF\xC3\xA9"); - // "日本語" (0xE6 0x97 0xA5 0xE6 0x9C 0xAC 0xE8 0xAA 0x9E) is returned verbatim. +// Non-ASCII strings are written as explicit UTF-8 byte escapes so the test does not +// depend on the source-file encoding. An escape is split before a following hex digit +// (e.g. "...\x9E" "E") so the \x does not absorb it. +// See https://github.com/apache/iceberg-cpp/issues/613. +TEST(StringUtilsTest, ToLowerUnicode) { + // "CAFÉ" -> "café" (É U+00C9 = 0xC3 0x89 -> é U+00E9 = 0xC3 0xA9). + ASSERT_EQ(StringUtils::ToLower("CAF\xC3\x89"), "caf\xC3\xA9"); + // "GROẞE" -> "große": capital sharp S (ẞ U+1E9E) lower-cases to ß (U+00DF), not "ss" + // as casefolding would produce. + ASSERT_EQ(StringUtils::ToLower("GRO\xE1\xBA\x9E" + "E"), + "gro\xC3\x9F" + "e"); + // "日本語" has no case mapping and is returned verbatim. ASSERT_EQ(StringUtils::ToLower("\xE6\x97\xA5\xE6\x9C\xAC\xE8\xAA\x9E"), "\xE6\x97\xA5\xE6\x9C\xAC\xE8\xAA\x9E"); + // ASCII prefix before the first non-ASCII byte takes the fast path; the rest goes + // through utf8proc. "ABÉ" -> "abé". + ASSERT_EQ(StringUtils::ToLower("AB\xC3\x89"), "ab\xC3\xA9"); + // An invalid UTF-8 byte (a lone 0xFF) passes through unchanged rather than erroring. + ASSERT_EQ(StringUtils::ToLower("\xFF"), "\xFF"); + // An invalid byte only passes through itself; the valid code points around it are + // still lower-cased ("AB" 0xFF "CÉ" -> "ab" 0xFF "cé"). + ASSERT_EQ(StringUtils::ToLower("AB\xFF" + "C\xC3\x89"), + "ab\xFF" + "c\xC3\xA9"); + // The invalid byte can abut a multi-byte code point with no ASCII between them; 0xFF + // passes through and the adjacent "É" still lower-cases to "é" (0xFF "É" -> 0xFF "é"). + ASSERT_EQ(StringUtils::ToLower("\xFF\xC3\x89"), "\xFF\xC3\xA9"); + // A truncated multi-byte sequence (0xC3 with no continuation byte) passes through + // without consuming the bytes after it. + ASSERT_EQ(StringUtils::ToLower("\xC3" + "AB"), + "\xC3" + "ab"); + // A stray continuation byte (0x80) behaves the same way. + ASSERT_EQ(StringUtils::ToLower("A\x80" + "B"), + "a\x80" + "b"); +} + +// ToUpper is intentionally ASCII-only; non-ASCII (multibyte UTF-8) bytes pass through. +TEST(StringUtilsTest, ToUpperAsciiOnly) { + // "café" -> "CAFé" (é stays unchanged). + ASSERT_EQ(StringUtils::ToUpper("caf\xC3\xA9"), "CAF\xC3\xA9"); ASSERT_EQ(StringUtils::ToUpper("\xE6\x97\xA5\xE6\x9C\xAC\xE8\xAA\x9E"), "\xE6\x97\xA5\xE6\x9C\xAC\xE8\xAA\x9E"); } @@ -63,10 +97,109 @@ TEST(StringUtilsTest, EqualsIgnoreCase) { ASSERT_TRUE(StringUtils::EqualsIgnoreCase("", "")); ASSERT_FALSE(StringUtils::EqualsIgnoreCase("abc", "abcd")); ASSERT_FALSE(StringUtils::EqualsIgnoreCase("abc", "abd")); - // ASCII case is folded; non-ASCII bytes are compared as-is. ("Café" vs "café") - ASSERT_TRUE(StringUtils::EqualsIgnoreCase("Caf\xC3\xA9", "caf\xC3\xA9")); - // "café" vs "cafe": the multibyte é differs from ASCII 'e'. + // Unicode-aware: "CAFÉ" matches "café". + ASSERT_TRUE(StringUtils::EqualsIgnoreCase("CAF\xC3\x89", "caf\xC3\xA9")); + // "GROẞE" matches "große" under lowercasing (ẞ -> ß). + ASSERT_TRUE( + StringUtils::EqualsIgnoreCase("GRO\xE1\xBA\x9E" + "E", + "gro\xC3\x9F" + "e")); + // Different letters still differ ("café" vs "cafe"). ASSERT_FALSE(StringUtils::EqualsIgnoreCase("caf\xC3\xA9", "cafe")); + // Fallback correctness: an ASCII operand can equal a non-ASCII one once lower-cased, + // even though their raw byte lengths differ. "İ" (U+0130 = 0xC4 0xB0, two bytes) + // lower-cases to one-byte "i", so it must compare equal to "i" and "I". + ASSERT_TRUE(StringUtils::EqualsIgnoreCase("i", "\xC4\xB0")); + ASSERT_TRUE(StringUtils::EqualsIgnoreCase("\xC4\xB0", "I")); + // The non-ASCII byte can appear after a matching ASCII prefix ("abi" vs "abİ"). + ASSERT_TRUE(StringUtils::EqualsIgnoreCase("abi", "ab\xC4\xB0")); + // Pure-ASCII operands that share a prefix but differ in length are not equal. + ASSERT_FALSE(StringUtils::EqualsIgnoreCase("abc", "ab")); + // Operands containing invalid UTF-8 are still compared case-insensitively on their + // valid parts; the invalid bytes themselves compare verbatim. + ASSERT_TRUE( + StringUtils::EqualsIgnoreCase("AB\xFF" + "C", + "ab\xFF" + "c")); + ASSERT_FALSE(StringUtils::EqualsIgnoreCase("\xFF", "\xFE")); +} + +TEST(StringUtilsTest, StartsWithIgnoreCase) { + ASSERT_TRUE(StringUtils::StartsWithIgnoreCase("AbCdef", "abc")); + ASSERT_TRUE(StringUtils::StartsWithIgnoreCase("abc", "ABC")); + ASSERT_FALSE(StringUtils::StartsWithIgnoreCase("abc", "abd")); + // Empty prefix always matches; a prefix longer than the string does not. + ASSERT_TRUE(StringUtils::StartsWithIgnoreCase("abc", "")); + ASSERT_FALSE(StringUtils::StartsWithIgnoreCase("ab", "abcd")); + // Regression (#760): lower-casing can change byte length, so the prefix must not be + // matched by byte-slicing. "İ" (U+0130 = 0xC4 0xB0) lower-cases to "i", so "İx" + // starts with "i" ... + ASSERT_TRUE(StringUtils::StartsWithIgnoreCase("\xC4\xB0x", "i")); + // ... and "i" starts with "İ" (both lower-case to "i"), which the old byte-length + // guard wrongly rejected. + ASSERT_TRUE(StringUtils::StartsWithIgnoreCase("i", "\xC4\xB0")); + // A matching Unicode prefix: "CAFÉbar" starts with "café". + ASSERT_TRUE( + StringUtils::StartsWithIgnoreCase("CAF\xC3\x89" + "bar", + "caf\xC3\xA9")); + // Invalid UTF-8 bytes compare verbatim in the prefix as well. + ASSERT_TRUE( + StringUtils::StartsWithIgnoreCase("AB\xFF" + "x", + "ab\xFF")); + ASSERT_FALSE(StringUtils::StartsWithIgnoreCase("ab\xFE", "ab\xFF")); +} + +// The ASCII fast paths in EqualsIgnoreCase / StartsWithIgnoreCase must agree with their +// documented ToLower-based semantics for every input, including length-changing case +// mappings and invalid UTF-8. Rather than enumerate cases by hand, exhaustively compare +// both functions against the ToLower oracle over all short strings built from a small +// alphabet that straddles those boundaries. This is the mechanical form of the #760 +// regression, where a fast path disagreed with ToLower on a length-changing mapping. +TEST(StringUtilsTest, IgnoreCaseAgreesWithToLowerOracle) { + // Atoms mix ASCII (upper/lower, including the lowercase targets of the multi-byte + // mappings) with a 2-byte code point that lower-cases to one byte ("İ" U+0130 -> "i"), + // a 3-byte one that also shrinks to one byte ("K" U+212A -> "k"), an ordinary 2-byte + // cased letter ("É"), and an invalid UTF-8 byte. + const std::vector atoms = { + "a", "I", "i", "k", "\xC4\xB0", "\xE2\x84\xAA", "\xC3\x89", "\xFF"}; + + // Build every string of 0..3 atoms, one generation (length) at a time. + std::vector inputs = {""}; + size_t generation_begin = 0; + for (int len = 0; len < 3; ++len) { + const size_t generation_end = inputs.size(); + for (size_t i = generation_begin; i < generation_end; ++i) { + for (const auto& atom : atoms) { + inputs.push_back(inputs[i] + atom); + } + } + generation_begin = generation_end; + } + + // Precompute the oracle so the O(n^2) comparison below does not re-lower each string. + std::vector lowered; + lowered.reserve(inputs.size()); + for (const auto& s : inputs) { + lowered.push_back(StringUtils::ToLower(s)); + } + + for (size_t i = 0; i < inputs.size(); ++i) { + for (size_t j = 0; j < inputs.size(); ++j) { + EXPECT_EQ(StringUtils::EqualsIgnoreCase(inputs[i], inputs[j]), + lowered[i] == lowered[j]) + << "EqualsIgnoreCase disagreed for a=" << testing::PrintToString(inputs[i]) + << " b=" << testing::PrintToString(inputs[j]); + EXPECT_EQ(StringUtils::StartsWithIgnoreCase(inputs[i], inputs[j]), + lowered[i].starts_with(lowered[j])) + << "StartsWithIgnoreCase disagreed for str=" + << testing::PrintToString(inputs[i]) + << " prefix=" << testing::PrintToString(inputs[j]); + } + } } } // namespace iceberg diff --git a/src/iceberg/test/table_metadata_builder_test.cc b/src/iceberg/test/table_metadata_builder_test.cc index f9080e0f2c..5fd970d620 100644 --- a/src/iceberg/test/table_metadata_builder_test.cc +++ b/src/iceberg/test/table_metadata_builder_test.cc @@ -125,7 +125,7 @@ TEST(TableMetadataTest, Make) { auto spec_fields = metadata->partition_specs[0]->fields() | std::ranges::to(); ASSERT_EQ(1, spec_fields.size()); - EXPECT_EQ(PartitionSpec::kInvalidPartitionFieldId + 1, spec_fields[0].field_id()); + EXPECT_EQ(PartitionSpec::kLegacyPartitionDataIdStart, spec_fields[0].field_id()); EXPECT_EQ(2, spec_fields[0].source_id()); EXPECT_EQ("part_name", spec_fields[0].name()); diff --git a/src/iceberg/test/update_properties_test.cc b/src/iceberg/test/update_properties_test.cc index 886558153f..2a22ff949d 100644 --- a/src/iceberg/test/update_properties_test.cc +++ b/src/iceberg/test/update_properties_test.cc @@ -129,6 +129,53 @@ TEST_F(UpdatePropertiesTest, UpgradeFormatVersionUnsupported) { EXPECT_THAT(result, HasErrorMessage("unsupported format version")); } +TEST_F(UpdatePropertiesTest, SetReservedPropertyUuid) { + ICEBERG_UNWRAP_OR_FAIL(auto update, table_->NewUpdateProperties()); + update->Set("uuid", "some-uuid"); + + auto result = update->Apply(); + EXPECT_THAT(result, IsError(ErrorKind::kValidationFailed)); + EXPECT_THAT(result, HasErrorMessage("Cannot set reserved property")); +} + +TEST_F(UpdatePropertiesTest, SetReservedPropertyCurrentSchema) { + ICEBERG_UNWRAP_OR_FAIL(auto update, table_->NewUpdateProperties()); + update->Set("current-schema", R"({"type": "struct"})"); + + auto result = update->Apply(); + EXPECT_THAT(result, IsError(ErrorKind::kValidationFailed)); + EXPECT_THAT(result, HasErrorMessage("Cannot set reserved property")); +} + +TEST_F(UpdatePropertiesTest, SetReservedPropertyCurrentSnapshotId) { + ICEBERG_UNWRAP_OR_FAIL(auto update, table_->NewUpdateProperties()); + update->Set("current-snapshot-id", "12345"); + + auto result = update->Apply(); + EXPECT_THAT(result, IsError(ErrorKind::kValidationFailed)); + EXPECT_THAT(result, HasErrorMessage("Cannot set reserved property")); +} + +TEST_F(UpdatePropertiesTest, SetFormatVersionStillAllowed) { + ICEBERG_UNWRAP_OR_FAIL(auto update, table_->NewUpdateProperties()); + update->Set("format-version", "3"); + + ICEBERG_UNWRAP_OR_FAIL(auto result, update->Apply()); + EXPECT_TRUE(result.updates.empty()); + ASSERT_TRUE(result.format_version.has_value()); + EXPECT_EQ(result.format_version.value(), 3); +} + +TEST_F(UpdatePropertiesTest, SetValidAndReservedProperties) { + ICEBERG_UNWRAP_OR_FAIL(auto update, table_->NewUpdateProperties()); + update->Set("valid.key", "valid.value"); + update->Set("snapshot-count", "10"); + + auto result = update->Apply(); + EXPECT_THAT(result, IsError(ErrorKind::kValidationFailed)); + EXPECT_THAT(result, HasErrorMessage("Cannot set reserved property")); +} + TEST_F(UpdatePropertiesTest, CommitSuccess) { ICEBERG_UNWRAP_OR_FAIL(auto empty_update, table_->NewUpdateProperties()); EXPECT_THAT(empty_update->Commit(), IsOk()); diff --git a/src/iceberg/test/update_schema_test.cc b/src/iceberg/test/update_schema_test.cc index 07872e69a3..db6b1d02e1 100644 --- a/src/iceberg/test/update_schema_test.cc +++ b/src/iceberg/test/update_schema_test.cc @@ -19,23 +19,88 @@ #include "iceberg/update/update_schema.h" +#include +#include #include +#include +#include +#include #include +#include #include +#include "iceberg/catalog/memory/in_memory_catalog.h" +#include "iceberg/expression/literal.h" #include "iceberg/schema.h" #include "iceberg/schema_field.h" +#include "iceberg/table.h" +#include "iceberg/table_identifier.h" +#include "iceberg/table_metadata.h" #include "iceberg/test/matchers.h" -#include "iceberg/test/update_test_base.h" +#include "iceberg/test/mock_io.h" +#include "iceberg/test/test_resource.h" #include "iceberg/type.h" #include "iceberg/util/checked_cast.h" +#include "iceberg/util/uuid.h" namespace iceberg { using internal::checked_cast; -class UpdateSchemaTest : public UpdateTestBase {}; +class UpdateSchemaTest : public ::testing::Test { + protected: + virtual std::string MetadataResource() const { return "TableMetadataV2Valid.json"; } + virtual std::string TableName() const { return "test_table"; } + + void SetUp() override { + table_ident_ = TableIdentifier{.name = TableName()}; + table_location_ = "/warehouse/" + TableName(); + + InitializeFileIO(); + RegisterTableFromResource(MetadataResource()); + } + + void InitializeFileIO() { + auto mock_file_io = std::make_shared<::testing::NiceMock>(); + auto* mock_file_io_ptr = mock_file_io.get(); + ON_CALL(*mock_file_io, ReadFile(::testing::_, ::testing::_)) + .WillByDefault([mock_file_io_ptr](const std::string& file_location, + std::optional length) { + return mock_file_io_ptr->FileIO::ReadFile(file_location, length); + }); + ON_CALL(*mock_file_io, WriteFile(::testing::_, ::testing::_)) + .WillByDefault([mock_file_io_ptr](const std::string& file_location, + std::string_view content) { + return mock_file_io_ptr->FileIO::WriteFile(file_location, content); + }); + file_io_ = std::move(mock_file_io); + catalog_ = + InMemoryCatalog::Make("test_catalog", file_io_, "/warehouse/", /*properties=*/{}); + } + + void RegisterTable(std::unique_ptr metadata) { + auto metadata_location = std::format("{}/metadata/00001-{}.metadata.json", + table_location_, Uuid::GenerateV7().ToString()); + metadata->location = table_location_; + ASSERT_THAT(TableMetadataUtil::Write(*file_io_, metadata_location, *metadata), + IsOk()); + + ICEBERG_UNWRAP_OR_FAIL(table_, + catalog_->RegisterTable(table_ident_, metadata_location)); + } + + void RegisterTableFromResource(const std::string& resource_name) { + ICEBERG_UNWRAP_OR_FAIL(auto metadata, ReadTableMetadataFromResource(resource_name)); + RegisterTable(std::move(metadata)); + } + + TableIdentifier table_ident_; + std::string table_location_; + std::shared_ptr file_io_; + std::shared_ptr catalog_; + std::shared_ptr table_; +}; TEST_F(UpdateSchemaTest, AddOptionalColumn) { ICEBERG_UNWRAP_OR_FAIL(auto update, table_->NewUpdateSchema()); @@ -82,6 +147,347 @@ TEST_F(UpdateSchemaTest, AddRequiredColumnWithAllowIncompatible) { EXPECT_EQ(new_field.doc(), "A required string column"); } +/// Uses v3 metadata for default-value tests. +class UpdateSchemaDefaultValueTest : public UpdateSchemaTest { + protected: + void SetUp() override { + table_ident_ = TableIdentifier{.name = TableName()}; + table_location_ = "/warehouse/" + TableName(); + + InitializeFileIO(); + + ICEBERG_UNWRAP_OR_FAIL(auto metadata, + ReadTableMetadataFromResource("TableMetadataV2Valid.json")); + metadata->format_version = TableMetadata::kSupportedTableFormatVersion; + metadata->next_row_id = TableMetadata::kInitialRowId; + RegisterTable(std::move(metadata)); + } +}; + +TEST_F(UpdateSchemaTest, AddColumnWithDefaultValueRequiresV3) { + ICEBERG_UNWRAP_OR_FAIL(auto update, table_->NewUpdateSchema()); + update->AddColumn("new_col", int32(), "An integer column", Literal::Int(42)); + + auto result = update->Apply(); + EXPECT_THAT(result, IsError(ErrorKind::kInvalidSchema)); + EXPECT_THAT(result, HasErrorMessage("is not supported until v3")); +} + +TEST_F(UpdateSchemaDefaultValueTest, AddColumnWithDefaultValue) { + ICEBERG_UNWRAP_OR_FAIL(auto update, table_->NewUpdateSchema()); + update->AddColumn("new_col", int32(), "An integer column", Literal::Int(42)); + + ICEBERG_UNWRAP_OR_FAIL(auto result, update->Apply()); + ICEBERG_UNWRAP_OR_FAIL(auto new_field_opt, result.schema->FindFieldByName("new_col")); + ASSERT_TRUE(new_field_opt.has_value()); + + const auto& new_field = new_field_opt->get(); + ASSERT_NE(new_field.initial_default(), nullptr); + EXPECT_EQ(*new_field.initial_default(), Literal::Int(42)); + ASSERT_NE(new_field.write_default(), nullptr); + EXPECT_EQ(*new_field.write_default(), Literal::Int(42)); +} + +TEST_F(UpdateSchemaDefaultValueTest, AddRequiredColumnWithDefaultValue) { + ICEBERG_UNWRAP_OR_FAIL(auto update, table_->NewUpdateSchema()); + update->AddRequiredColumn("required_col", string(), "A required string column", + Literal::String("n/a")); + + ICEBERG_UNWRAP_OR_FAIL(auto result, update->Apply()); + ICEBERG_UNWRAP_OR_FAIL(auto new_field_opt, + result.schema->FindFieldByName("required_col")); + ASSERT_TRUE(new_field_opt.has_value()); + + const auto& new_field = new_field_opt->get(); + EXPECT_FALSE(new_field.optional()); + ASSERT_NE(new_field.initial_default(), nullptr); + EXPECT_EQ(*new_field.initial_default(), Literal::String("n/a")); + ASSERT_NE(new_field.write_default(), nullptr); + EXPECT_EQ(*new_field.write_default(), Literal::String("n/a")); +} + +TEST_F(UpdateSchemaDefaultValueTest, AddColumnWithMismatchedDefaultValueFails) { + ICEBERG_UNWRAP_OR_FAIL(auto update, table_->NewUpdateSchema()); + update->AddColumn("new_col", int32(), "An integer column", Literal::String("oops")); + + auto result = update->Apply(); + EXPECT_THAT(result, IsError(ErrorKind::kValidationFailed)); + EXPECT_THAT(result, HasErrorMessage("Cannot cast default value")); +} + +TEST_F(UpdateSchemaDefaultValueTest, AddColumnWithNarrowingDefaultValueFails) { + ICEBERG_UNWRAP_OR_FAIL(auto update, table_->NewUpdateSchema()); + update->AddColumn("new_col", int32(), "An integer column", + Literal::Long(std::numeric_limits::max())); + + auto result = update->Apply(); + EXPECT_THAT(result, IsError(ErrorKind::kValidationFailed)); + EXPECT_THAT(result, HasErrorMessage("Cannot cast default value")); +} + +TEST_F(UpdateSchemaDefaultValueTest, UpdateColumnDefault) { + ICEBERG_UNWRAP_OR_FAIL(auto update, table_->NewUpdateSchema()); + update->AddColumn("new_col", int32(), "An integer column", Literal::Int(42)) + .UpdateColumnDefault("new_col", Literal::Int(7)); + + ICEBERG_UNWRAP_OR_FAIL(auto result, update->Apply()); + ICEBERG_UNWRAP_OR_FAIL(auto new_field_opt, result.schema->FindFieldByName("new_col")); + ASSERT_TRUE(new_field_opt.has_value()); + + const auto& new_field = new_field_opt->get(); + ASSERT_NE(new_field.initial_default(), nullptr); + EXPECT_EQ(*new_field.initial_default(), Literal::Int(42)); + ASSERT_NE(new_field.write_default(), nullptr); + EXPECT_EQ(*new_field.write_default(), Literal::Int(7)); +} + +TEST_F(UpdateSchemaDefaultValueTest, UpdateColumnDefaultOnExistingColumn) { + ICEBERG_UNWRAP_OR_FAIL(auto update, table_->NewUpdateSchema()); + update->UpdateColumnDefault("x", Literal::Long(0)); + + ICEBERG_UNWRAP_OR_FAIL(auto result, update->Apply()); + ICEBERG_UNWRAP_OR_FAIL(auto field_opt, result.schema->FindFieldByName("x")); + ASSERT_TRUE(field_opt.has_value()); + + const auto& field = field_opt->get(); + EXPECT_EQ(field.initial_default(), nullptr); + ASSERT_NE(field.write_default(), nullptr); + EXPECT_EQ(*field.write_default(), Literal::Long(0)); +} + +TEST_F(UpdateSchemaDefaultValueTest, UpdateColumnDefaultClearsWithNullopt) { + ICEBERG_UNWRAP_OR_FAIL(auto update, table_->NewUpdateSchema()); + update->AddColumn("new_col", int32(), "An integer column", Literal::Int(42)) + .UpdateColumnDefault("new_col", std::nullopt); + + ICEBERG_UNWRAP_OR_FAIL(auto result, update->Apply()); + ICEBERG_UNWRAP_OR_FAIL(auto field_opt, result.schema->FindFieldByName("new_col")); + ASSERT_TRUE(field_opt.has_value()); + + const auto& field = field_opt->get(); + ASSERT_NE(field.initial_default(), nullptr); + EXPECT_EQ(*field.initial_default(), Literal::Int(42)); + EXPECT_EQ(field.write_default(), nullptr); +} + +TEST_F(UpdateSchemaDefaultValueTest, AddNestedColumnPreservesNestedDefaults) { + auto nested_type = std::make_shared(std::vector{ + SchemaField(/*field_id=*/100, "inner", int32(), /*optional=*/false, /*doc=*/{}, + std::make_shared(Literal::Int(5)), + std::make_shared(Literal::Int(9)))}); + + ICEBERG_UNWRAP_OR_FAIL(auto update, table_->NewUpdateSchema()); + update->AddColumn("outer", nested_type, "A nested column"); + + ICEBERG_UNWRAP_OR_FAIL(auto result, update->Apply()); + ICEBERG_UNWRAP_OR_FAIL(auto outer_opt, result.schema->FindFieldByName("outer")); + ASSERT_TRUE(outer_opt.has_value()); + + const auto& outer_struct = + internal::checked_cast(*outer_opt->get().type()); + ASSERT_EQ(outer_struct.fields().size(), 1); + const SchemaField& inner = outer_struct.fields()[0]; + ASSERT_NE(inner.initial_default(), nullptr); + EXPECT_EQ(*inner.initial_default(), Literal::Int(5)); + ASSERT_NE(inner.write_default(), nullptr); + EXPECT_EQ(*inner.write_default(), Literal::Int(9)); +} + +TEST_F(UpdateSchemaDefaultValueTest, UpdateColumnDefaultCastsToColumnType) { + ICEBERG_UNWRAP_OR_FAIL(auto update, table_->NewUpdateSchema()); + update->UpdateColumnDefault("x", Literal::Int(5)); + + ICEBERG_UNWRAP_OR_FAIL(auto result, update->Apply()); + ICEBERG_UNWRAP_OR_FAIL(auto field_opt, result.schema->FindFieldByName("x")); + ASSERT_TRUE(field_opt.has_value()); + + const auto& field = field_opt->get(); + ASSERT_NE(field.write_default(), nullptr); + EXPECT_EQ(*field.write_default(), Literal::Long(5)); +} + +TEST_F(UpdateSchemaDefaultValueTest, RequireColumnAddedWithDefault) { + ICEBERG_UNWRAP_OR_FAIL(auto update, table_->NewUpdateSchema()); + update->AddColumn("new_col", int32(), "An integer column", Literal::Int(42)) + .RequireColumn("new_col"); + + ICEBERG_UNWRAP_OR_FAIL(auto result, update->Apply()); + ICEBERG_UNWRAP_OR_FAIL(auto new_field_opt, result.schema->FindFieldByName("new_col")); + ASSERT_TRUE(new_field_opt.has_value()); + EXPECT_FALSE(new_field_opt->get().optional()); +} + +TEST_F(UpdateSchemaDefaultValueTest, RequireNestedMapListColumnAddedWithDefault) { + // Map/list paths omit value/element. + auto map_key_struct = std::make_shared( + std::vector{SchemaField(20, "address", string(), false)}); + auto map_value_struct = std::make_shared( + std::vector{SchemaField(12, "lat", float32(), false), + SchemaField(13, "long", float32(), false)}); + auto map_type = + std::make_shared(SchemaField(10, "key", map_key_struct, false), + SchemaField(11, "value", map_value_struct, false)); + auto list_element_struct = std::make_shared(std::vector{ + SchemaField(15, "x", int64(), false), SchemaField(16, "y", int64(), false)}); + auto list_type = + std::make_shared(SchemaField(14, "element", list_element_struct, true)); + + ICEBERG_UNWRAP_OR_FAIL(auto setup_update, table_->NewUpdateSchema()); + setup_update->AddColumn("locations", map_type, "map of address to coordinate") + .AddColumn("points", list_type, "2-D cartesian points"); + EXPECT_THAT(setup_update->Commit(), IsOk()); + + ICEBERG_UNWRAP_OR_FAIL(auto reloaded, catalog_->LoadTable(table_ident_)); + ICEBERG_UNWRAP_OR_FAIL(auto update, reloaded->NewUpdateSchema()); + update->AddColumn("locations", "alt", float32(), "altitude", Literal::Float(0.0f)) + .RequireColumn("locations.alt") + .AddColumn("points", "z", int64(), "z coordinate", Literal::Long(0)) + .RequireColumn("points.z"); + + ICEBERG_UNWRAP_OR_FAIL(auto result, update->Apply()); + + ICEBERG_UNWRAP_OR_FAIL(auto locations_opt, result.schema->FindFieldByName("locations")); + ASSERT_TRUE(locations_opt.has_value()); + const auto& map = checked_cast(*locations_opt->get().type()); + const auto& value_struct = checked_cast(*map.value().type()); + ICEBERG_UNWRAP_OR_FAIL(auto alt_opt, value_struct.GetFieldByName("alt")); + ASSERT_TRUE(alt_opt.has_value()); + EXPECT_FALSE(alt_opt->get().optional()); + ASSERT_NE(alt_opt->get().initial_default(), nullptr); + + ICEBERG_UNWRAP_OR_FAIL(auto points_opt, result.schema->FindFieldByName("points")); + ASSERT_TRUE(points_opt.has_value()); + const auto& list = checked_cast(*points_opt->get().type()); + const auto& element_struct = checked_cast(*list.element().type()); + ICEBERG_UNWRAP_OR_FAIL(auto z_opt, element_struct.GetFieldByName("z")); + ASSERT_TRUE(z_opt.has_value()); + EXPECT_FALSE(z_opt->get().optional()); + ASSERT_NE(z_opt->get().initial_default(), nullptr); +} + +TEST_F(UpdateSchemaDefaultValueTest, UpdateColumnDocPreservesDefaultValues) { + ICEBERG_UNWRAP_OR_FAIL(auto update, table_->NewUpdateSchema()); + update->AddColumn("new_col", int32(), "An integer column", Literal::Int(42)) + .UpdateColumnDoc("new_col", "updated doc"); + + ICEBERG_UNWRAP_OR_FAIL(auto result, update->Apply()); + ICEBERG_UNWRAP_OR_FAIL(auto field_opt, result.schema->FindFieldByName("new_col")); + ASSERT_TRUE(field_opt.has_value()); + + const auto& field = field_opt->get(); + EXPECT_EQ(field.doc(), "updated doc"); + ASSERT_NE(field.initial_default(), nullptr); + EXPECT_EQ(*field.initial_default(), Literal::Int(42)); + ASSERT_NE(field.write_default(), nullptr); + EXPECT_EQ(*field.write_default(), Literal::Int(42)); +} + +TEST_F(UpdateSchemaDefaultValueTest, UpdateColumnTypePromotesDefaultValues) { + ICEBERG_UNWRAP_OR_FAIL(auto update, table_->NewUpdateSchema()); + update->AddColumn("new_col", int32(), "An integer column", Literal::Int(42)) + .UpdateColumn("new_col", int64()); + + ICEBERG_UNWRAP_OR_FAIL(auto result, update->Apply()); + ICEBERG_UNWRAP_OR_FAIL(auto field_opt, result.schema->FindFieldByName("new_col")); + ASSERT_TRUE(field_opt.has_value()); + + const auto& field = field_opt->get(); + EXPECT_EQ(field.type(), int64()); + ASSERT_NE(field.initial_default(), nullptr); + EXPECT_EQ(*field.initial_default(), Literal::Long(42)); + ASSERT_NE(field.write_default(), nullptr); + EXPECT_EQ(*field.write_default(), Literal::Long(42)); +} + +TEST_F(UpdateSchemaDefaultValueTest, UpdateColumnTypePromotesDecimalDefault) { + ICEBERG_UNWRAP_OR_FAIL(auto update, table_->NewUpdateSchema()); + update + ->AddColumn("new_col", decimal(9, 2), "A decimal column", + Literal::Decimal(1234, 9, 2)) + .UpdateColumn("new_col", decimal(18, 2)); + + ICEBERG_UNWRAP_OR_FAIL(auto result, update->Apply()); + ICEBERG_UNWRAP_OR_FAIL(auto field_opt, result.schema->FindFieldByName("new_col")); + ASSERT_TRUE(field_opt.has_value()); + + const auto& field = field_opt->get(); + EXPECT_EQ(field.type()->ToString(), decimal(18, 2)->ToString()); + ASSERT_NE(field.initial_default(), nullptr); + EXPECT_EQ(*field.initial_default(), Literal::Decimal(1234, 18, 2)); + ASSERT_NE(field.write_default(), nullptr); + EXPECT_EQ(*field.write_default(), Literal::Decimal(1234, 18, 2)); +} + +TEST_F(UpdateSchemaDefaultValueTest, AddColumnWithWiderPrecisionDecimalDefault) { + ICEBERG_UNWRAP_OR_FAIL(auto update, table_->NewUpdateSchema()); + update->AddColumn("new_col", decimal(18, 2), "A decimal column", + Literal::Decimal(1234, 9, 2)); + + ICEBERG_UNWRAP_OR_FAIL(auto result, update->Apply()); + ICEBERG_UNWRAP_OR_FAIL(auto field_opt, result.schema->FindFieldByName("new_col")); + ASSERT_TRUE(field_opt.has_value()); + + const auto& field = field_opt->get(); + ASSERT_NE(field.initial_default(), nullptr); + EXPECT_EQ(*field.initial_default(), Literal::Decimal(1234, 18, 2)); + ASSERT_NE(field.write_default(), nullptr); + EXPECT_EQ(*field.write_default(), Literal::Decimal(1234, 18, 2)); +} + +TEST_F(UpdateSchemaDefaultValueTest, UpdateColumnDefaultWiderPrecisionDecimal) { + ICEBERG_UNWRAP_OR_FAIL(auto update, table_->NewUpdateSchema()); + update->AddColumn("new_col", decimal(18, 2), "A decimal column") + .UpdateColumnDefault("new_col", Literal::Decimal(1234, 9, 2)); + + ICEBERG_UNWRAP_OR_FAIL(auto result, update->Apply()); + ICEBERG_UNWRAP_OR_FAIL(auto field_opt, result.schema->FindFieldByName("new_col")); + ASSERT_TRUE(field_opt.has_value()); + + const auto& field = field_opt->get(); + ASSERT_NE(field.write_default(), nullptr); + EXPECT_EQ(*field.write_default(), Literal::Decimal(1234, 18, 2)); +} + +TEST_F(UpdateSchemaDefaultValueTest, AddColumnWithDifferentScaleDecimalDefaultFails) { + ICEBERG_UNWRAP_OR_FAIL(auto update, table_->NewUpdateSchema()); + update->AddColumn("new_col", decimal(18, 2), "A decimal column", + Literal::Decimal(1234, 9, 3)); + + auto result = update->Apply(); + EXPECT_THAT(result, IsError(ErrorKind::kValidationFailed)); + EXPECT_THAT(result, HasErrorMessage("Cannot cast default value")); +} + +TEST_F(UpdateSchemaDefaultValueTest, UpdateColumnDefaultDifferentScaleDecimalFails) { + ICEBERG_UNWRAP_OR_FAIL(auto update, table_->NewUpdateSchema()); + update->AddColumn("new_col", decimal(18, 2), "A decimal column") + .UpdateColumnDefault("new_col", Literal::Decimal(1234, 9, 3)); + + auto result = update->Apply(); + EXPECT_THAT(result, IsError(ErrorKind::kValidationFailed)); + EXPECT_THAT(result, HasErrorMessage("Cannot cast default value")); +} + +TEST_F(UpdateSchemaDefaultValueTest, AddColumnWithOutOfPrecisionDecimalDefaultFails) { + ICEBERG_UNWRAP_OR_FAIL(auto update, table_->NewUpdateSchema()); + update->AddColumn("new_col", decimal(4, 2), "A decimal column", + Literal::Decimal(1234567, 9, 2)); + + auto result = update->Apply(); + EXPECT_THAT(result, IsError(ErrorKind::kValidationFailed)); + EXPECT_THAT(result, HasErrorMessage("Cannot cast default value")); +} + +TEST_F(UpdateSchemaDefaultValueTest, AddColumnWithTypedNullDecimalDefaultFails) { + ICEBERG_UNWRAP_OR_FAIL(auto update, table_->NewUpdateSchema()); + update->AddColumn("new_col", decimal(18, 2), "A decimal column", + Literal::Null(decimal(18, 2))); + + auto result = update->Apply(); + EXPECT_THAT(result, IsError(ErrorKind::kValidationFailed)); + EXPECT_THAT(result, HasErrorMessage("Cannot cast default value")); +} + TEST_F(UpdateSchemaTest, AddMultipleColumns) { ICEBERG_UNWRAP_OR_FAIL(auto update, table_->NewUpdateSchema()); update->AddColumn("col1", int32(), "First column") diff --git a/src/iceberg/transaction.cc b/src/iceberg/transaction.cc index 7abce27cb2..503121658d 100644 --- a/src/iceberg/transaction.cc +++ b/src/iceberg/transaction.cc @@ -23,6 +23,7 @@ #include #include "iceberg/catalog.h" +#include "iceberg/location_provider.h" #include "iceberg/schema.h" #include "iceberg/snapshot.h" #include "iceberg/statistics_file.h" @@ -86,6 +87,11 @@ const TableMetadata& TransactionContext::current() const { return metadata_builder->current(); } +Result> TransactionContext::NewLocationProvider() + const { + return iceberg::LocationProvider::Make(current().location, current().properties); +} + std::string TransactionContext::MetadataFileLocation(std::string_view filename) const { const auto metadata_location = current().properties.Get(TableProperties::kWriteMetadataLocation); diff --git a/src/iceberg/transaction.h b/src/iceberg/transaction.h index 47714ba70e..42a5045653 100644 --- a/src/iceberg/transaction.h +++ b/src/iceberg/transaction.h @@ -20,6 +20,9 @@ #pragma once +/// \file iceberg/transaction.h +/// \brief Define transaction APIs. + #include #include #include @@ -185,6 +188,7 @@ class ICEBERG_EXPORT TransactionContext { const TableMetadata* base() const; const TableMetadata& current() const; std::string MetadataFileLocation(std::string_view filename) const; + Result> NewLocationProvider() const; std::shared_ptr
table; std::unique_ptr metadata_builder; diff --git a/src/iceberg/type_fwd.h b/src/iceberg/type_fwd.h index 35a493b48a..91e18b24c8 100644 --- a/src/iceberg/type_fwd.h +++ b/src/iceberg/type_fwd.h @@ -104,6 +104,7 @@ class GeometryType; /// \brief Data values. class Decimal; +class Literal; class Uuid; /// \brief Schema. diff --git a/src/iceberg/update/merging_snapshot_update.cc b/src/iceberg/update/merging_snapshot_update.cc index f1f0f511d0..09e495cf12 100644 --- a/src/iceberg/update/merging_snapshot_update.cc +++ b/src/iceberg/update/merging_snapshot_update.cc @@ -21,10 +21,13 @@ #include #include +#include +#include #include #include #include #include +#include #include #include "iceberg/constants.h" @@ -32,6 +35,8 @@ #include "iceberg/expression/expressions.h" #include "iceberg/expression/manifest_evaluator.h" #include "iceberg/expression/projections.h" +#include "iceberg/file_io.h" +#include "iceberg/location_provider.h" #include "iceberg/manifest/manifest_entry.h" #include "iceberg/manifest/manifest_group.h" #include "iceberg/manifest/manifest_list.h" @@ -39,6 +44,7 @@ #include "iceberg/manifest/manifest_util_internal.h" #include "iceberg/manifest/manifest_writer.h" #include "iceberg/partition_spec.h" +#include "iceberg/puffin_dv_io.h" #include "iceberg/schema.h" #include "iceberg/snapshot.h" #include "iceberg/table.h" @@ -48,6 +54,7 @@ #include "iceberg/util/content_file_util.h" #include "iceberg/util/macros.h" #include "iceberg/util/snapshot_util_internal.h" +#include "iceberg/util/struct_like_set.h" namespace iceberg { @@ -751,9 +758,11 @@ Result> MergingSnapshotUpdate::WriteNewDataManifests() } Result> -MergingSnapshotUpdate::MergeDVs() const { +MergingSnapshotUpdate::MergeDVs() { std::vector result; result.reserve(dvs_by_referenced_file_.size()); + std::vector groups; + std::vector> data_sequence_numbers; for (const auto& entry : dvs_by_referenced_file_.entries()) { const auto& referenced_file = entry.referenced_file; @@ -761,16 +770,85 @@ MergingSnapshotUpdate::MergeDVs() const { if (dvs.empty()) { continue; } - if (dvs.size() > 1) { - // TODO(Guotao): Merge duplicate DVs for one referenced data file once C++ - // has DVUtil/Puffin DV rewriting; Java merges them before writing manifests. - return NotImplemented( - "Merging multiple deletion vectors is not supported yet for referenced " - "data file: {}", - referenced_file); + if (dvs.size() == 1) { + result.push_back(dvs.front()); + continue; + } + + const auto& first_file = dvs.front().file; + const auto first_data_sequence_number = dvs.front().data_sequence_number; + const auto first_spec_id = first_file->partition_spec_id; + ICEBERG_PRECHECK(first_spec_id.has_value(), "DV must have a partition spec ID: {}", + first_file->file_path); + + for (const auto& dv : dvs) { + ICEBERG_PRECHECK(dv.data_sequence_number == first_data_sequence_number, + "Cannot merge DVs, mismatched sequence numbers for {}", + referenced_file); + ICEBERG_PRECHECK(dv.file->partition_spec_id == first_spec_id, + "Cannot merge DVs, mismatched partition specs for {}", + referenced_file); + // TODO(gangwu): Java DVUtil.validateCanMerge compares partitions with + // Comparators.forType(spec.partitionType()). Use a typed C++ comparator if + // StructLikeEqual's scalar equality is not enough for future partition types. + ICEBERG_ASSIGN_OR_RAISE(auto same_partition, + StructLikeEqual(dv.file->partition, first_file->partition)); + ICEBERG_PRECHECK(same_partition, + "Cannot merge DVs, mismatched partition tuples for {}", + referenced_file); + } + + ICEBERG_ASSIGN_OR_RAISE(auto spec, base().PartitionSpecById(*first_spec_id)); + DeletionVectorMergeGroup group{ + .referenced_data_file = referenced_file, + .spec = std::move(spec), + .partition = first_file->partition, + }; + group.delete_files.reserve(dvs.size()); + for (const auto& dv : dvs) { + group.delete_files.push_back(dv.file); } + groups.push_back(std::move(group)); + data_sequence_numbers.push_back(first_data_sequence_number); + } + + if (groups.empty()) { + return result; + } + + ICEBERG_ASSIGN_OR_RAISE(auto location_provider, ctx_->NewLocationProvider()); + auto output_path = location_provider->NewDataLocation( + std::format("merged-dvs-{}-{}.puffin", SnapshotId(), ++dv_merge_attempt_)); - result.push_back(dvs.front()); + auto merged_files = + PuffinDVIORegistry::MergeAndWriteDVs(groups, output_path, ctx_->table->io()); + if (!merged_files) { + std::ignore = DeleteFile(output_path); + return std::unexpected(std::move(merged_files.error())); + } + + std::unordered_map> merged_by_ref; + for (auto& file : merged_files.value()) { + ICEBERG_PRECHECK(file != nullptr, "Merged DV file must not be null"); + ICEBERG_PRECHECK(file->referenced_data_file.has_value(), + "Merged DV must have a referenced data file: {}", file->file_path); + const auto referenced_data_file = *file->referenced_data_file; + auto insert_result = merged_by_ref.emplace(referenced_data_file, std::move(file)); + ICEBERG_PRECHECK(insert_result.second, "Duplicate merged DV for {}", + referenced_data_file); + } + ICEBERG_PRECHECK(merged_by_ref.size() == groups.size(), + "Expected {} merged DVs but got {}", groups.size(), + merged_by_ref.size()); + + for (size_t pos = 0; pos < groups.size(); ++pos) { + auto iter = merged_by_ref.find(groups[pos].referenced_data_file); + ICEBERG_CHECK(iter != merged_by_ref.end(), "Missing merged DV for {}", + groups[pos].referenced_data_file); + PendingDeleteFile merged{.file = std::move(iter->second), + .data_sequence_number = data_sequence_numbers[pos]}; + merged_dvs_.push_back(merged); + result.push_back(std::move(merged)); } return result; @@ -783,6 +861,13 @@ Result> MergingSnapshotUpdate::WriteNewDeleteManifests std::ignore = DeleteFile(m.manifest_path); } cached_new_delete_manifests_.clear(); + std::unordered_set deleted_merged_dv_paths; + for (const auto& dv : merged_dvs_) { + if (deleted_merged_dv_paths.insert(dv.file->file_path).second) { + std::ignore = DeleteFile(dv.file->file_path); + } + } + merged_dvs_.clear(); added_delete_files_summary_.Clear(); } @@ -988,6 +1073,15 @@ Status MergingSnapshotUpdate::CleanUncommittedAppends( // rewritten_append_manifests_ are always owned by the table. ICEBERG_RETURN_UNEXPECTED( DeleteUncommitted(rewritten_append_manifests_, committed, /*clear=*/true)); + if (committed.empty()) { + std::unordered_set deleted_merged_dv_paths; + for (const auto& dv : merged_dvs_) { + if (deleted_merged_dv_paths.insert(dv.file->file_path).second) { + std::ignore = DeleteFile(dv.file->file_path); + } + } + } + merged_dvs_.clear(); // append_manifests_ are only owned by the table if the commit succeeded. if (!committed.empty()) { diff --git a/src/iceberg/update/merging_snapshot_update.h b/src/iceberg/update/merging_snapshot_update.h index dbfb79937f..1c08a0667d 100644 --- a/src/iceberg/update/merging_snapshot_update.h +++ b/src/iceberg/update/merging_snapshot_update.h @@ -347,7 +347,7 @@ class ICEBERG_EXPORT MergingSnapshotUpdate : public SnapshotUpdate { void SetSummaryProperty(const std::string& property, const std::string& value) override; - Result> MergeDVs() const; + Result> MergeDVs(); /// \brief Write new data manifests for staged data files; caches the result. Result> WriteNewDataManifests(); @@ -396,6 +396,8 @@ class ICEBERG_EXPORT MergingSnapshotUpdate : public SnapshotUpdate { std::vector cached_new_data_manifests_; std::vector cached_new_delete_manifests_; + std::vector merged_dvs_; + int32_t dv_merge_attempt_ = 0; }; } // namespace iceberg diff --git a/src/iceberg/update/snapshot_update.h b/src/iceberg/update/snapshot_update.h index 10bb16fbcb..e29d96d579 100644 --- a/src/iceberg/update/snapshot_update.h +++ b/src/iceberg/update/snapshot_update.h @@ -19,6 +19,9 @@ #pragma once +/// \file iceberg/update/snapshot_update.h +/// \brief Define base APIs for snapshot updates. + #include #include #include diff --git a/src/iceberg/update/update_properties.cc b/src/iceberg/update/update_properties.cc index 1837ab009c..d2e75c3012 100644 --- a/src/iceberg/update/update_properties.cc +++ b/src/iceberg/update/update_properties.cc @@ -50,10 +50,10 @@ UpdateProperties& UpdateProperties::Set(const std::string& key, "Cannot set property '{}' that is already marked for removal", key); - if (!TableProperties::reserved_properties().contains(key) || - key == TableProperties::kFormatVersion.key()) { - updates_.insert_or_assign(key, value); - } + ICEBERG_BUILDER_CHECK(!TableProperties::reserved_properties().contains(key) || + key == TableProperties::kFormatVersion.key(), + "Cannot set reserved property: '{}'", key); + updates_.insert_or_assign(key, value); return *this; } diff --git a/src/iceberg/update/update_properties.h b/src/iceberg/update/update_properties.h index dee923410b..18eba427bb 100644 --- a/src/iceberg/update/update_properties.h +++ b/src/iceberg/update/update_properties.h @@ -51,8 +51,9 @@ class ICEBERG_EXPORT UpdateProperties : public PendingUpdate { /// \brief Sets a property key to a specified value. /// - /// The key must not have been previously marked for removal and reserved property keys - /// will be ignored. + /// The key must not have been previously marked for removal and must not be a + /// reserved property key (except `format-version`). Setting a reserved property + /// will result in a validation error at Apply() time. /// /// \param key The property key to set /// \param value The property value to set diff --git a/src/iceberg/update/update_schema.cc b/src/iceberg/update/update_schema.cc index 5c50ee41da..56e167e108 100644 --- a/src/iceberg/update/update_schema.cc +++ b/src/iceberg/update/update_schema.cc @@ -29,6 +29,7 @@ #include #include +#include "iceberg/expression/literal.h" #include "iceberg/json_serde_internal.h" #include "iceberg/name_mapping.h" #include "iceberg/schema.h" @@ -39,6 +40,7 @@ #include "iceberg/type.h" #include "iceberg/util/checked_cast.h" #include "iceberg/util/error_collector.h" +#include "iceberg/util/formatter.h" // IWYU pragma: keep #include "iceberg/util/macros.h" #include "iceberg/util/string_util.h" #include "iceberg/util/type_util.h" @@ -214,11 +216,9 @@ class ApplyChangesVisitor { if (update_it != updates_.end()) { const auto& update_field = update_it->second; - return SchemaField(field_id, update_field->name(), std::move(result_type), - update_field->optional(), update_field->doc()); + return update_field->WithType(std::move(result_type)); } else if (result_type != field.type()) { - return SchemaField(field_id, field.name(), std::move(result_type), field.optional(), - field.doc()); + return field.WithType(std::move(result_type)); } else { return field; } @@ -282,6 +282,10 @@ std::vector ApplyChangesVisitor::MoveFields( return reordered; } +bool ValidateDefaultLiteral(const Literal& value) { + return !value.IsNull() && !value.IsAboveMax() && !value.IsBelowMin(); +} + } // namespace Result> UpdateSchema::Make( @@ -345,39 +349,43 @@ UpdateSchema& UpdateSchema::CaseSensitive(bool case_sensitive) { } UpdateSchema& UpdateSchema::AddColumn(std::string_view name, std::shared_ptr type, - std::string_view doc) { + std::string_view doc, + std::optional default_value) { ICEBERG_BUILDER_CHECK(!name.contains('.'), "Cannot add column with ambiguous name: {}, use " "AddColumn(parent, name, type, doc)", name); - return AddColumnInternal(std::nullopt, name, /*is_optional=*/true, std::move(type), - doc); + return AddColumnInternal(/*parent=*/std::nullopt, name, /*is_optional=*/true, + std::move(type), doc, std::move(default_value)); } UpdateSchema& UpdateSchema::AddColumn(std::optional parent, std::string_view name, std::shared_ptr type, - std::string_view doc) { + std::string_view doc, + std::optional default_value) { return AddColumnInternal(std::move(parent), name, /*is_optional=*/true, std::move(type), - doc); + doc, std::move(default_value)); } UpdateSchema& UpdateSchema::AddRequiredColumn(std::string_view name, std::shared_ptr type, - std::string_view doc) { + std::string_view doc, + std::optional default_value) { ICEBERG_BUILDER_CHECK(!name.contains('.'), "Cannot add column with ambiguous name: {}, use " "AddRequiredColumn(parent, name, type, doc)", name); return AddColumnInternal(std::nullopt, name, /*is_optional=*/false, std::move(type), - doc); + doc, std::move(default_value)); } UpdateSchema& UpdateSchema::AddRequiredColumn(std::optional parent, std::string_view name, std::shared_ptr type, - std::string_view doc) { + std::string_view doc, + std::optional default_value) { return AddColumnInternal(std::move(parent), name, /*is_optional=*/false, - std::move(type), doc); + std::move(type), doc, std::move(default_value)); } UpdateSchema& UpdateSchema::UpdateColumn(std::string_view name, @@ -399,8 +407,25 @@ UpdateSchema& UpdateSchema::UpdateColumn(std::string_view name, "Cannot change column type: {}: {} -> {}", name, field.type()->ToString(), new_type->ToString()); - updates_[field_id] = std::make_shared( - field.field_id(), field.name(), new_type, field.optional(), field.doc()); + // Defaults must follow the promoted type. + auto promote_default = [&](const std::shared_ptr& value) + -> Result> { + if (value == nullptr) { + return nullptr; + } + ICEBERG_ASSIGN_OR_RAISE(Literal promoted, + SchemaField::CastDefaultValue(*value, new_type)); + return std::make_shared(std::move(promoted)); + }; + ICEBERG_BUILDER_ASSIGN_OR_RETURN(std::shared_ptr initial_default, + promote_default(field.initial_default())); + ICEBERG_BUILDER_ASSIGN_OR_RETURN(std::shared_ptr write_default, + promote_default(field.write_default())); + + updates_[field_id] = + std::make_shared(field.WithType(new_type) + .WithInitialDefault(std::move(initial_default)) + .WithWriteDefault(std::move(write_default))); return *this; } @@ -420,9 +445,44 @@ UpdateSchema& UpdateSchema::UpdateColumnDoc(std::string_view name, return *this; } - updates_[field_id] = - std::make_shared(field.field_id(), field.name(), field.type(), - field.optional(), std::string(new_doc)); + updates_[field_id] = std::make_shared(field.WithDoc(new_doc)); + + return *this; +} + +UpdateSchema& UpdateSchema::UpdateColumnDefault(std::string_view name, + std::optional new_default) { + ICEBERG_BUILDER_ASSIGN_OR_RETURN(auto field_opt, FindFieldForUpdate(name)); + ICEBERG_BUILDER_CHECK(field_opt.has_value(), "Cannot update missing column: {}", name); + + const auto& field = field_opt->get(); + int32_t field_id = field.field_id(); + + ICEBERG_BUILDER_CHECK(!deletes_.contains(field_id), + "Cannot update a column that will be deleted: {}", field.name()); + + if (!new_default.has_value()) { + updates_[field_id] = std::make_shared(field.WithWriteDefault(nullptr)); + return *this; + } + + ICEBERG_BUILDER_CHECK(field.type()->is_primitive(), + "Invalid default value for {}: {} (must be null)", *field.type(), + *new_default); + ICEBERG_BUILDER_ASSIGN_OR_RETURN_WITH_ERROR( + Literal typed_default, + SchemaField::CastDefaultValue( + *new_default, internal::checked_pointer_cast(field.type())), + "Cannot cast default value to {}: {}", *field.type(), *new_default); + ICEBERG_BUILDER_CHECK(ValidateDefaultLiteral(typed_default), + "Cannot cast default value to {}: {}", *field.type(), + *new_default); + if (field.write_default() != nullptr && *field.write_default() == typed_default) { + return *this; + } + + updates_[field_id] = std::make_shared( + field.WithWriteDefault(std::make_shared(std::move(typed_default)))); return *this; } @@ -443,9 +503,7 @@ UpdateSchema& UpdateSchema::RenameColumn(std::string_view name, const SchemaField& base_field = update_it != updates_.end() ? *update_it->second : field; - updates_[field_id] = std::make_shared( - base_field.field_id(), std::string(new_name), base_field.type(), - base_field.optional(), base_field.doc()); + updates_[field_id] = std::make_shared(base_field.WithName(new_name)); auto it = std::ranges::find(identifier_field_names_, name); if (it != identifier_field_names_.end()) { @@ -474,9 +532,9 @@ UpdateSchema& UpdateSchema::UpdateColumnRequirementInternal(std::string_view nam return *this; } - // TODO(GuotaoYu): support added column with default value - // bool is_defaulted_add = IsAdded(name) && field.initial_default() != null; - bool is_defaulted_add = false; + // Defaulted adds can become required. + bool is_defaulted_add = added_name_to_id_.contains(CaseSensitivityAwareName(name)) && + field.initial_default() != nullptr; ICEBERG_BUILDER_CHECK(is_optional || is_defaulted_add || allow_incompatible_changes_, "Cannot change column nullability: {}: optional -> required", @@ -625,13 +683,15 @@ Result UpdateSchema::Apply() { .updated_props = std::move(updated_props)}; } -// TODO(Guotao Yu): v3 default value is not yet supported UpdateSchema& UpdateSchema::AddColumnInternal(std::optional parent, std::string_view name, bool is_optional, std::shared_ptr type, - std::string_view doc) { + std::string_view doc, + std::optional default_value) { int32_t parent_id = kTableRootId; std::string full_name; + // For map/list adds, this omits synthetic value/element path segments. + std::string short_name; if (parent.has_value()) { ICEBERG_BUILDER_CHECK(!parent->empty(), "Parent name cannot be empty"); @@ -661,13 +721,13 @@ UpdateSchema& UpdateSchema::AddColumnInternal(std::optional pa ICEBERG_BUILDER_CHECK(!deletes_.contains(parent_id), "Cannot add to a column that will be deleted: {}", *parent); - auto current_name = std::format("{}.{}", *parent, name); - ICEBERG_BUILDER_ASSIGN_OR_RETURN(auto current_field, FindField(current_name)); + short_name = std::format("{}.{}", *parent, name); + ICEBERG_BUILDER_ASSIGN_OR_RETURN(auto current_field, FindField(short_name)); ICEBERG_BUILDER_CHECK( !current_field.has_value() || deletes_.contains(current_field->get().field_id()), "Cannot add column, name already exists: {}.{}", *parent, name); - // Build full name using canonical name of parent + // Use the canonical parent name for schema updates. ICEBERG_BUILDER_ASSIGN_OR_RETURN(auto parent_name_opt, schema_->FindColumnNameById(parent_id)); ICEBERG_BUILDER_CHECK(parent_name_opt.has_value(), @@ -680,16 +740,19 @@ UpdateSchema& UpdateSchema::AddColumnInternal(std::optional pa "Cannot add column, name already exists: {}", name); full_name = std::string(name); + short_name = full_name; } ICEBERG_BUILDER_CHECK( - is_optional || allow_incompatible_changes_, + is_optional || default_value.has_value() || allow_incompatible_changes_, "Incompatible change: cannot add required column without a default value: {}", full_name); int32_t new_id = AssignNewColumnId(); added_name_to_id_[CaseSensitivityAwareName(full_name)] = new_id; + // Needed for same-update lookups by user-facing path. + added_name_to_id_[CaseSensitivityAwareName(short_name)] = new_id; if (parent_id != kTableRootId) { id_to_parent_[new_id] = parent_id; } @@ -697,11 +760,30 @@ UpdateSchema& UpdateSchema::AddColumnInternal(std::optional pa AssignFreshIdVisitor id_assigner([this]() { return AssignNewColumnId(); }); auto type_with_fresh_ids = id_assigner.Visit(type); - auto new_field = std::make_shared(new_id, std::string(name), - std::move(type_with_fresh_ids), - is_optional, std::string(doc)); + std::shared_ptr initial_default; + std::shared_ptr write_default; + if (default_value.has_value()) { + ICEBERG_BUILDER_CHECK(type_with_fresh_ids->is_primitive(), + "Invalid default value for {}: {} (must be null)", + *type_with_fresh_ids, *default_value); + ICEBERG_BUILDER_ASSIGN_OR_RETURN_WITH_ERROR( + Literal typed_default, + SchemaField::CastDefaultValue( + *default_value, + internal::checked_pointer_cast(type_with_fresh_ids)), + "Cannot cast default value to {}: {}", *type_with_fresh_ids, *default_value); + ICEBERG_BUILDER_CHECK(ValidateDefaultLiteral(typed_default), + "Cannot cast default value to {}: {}", *type_with_fresh_ids, + *default_value); + // Add-column defaults set both default kinds. + auto shared_default = std::make_shared(std::move(typed_default)); + initial_default = shared_default; + write_default = std::move(shared_default); + } - updates_[new_id] = std::move(new_field); + updates_[new_id] = std::make_shared( + new_id, name, std::move(type_with_fresh_ids), is_optional, doc, + std::move(initial_default), std::move(write_default)); parent_to_added_ids_[parent_id].push_back(new_id); return *this; diff --git a/src/iceberg/update/update_schema.h b/src/iceberg/update/update_schema.h index 2be3732a06..d0dd944ef1 100644 --- a/src/iceberg/update/update_schema.h +++ b/src/iceberg/update/update_schema.h @@ -30,6 +30,7 @@ #include #include +#include "iceberg/expression/literal.h" #include "iceberg/iceberg_export.h" #include "iceberg/result.h" #include "iceberg/type_fwd.h" @@ -41,10 +42,6 @@ namespace iceberg { /// /// When committing, these changes will be applied to the current table metadata. /// Commit conflicts will not be resolved and will result in a CommitFailed error. -/// -/// TODO(Guotao Yu): Add support for V3 default values when adding columns. Currently, all -/// added columns use null as the default value, but Iceberg V3 supports custom -/// default values for new columns. class ICEBERG_EXPORT UpdateSchema : public PendingUpdate { public: static Result> Make( @@ -78,15 +75,17 @@ class ICEBERG_EXPORT UpdateSchema : public PendingUpdate { /// If type is a nested type, its field IDs are reassigned when added to the /// existing schema. /// - /// The added column will be optional with a null default value. + /// v3+: `default_value` sets `initial-default` and `write-default`. /// /// \param name Name for the new column. /// \param type Type for the new column. /// \param doc Documentation string for the new column. + /// \param default_value Optional default value for the new column (v3+). /// \return Reference to this for method chaining. /// \note InvalidArgument will be reported if name contains ".". UpdateSchema& AddColumn(std::string_view name, std::shared_ptr type, - std::string_view doc = ""); + std::string_view doc = "", + std::optional default_value = std::nullopt); /// \brief Add a new optional column to a nested struct with documentation. /// @@ -102,22 +101,23 @@ class ICEBERG_EXPORT UpdateSchema : public PendingUpdate { /// If type is a nested type, its field IDs are reassigned when added to the /// existing schema. /// - /// The added column will be optional with a null default value. + /// v3+: `default_value` sets `initial-default` and `write-default`. /// /// \param parent Name of the parent struct to which the column will be added. /// \param name Name for the new column. /// \param type Type for the new column. /// \param doc Documentation string for the new column. + /// \param default_value Optional default value for the new column (v3+). /// \return Reference to this for method chaining. /// \note InvalidArgument will be reported if parent doesn't identify a struct. UpdateSchema& AddColumn(std::optional parent, std::string_view name, - std::shared_ptr type, std::string_view doc = ""); + std::shared_ptr type, std::string_view doc = "", + std::optional default_value = std::nullopt); /// \brief Add a new required top-level column with documentation. /// - /// Adding a required column without a default is an incompatible change that can - /// break reading older data. To suppress exceptions thrown when an incompatible - /// change is detected, call AllowIncompatibleChanges(). + /// Required adds need AllowIncompatibleChanges() unless `default_value` is set. + /// v3+: `default_value` sets `initial-default` and `write-default`. /// /// Because "." may be interpreted as a column path separator or may be used in /// field names, it is not allowed in names passed to this method. To add to nested @@ -130,16 +130,17 @@ class ICEBERG_EXPORT UpdateSchema : public PendingUpdate { /// \param name Name for the new column. /// \param type Type for the new column. /// \param doc Documentation string for the new column. + /// \param default_value Optional default value for the new column (v3+). /// \return Reference to this for method chaining. /// \note InvalidArgument will be reported if name contains ".". UpdateSchema& AddRequiredColumn(std::string_view name, std::shared_ptr type, - std::string_view doc = ""); + std::string_view doc = "", + std::optional default_value = std::nullopt); /// \brief Add a new required column to a nested struct with documentation. /// - /// Adding a required column without a default is an incompatible change that can - /// break reading older data. To suppress exceptions thrown when an incompatible - /// change is detected, call AllowIncompatibleChanges(). + /// Required adds need AllowIncompatibleChanges() unless `default_value` is set. + /// v3+: `default_value` sets `initial-default` and `write-default`. /// /// The parent name is used to find the parent using Schema::FindFieldByName(). If /// the parent name is null or empty, the new column will be added to the root as a @@ -157,11 +158,13 @@ class ICEBERG_EXPORT UpdateSchema : public PendingUpdate { /// \param name Name for the new column. /// \param type Type for the new column. /// \param doc Documentation string for the new column. + /// \param default_value Optional default value for the new column (v3+). /// \return Reference to this for method chaining. /// \note InvalidArgument will be reported if parent doesn't identify a struct. UpdateSchema& AddRequiredColumn(std::optional parent, std::string_view name, std::shared_ptr type, - std::string_view doc = ""); + std::string_view doc = "", + std::optional default_value = std::nullopt); /// \brief Rename a column in the schema. /// @@ -210,6 +213,20 @@ class ICEBERG_EXPORT UpdateSchema : public PendingUpdate { /// the column will be deleted. UpdateSchema& UpdateColumnDoc(std::string_view name, std::string_view new_doc); + /// \brief Update the `write-default` value for a column (v3+). + /// + /// The name is used to find the column to update using Schema::FindFieldByName(). + /// Only `write-default` changes; `initial-default` is fixed. + /// + /// \param name Name of the column to update the default value for. + /// \param new_default Replacement `write-default` value for the column, or + /// `std::nullopt` to clear it. + /// \return Reference to this for method chaining. + /// \note InvalidArgument will be reported if name doesn't identify a column in the + /// schema or if the column will be deleted. + UpdateSchema& UpdateColumnDefault(std::string_view name, + std::optional new_default); + /// \brief Update a column to be optional. /// /// \param name Name of the column to mark optional. @@ -364,10 +381,12 @@ class ICEBERG_EXPORT UpdateSchema : public PendingUpdate { /// \param is_optional Whether the column is optional. /// \param type Type for the new column. /// \param doc Optional documentation string. + /// \param default_value Optional default value for the new column (v3+). /// \return Reference to this for method chaining. UpdateSchema& AddColumnInternal(std::optional parent, std::string_view name, bool is_optional, - std::shared_ptr type, std::string_view doc); + std::shared_ptr type, std::string_view doc, + std::optional default_value); /// \brief Internal implementation for updating column requirement (optional/required). /// diff --git a/src/iceberg/util/bucket_util.h b/src/iceberg/util/bucket_util.h index 31a574b480..459f487bb6 100644 --- a/src/iceberg/util/bucket_util.h +++ b/src/iceberg/util/bucket_util.h @@ -19,6 +19,9 @@ #pragma once +/// \file iceberg/util/bucket_util.h +/// \brief Provide bucket transform utilities. + #include #include diff --git a/src/iceberg/util/config.h b/src/iceberg/util/config.h index 089c78f5d8..ed847eb631 100644 --- a/src/iceberg/util/config.h +++ b/src/iceberg/util/config.h @@ -18,6 +18,9 @@ */ #pragma once +/// \file iceberg/util/config.h +/// \brief Provide typed configuration helpers. + #include #include #include diff --git a/src/iceberg/util/endian.h b/src/iceberg/util/endian.h index 638273a1cb..8aa559f406 100644 --- a/src/iceberg/util/endian.h +++ b/src/iceberg/util/endian.h @@ -112,4 +112,21 @@ T ReadLittleEndian(const void* input) { return FromLittleEndian(value); } +/// \brief Write a value in big-endian format to a buffer. +/// \note Caller must ensure output has at least sizeof(T) bytes available. +template +void WriteBigEndian(T value, void* output) { + auto be = ToBigEndian(value); + std::memcpy(output, &be, sizeof(be)); +} + +/// \brief Read a value in big-endian format from a buffer. +/// \note Caller must ensure input has at least sizeof(T) bytes available. +template +T ReadBigEndian(const void* input) { + T value; + std::memcpy(&value, input, sizeof(value)); + return FromBigEndian(value); +} + } // namespace iceberg diff --git a/src/iceberg/util/executor.h b/src/iceberg/util/executor.h index 01aa83d7a7..749502b179 100644 --- a/src/iceberg/util/executor.h +++ b/src/iceberg/util/executor.h @@ -19,6 +19,9 @@ #pragma once +/// \file iceberg/util/executor.h +/// \brief Define task executor interfaces. + #include #include diff --git a/src/iceberg/util/functional.h b/src/iceberg/util/functional.h index c5ab2164b0..dbb35a3761 100644 --- a/src/iceberg/util/functional.h +++ b/src/iceberg/util/functional.h @@ -22,6 +22,9 @@ #pragma once +/// \file iceberg/util/functional.h +/// \brief Provide move-only function wrappers. + #include #include #include diff --git a/src/iceberg/util/gzip_internal.cc b/src/iceberg/util/gzip_internal.cc index 680e2bc335..63e5acb2a4 100644 --- a/src/iceberg/util/gzip_internal.cc +++ b/src/iceberg/util/gzip_internal.cc @@ -19,11 +19,15 @@ #include "iceberg/util/gzip_internal.h" +#include #include #include +#include +#include #include +#include "iceberg/result.h" #include "iceberg/util/macros.h" namespace iceberg { diff --git a/src/iceberg/util/location_util.h b/src/iceberg/util/location_util.h index eb78dece3b..c213a3bf78 100644 --- a/src/iceberg/util/location_util.h +++ b/src/iceberg/util/location_util.h @@ -19,6 +19,9 @@ #pragma once +/// \file iceberg/util/location_util.h +/// \brief Provide location path utilities. + #include #include "iceberg/iceberg_export.h" diff --git a/src/iceberg/util/macros.h b/src/iceberg/util/macros.h index e391d2bfd1..400573d8b1 100644 --- a/src/iceberg/util/macros.h +++ b/src/iceberg/util/macros.h @@ -19,6 +19,9 @@ #pragma once +/// \file iceberg/util/macros.h +/// \brief Define common Iceberg utility macros. + #include #include "iceberg/exception.h" diff --git a/src/iceberg/util/partition_value_util.h b/src/iceberg/util/partition_value_util.h index 0113623ba7..abfc1ca873 100644 --- a/src/iceberg/util/partition_value_util.h +++ b/src/iceberg/util/partition_value_util.h @@ -19,7 +19,7 @@ #pragma once -/// \file iceberg/row/partition_values.h +/// \file iceberg/util/partition_value_util.h /// Wrapper classes for partition value related data structures. #include diff --git a/src/iceberg/util/property_util.h b/src/iceberg/util/property_util.h index 4e3e9b1259..c46d3a6abd 100644 --- a/src/iceberg/util/property_util.h +++ b/src/iceberg/util/property_util.h @@ -19,6 +19,9 @@ #pragma once +/// \file iceberg/util/property_util.h +/// \brief Provide property conversion helpers. + #include #include diff --git a/src/iceberg/util/retry_util.h b/src/iceberg/util/retry_util.h index 08e3fb81d6..8d2ed2a183 100644 --- a/src/iceberg/util/retry_util.h +++ b/src/iceberg/util/retry_util.h @@ -19,6 +19,9 @@ #pragma once +/// \file iceberg/util/retry_util.h +/// \brief Provide retry policy and runner utilities. + #include #include #include diff --git a/src/iceberg/util/string_util.cc b/src/iceberg/util/string_util.cc index 0454a62b58..b4c717861f 100644 --- a/src/iceberg/util/string_util.cc +++ b/src/iceberg/util/string_util.cc @@ -19,10 +19,60 @@ #include "iceberg/util/string_util.h" +#include + +#include +#include +#include +#include +#include +#include + +#include "iceberg/result.h" #include "iceberg/util/macros.h" namespace iceberg { +std::string StringUtils::ToLower(std::string_view str) { + std::string result; + result.reserve(str.size()); + + // Lower-case ASCII bytes directly; hand non-ASCII bytes to utf8proc. The common inputs + // (modes, UUIDs, header/property names, enum-like strings) are pure ASCII and never + // touch utf8proc. utf8proc has no string-level helper, so each non-ASCII code point is + // decoded, mapped with utf8proc_tolower (simple 1:1 mapping, not casefolding), and + // re-encoded. + const auto* data = reinterpret_cast(str.data()); + const auto size = static_cast(str.size()); + utf8proc_ssize_t offset = 0; + while (offset < size) { + // An ASCII byte is a complete 1-byte code point (never a UTF-8 continuation byte), + // and utf8proc_tolower agrees with ToLowerAscii on it, so handle it without utf8proc. + if (IsAsciiByte(str[offset])) { + result.push_back(ToLowerAscii(str[offset])); + ++offset; + continue; + } + utf8proc_int32_t code_point = 0; + utf8proc_ssize_t consumed = + utf8proc_iterate(data + offset, size - offset, &code_point); + if (consumed < 0) { + // Invalid UTF-8: pass the offending byte through unchanged and resume decoding at + // the next byte, so the valid code points around it are still lower-cased. + result.push_back(str[offset]); + ++offset; + continue; + } + const utf8proc_int32_t lowered = utf8proc_tolower(code_point); + std::array encoded{}; + const utf8proc_ssize_t written = utf8proc_encode_char(lowered, encoded.data()); + result.append(reinterpret_cast(encoded.data()), + static_cast(written)); + offset += consumed; + } + return result; +} + Result> StringUtils::HexStringToBytes(std::string_view hex) { if (hex.size() % 2 != 0) [[unlikely]] { return InvalidArgument("Hex string must have even length, got: {}", hex.size()); diff --git a/src/iceberg/util/string_util.h b/src/iceberg/util/string_util.h index 01b6087b8c..8202e60a39 100644 --- a/src/iceberg/util/string_util.h +++ b/src/iceberg/util/string_util.h @@ -19,10 +19,13 @@ #pragma once +/// \file iceberg/util/string_util.h +/// \brief Provide string utility helpers. + #include -#include #include #include +#include #include #include #include @@ -41,29 +44,61 @@ concept FromChars = requires(const char* p, T& v) { std::from_chars(p, p, v); }; class ICEBERG_EXPORT StringUtils { public: - // NOTE: These convert ASCII letters only; all other bytes, including non-ASCII - // (multibyte UTF-8) bytes, are passed through unchanged. - // See https://github.com/apache/iceberg-cpp/issues/613. - static std::string ToLower(std::string_view str) { - return str | std::ranges::views::transform(ToLowerAscii) | - std::ranges::to(); - } - + /// \brief Lower-case a UTF-8 string using Unicode simple (1:1) case mapping. + /// + /// Intended for case-insensitive name matching, similar to Iceberg Java's + /// toLowerCase(Locale.ROOT). The mapping is locale-independent, matching the intent + /// of Locale.ROOT. It uses simple (1:1) case mapping rather than Java's full case + /// mapping, so results differ for a few code points; e.g. U+0130 (capital I with dot + /// above) maps to U+0069 ("i") here, but to U+0069 U+0307 ("i" + combining dot above) + /// in Java. For ASCII and the large majority of letters the two agree. + /// + /// Pure-ASCII input takes a byte-wise fast path; utf8proc is only invoked when a + /// non-ASCII byte (>= 0x80) is present. The function is total: it never fails, and + /// input need not be valid UTF-8. A byte that does not begin a valid UTF-8 sequence + /// is copied through unchanged and decoding resumes at the next byte, so the valid + /// code points around it are still lower-cased. + /// See https://github.com/apache/iceberg-cpp/issues/613. + static std::string ToLower(std::string_view str); + + /// \brief Upper-case the ASCII letters (a-z) in a string; all other bytes, including + /// multi-byte UTF-8 sequences, are left unchanged. + /// + /// Deliberately ASCII-only and, unlike ToLower, not Unicode-aware. It is only used to + /// normalize ASCII enum/codec strings (e.g. "gzip" -> "GZIP", "all" -> "ALL") for + /// case-insensitive comparison. A Unicode upper-case is intentionally not provided: + /// simple case mapping would be wrong for some letters (e.g. "ß" (U+00DF) would stay + /// unchanged instead of becoming "SS"), and no caller needs it. static std::string ToUpper(std::string_view str) { return str | std::ranges::views::transform(ToUpperAscii) | std::ranges::to(); } + /// \brief Case-insensitive equality using Unicode simple (1:1) case mapping. + /// + /// Equal when the ToLower forms of both operands are equal, so folding follows + /// ToLower's rules (e.g. "İ" (U+0130) folds to "i"). Defined for any byte sequence: + /// ToLower passes invalid UTF-8 bytes through unchanged, so they compare verbatim. static bool EqualsIgnoreCase(std::string_view lhs, std::string_view rhs) { - return std::ranges::equal( - lhs, rhs, [](char lc, char rc) { return ToLowerAscii(lc) == ToLowerAscii(rc); }); + const std::optional fast = AsciiEqualsIgnoreCase(lhs, rhs); + return fast.has_value() ? *fast : (ToLower(lhs) == ToLower(rhs)); } + /// \brief Case-insensitive prefix test using Unicode simple (1:1) case mapping. + /// + /// True when the ToLower form of str starts with the ToLower form of prefix, so folding + /// follows ToLower's rules (e.g. "İ" (U+0130) folds to "i"). Defined for any byte + /// sequence: ToLower passes invalid UTF-8 bytes through unchanged, so they compare + /// verbatim. static bool StartsWithIgnoreCase(std::string_view str, std::string_view prefix) { - if (str.size() < prefix.size()) { - return false; + if (prefix.size() <= str.size()) { + const std::optional fast = + AsciiEqualsIgnoreCase(str.substr(0, prefix.size()), prefix); + if (fast.has_value()) { + return *fast; + } } - return EqualsIgnoreCase(str.substr(0, prefix.size()), prefix); + return ToLower(str).starts_with(ToLower(prefix)); } /// \brief Count the number of code points in a UTF-8 string. @@ -134,16 +169,37 @@ class ICEBERG_EXPORT StringUtils { } private: - // ASCII-only case conversion using explicit range checks rather than - // std::tolower/std::toupper. This is independent of the current C locale and never - // touches non-ASCII (high-bit) bytes, so multibyte UTF-8 sequences are preserved. It - // also sidesteps the undefined behavior of passing a negative char to . + // ASCII-only case mappings. These avoid std::toupper/std::tolower, which are + // locale-dependent and have undefined behavior for negative char values. + static constexpr char ToUpperAscii(char c) noexcept { + return (c >= 'a' && c <= 'z') ? static_cast(c - 'a' + 'A') : c; + } static constexpr char ToLowerAscii(char c) noexcept { return (c >= 'A' && c <= 'Z') ? static_cast(c - 'A' + 'a') : c; } - static constexpr char ToUpperAscii(char c) noexcept { - return (c >= 'a' && c <= 'z') ? static_cast(c - 'a' + 'A') : c; + // True if c is a 7-bit ASCII byte (< 0x80). The cast is required because char may be + // signed, which would make bytes >= 0x80 compare as negative. + static constexpr bool IsAsciiByte(char c) noexcept { + return (static_cast(c) & 0x80) == 0; + } + + // Case-insensitive equality decided in a single byte-wise pass, without allocating. + // Returns nullopt once a byte of either operand is non-ASCII, because folding can then + // be non-ASCII and length-changing (e.g. "İ" (U+0130) -> "i"), which only ToLower + // knows. + static std::optional AsciiEqualsIgnoreCase(std::string_view a, + std::string_view b) { + const size_t n = std::min(a.size(), b.size()); + for (size_t i = 0; i < n; ++i) { + if (!IsAsciiByte(a[i]) || !IsAsciiByte(b[i])) { + return std::nullopt; + } + if (ToLowerAscii(a[i]) != ToLowerAscii(b[i])) { + return false; + } + } + return a.size() == b.size(); } }; @@ -170,4 +226,11 @@ struct ICEBERG_EXPORT StringEqual { } }; +/// \brief Transparent less function that supports std::string_view as lookup key +struct ICEBERG_EXPORT StringLess { + using is_transparent = void; + + bool operator()(std::string_view lhs, std::string_view rhs) const { return lhs < rhs; } +}; + } // namespace iceberg diff --git a/src/iceberg/util/struct_like_set.cc b/src/iceberg/util/struct_like_set.cc index 35a0f9e280..fe622c8e5b 100644 --- a/src/iceberg/util/struct_like_set.cc +++ b/src/iceberg/util/struct_like_set.cc @@ -365,7 +365,7 @@ Status ValidateRowAgainstTypes(const StructLike& row, return {}; } -Result StructLikeEqual(const StructLike& lhs, const StructLike& rhs) { +Result StructLikeEqualInternal(const StructLike& lhs, const StructLike& rhs) { if (lhs.num_fields() != rhs.num_fields()) { return false; } @@ -417,7 +417,7 @@ Result MapLikeEqual(const MapLike& lhs, const MapLike& rhs) { } bool StructLikeEqualUnchecked(const StructLike& lhs, const StructLike& rhs) noexcept { - auto result = StructLikeEqual(lhs, rhs); + auto result = StructLikeEqualInternal(lhs, rhs); ICEBERG_DCHECK(result.has_value(), "Validated StructLike equality must not fail"); return result.value_or(false); } @@ -455,7 +455,7 @@ Result ScalarEqual(const Scalar& lhs, const Scalar& rhs) { const auto& r = std::get>(other); if (!l && !r) return true; if (!l || !r) return false; - return StructLikeEqual(*l, *r); + return StructLikeEqualInternal(*l, *r); }, [](const std::shared_ptr& l, const Scalar& other) -> Result { const auto& r = std::get>(other); @@ -475,6 +475,10 @@ Result ScalarEqual(const Scalar& lhs, const Scalar& rhs) { } // namespace +Result StructLikeEqual(const StructLike& lhs, const StructLike& rhs) { + return StructLikeEqualInternal(lhs, rhs); +} + template StructLikeSet::StructLikeSet(const StructType& type, size_t arena_initial_size) : arena_(arena_initial_size) { diff --git a/src/iceberg/util/struct_like_set.h b/src/iceberg/util/struct_like_set.h index 1dda0621ca..db263c06df 100644 --- a/src/iceberg/util/struct_like_set.h +++ b/src/iceberg/util/struct_like_set.h @@ -34,6 +34,9 @@ namespace iceberg { +/// \brief Compare two StructLike rows by scalar values. +ICEBERG_EXPORT Result StructLikeEqual(const StructLike& lhs, const StructLike& rhs); + /// \brief A set of StructLike rows with type-aware hashing and equality. /// /// As StructLike uses view semantics, this set makes deep copies of inserted rows diff --git a/src/iceberg/util/task_group.h b/src/iceberg/util/task_group.h index 5263816e35..3d345c240d 100644 --- a/src/iceberg/util/task_group.h +++ b/src/iceberg/util/task_group.h @@ -19,6 +19,9 @@ #pragma once +/// \file iceberg/util/task_group.h +/// \brief Provide grouped task execution utilities. + #include #include #include diff --git a/src/iceberg/util/temporal_util.h b/src/iceberg/util/temporal_util.h index acb1f0e73b..4a0642daa6 100644 --- a/src/iceberg/util/temporal_util.h +++ b/src/iceberg/util/temporal_util.h @@ -19,6 +19,9 @@ #pragma once +/// \file iceberg/util/temporal_util.h +/// \brief Provide date and timestamp conversion utilities. + #include #include #include diff --git a/src/iceberg/util/timepoint.h b/src/iceberg/util/timepoint.h index 34ffedfc79..3a12ccd227 100644 --- a/src/iceberg/util/timepoint.h +++ b/src/iceberg/util/timepoint.h @@ -19,6 +19,9 @@ #pragma once +/// \file iceberg/util/timepoint.h +/// \brief Provide time point conversion helpers. + #include #include "iceberg/iceberg_export.h" diff --git a/src/iceberg/util/transform_util.h b/src/iceberg/util/transform_util.h index e0918dcf30..77197d6ab7 100644 --- a/src/iceberg/util/transform_util.h +++ b/src/iceberg/util/transform_util.h @@ -19,6 +19,9 @@ #pragma once +/// \file iceberg/util/transform_util.h +/// \brief Provide partition transform utility helpers. + #include #include diff --git a/src/iceberg/util/truncate_util.h b/src/iceberg/util/truncate_util.h index 7fee86eba3..0ed6865463 100644 --- a/src/iceberg/util/truncate_util.h +++ b/src/iceberg/util/truncate_util.h @@ -19,6 +19,9 @@ #pragma once +/// \file iceberg/util/truncate_util.h +/// \brief Provide truncate transform utility helpers. + #include #include #include diff --git a/src/iceberg/util/type_util.cc b/src/iceberg/util/type_util.cc index 1f1e02747e..6656c5636c 100644 --- a/src/iceberg/util/type_util.cc +++ b/src/iceberg/util/type_util.cc @@ -385,9 +385,9 @@ std::shared_ptr AssignFreshIdVisitor::Visit(const StructType& type) std::vector fresh_fields; for (size_t i = 0; i < type.fields().size(); ++i) { const auto& field = type.fields()[i]; - fresh_fields.emplace_back(fresh_ids[i], std::string(field.name()), - Visit(field.type()), field.optional(), - std::string(field.doc())); + fresh_fields.emplace_back( + fresh_ids[i], std::string(field.name()), Visit(field.type()), field.optional(), + std::string(field.doc()), field.initial_default(), field.write_default()); } return std::make_shared(std::move(fresh_fields)); } @@ -397,7 +397,8 @@ std::shared_ptr AssignFreshIdVisitor::Visit(const ListType& type) cons int32_t fresh_id = next_id_(); SchemaField fresh_elem_field(fresh_id, std::string(elem_field.name()), Visit(elem_field.type()), elem_field.optional(), - std::string(elem_field.doc())); + std::string(elem_field.doc()), + elem_field.initial_default(), elem_field.write_default()); return std::make_shared(std::move(fresh_elem_field)); } @@ -410,10 +411,12 @@ std::shared_ptr AssignFreshIdVisitor::Visit(const MapType& type) const SchemaField fresh_key_field(fresh_key_id, std::string(key_field.name()), Visit(key_field.type()), key_field.optional(), - std::string(key_field.doc())); - SchemaField fresh_value_field(fresh_value_id, std::string(value_field.name()), - Visit(value_field.type()), value_field.optional(), - std::string(value_field.doc())); + std::string(key_field.doc()), key_field.initial_default(), + key_field.write_default()); + SchemaField fresh_value_field( + fresh_value_id, std::string(value_field.name()), Visit(value_field.type()), + value_field.optional(), std::string(value_field.doc()), + value_field.initial_default(), value_field.write_default()); return std::make_shared(std::move(fresh_key_field), std::move(fresh_value_field)); } diff --git a/src/iceberg/util/visitor_generate.h b/src/iceberg/util/visitor_generate.h index ad6f5eb2d1..37a9bb6c1a 100644 --- a/src/iceberg/util/visitor_generate.h +++ b/src/iceberg/util/visitor_generate.h @@ -19,6 +19,9 @@ #pragma once +/// \file iceberg/util/visitor_generate.h +/// \brief Define macros for generating type visitors. + namespace iceberg { #define ICEBERG_GENERATE_FOR_ALL_TYPES(ACTION) \ diff --git a/src/iceberg/version.h.in b/src/iceberg/version.h.in index af7fe01e6b..c7f8149fad 100644 --- a/src/iceberg/version.h.in +++ b/src/iceberg/version.h.in @@ -23,6 +23,8 @@ #define ICEBERG_VERSION_MINOR @PROJECT_VERSION_MINOR@ #define ICEBERG_VERSION_PATCH @PROJECT_VERSION_PATCH@ +#define ICEBERG_GIT_COMMIT_ID "@ICEBERG_GIT_COMMIT_ID@" #define ICEBERG_VERSION_STRING "@ICEBERG_VERSION_STRING@" +#define ICEBERG_FULL_VERSION_STRING "@ICEBERG_FULL_VERSION_STRING@" #define ICEBERG_PROJECT_NAME "@PROJECT_NAME@" diff --git a/subprojects/google-benchmark.wrap b/subprojects/google-benchmark.wrap new file mode 100644 index 0000000000..51f4e24d48 --- /dev/null +++ b/subprojects/google-benchmark.wrap @@ -0,0 +1,30 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +[wrap-file] +directory = benchmark-1.8.4 +source_url = https://github.com/google/benchmark/archive/refs/tags/v1.8.4.tar.gz +source_filename = benchmark-1.8.4.tar.gz +source_hash = 3e7059b6b11fb1bbe28e33e02519398ca94c1818874ebed18e504dc6f709be45 +source_fallback_url = https://github.com/mesonbuild/wrapdb/releases/download/google-benchmark_1.8.4-5/benchmark-1.8.4.tar.gz +patch_filename = google-benchmark_1.8.4-5_patch.zip +patch_url = https://wrapdb.mesonbuild.com/v2/google-benchmark_1.8.4-5/get_patch +patch_hash = 671ffed65f1e95e8c20edb7a06eb54476797e58169160b255f52dc71f4b83957 +wrapdb_version = 1.8.4-5 + +[provide] +dependency_names = benchmark, benchmark_main diff --git a/subprojects/utf8proc.wrap b/subprojects/utf8proc.wrap new file mode 100644 index 0000000000..9b33b3beaf --- /dev/null +++ b/subprojects/utf8proc.wrap @@ -0,0 +1,30 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +[wrap-file] +directory = utf8proc-2.10.0 +source_url = https://github.com/JuliaStrings/utf8proc/releases/download/v2.10.0/utf8proc-2.10.0.tar.gz +source_filename = utf8proc-2.10.0.tar.gz +source_hash = 276a37dc4d1dd24d7896826a579f4439d1e5fe33603add786bb083cab802e23e +patch_filename = utf8proc_2.10.0-1_patch.zip +patch_url = https://wrapdb.mesonbuild.com/v2/utf8proc_2.10.0-1/get_patch +patch_hash = be16c4514603e922f9636045699fe1a6f844d340b9b7c14b809e47253b06a844 +source_fallback_url = https://github.com/mesonbuild/wrapdb/releases/download/utf8proc_2.10.0-1/utf8proc-2.10.0.tar.gz +wrapdb_version = 2.10.0-1 + +[provide] +libutf8proc = utf8proc_dep