From 131f9763ecca8b5985e48bb06a957b4bebc6779f Mon Sep 17 00:00:00 2001 From: Abanoub Doss Date: Fri, 3 Jul 2026 22:17:12 -0500 Subject: [PATCH 01/23] ci: scope push trigger to main to drop redundant PR runs (#802) ## Change Make push triggers more consistent by having the test workflows run branch pushes only on `main`. Pull requests still run the same checks, and tag builds are unchanged. The duplicate-run case is narrow: it only happens for branches pushed directly to the Apache repo, so it mostly affects committers and bots rather than normal fork-based contributors. Pull request checks cover those branch updates; direct `main` runs still give us post-merge CI and warm caches. Also cleans up nearby `on:` sections so the triggers are easier to compare, including removing `zizmor`'s redundant all-branches pull request filter. ## Validation Checked that all workflow YAML files still parse. ## Remaining Trigger Inconsistencies These are intentional or out of scope, so this PR leaves them alone: - Test workflows still run on all tag pushes to preserve release behavior. - `rc` only runs on release-candidate tags. - `pre-commit` still runs on regular branch pushes because it is cheap. - `docs`, `cpp-linter`, `license_check`, and `codeql` keep their narrower rules. Related discussion: apache/iceberg-cpp#799. Co-authored-by: Abanoub Doss --- .github/workflows/asf-allowlist-check.yml | 4 ++-- .github/workflows/aws_test.yml | 3 +-- .github/workflows/codeql.yml | 6 ++++-- .github/workflows/cpp-linter.yml | 4 ++-- .github/workflows/pre-commit.yml | 4 ++-- .github/workflows/sanitizer_test.yml | 3 +-- .github/workflows/sql_catalog_test.yml | 3 +-- .github/workflows/test.yml | 3 +-- .github/workflows/zizmor.yml | 4 ++-- 9 files changed, 16 insertions(+), 18 deletions(-) diff --git a/.github/workflows/asf-allowlist-check.yml b/.github/workflows/asf-allowlist-check.yml index a2a36d4bb0..d642d5c590 100644 --- a/.github/workflows/asf-allowlist-check.yml +++ b/.github/workflows/asf-allowlist-check.yml @@ -24,11 +24,11 @@ name: "ASF Allowlist Check" on: - pull_request: - types: [opened, synchronize, reopened, ready_for_review] push: branches: - main + pull_request: + types: [opened, synchronize, reopened, ready_for_review] permissions: contents: read diff --git a/.github/workflows/aws_test.yml b/.github/workflows/aws_test.yml index cb486034ba..7726b94a7a 100644 --- a/.github/workflows/aws_test.yml +++ b/.github/workflows/aws_test.yml @@ -20,8 +20,7 @@ name: AWS Tests on: push: branches: - - '**' - - '!dependabot/**' + - main tags: - '**' pull_request: diff --git a/.github/workflows/codeql.yml b/.github/workflows/codeql.yml index 0060e8ffc1..2c03085ebd 100644 --- a/.github/workflows/codeql.yml +++ b/.github/workflows/codeql.yml @@ -21,10 +21,12 @@ name: "CodeQL" on: push: - branches: [ "main" ] + branches: + - main pull_request: types: [opened, synchronize, reopened, ready_for_review] - branches: [ "main" ] + branches: + - main schedule: - cron: '16 4 * * 1' diff --git a/.github/workflows/cpp-linter.yml b/.github/workflows/cpp-linter.yml index 304d1a58b2..49b056afee 100644 --- a/.github/workflows/cpp-linter.yml +++ b/.github/workflows/cpp-linter.yml @@ -27,12 +27,12 @@ on: - 'cmake_modules/**' pull_request: types: [opened, synchronize, reopened, ready_for_review] + branches: + - main paths-ignore: - '.github/**' - 'ci/**' - 'cmake_modules/**' - branches: - - main concurrency: group: ${{ github.repository }}-${{ github.head_ref || github.sha }}-${{ github.workflow }} diff --git a/.github/workflows/pre-commit.yml b/.github/workflows/pre-commit.yml index 417f1d5dd0..b79c9808aa 100644 --- a/.github/workflows/pre-commit.yml +++ b/.github/workflows/pre-commit.yml @@ -18,12 +18,12 @@ name: pre-commit on: - pull_request: - types: [opened, synchronize, reopened, ready_for_review] push: branches: - '**' - '!dependabot/**' + pull_request: + types: [opened, synchronize, reopened, ready_for_review] concurrency: group: ${{ github.repository }}-${{ github.head_ref || github.sha }}-${{ github.workflow }} diff --git a/.github/workflows/sanitizer_test.yml b/.github/workflows/sanitizer_test.yml index 48eb578466..f5eea55b90 100644 --- a/.github/workflows/sanitizer_test.yml +++ b/.github/workflows/sanitizer_test.yml @@ -20,8 +20,7 @@ name: ASAN and UBSAN Tests on: push: branches: - - '**' - - '!dependabot/**' + - main tags: - '**' pull_request: diff --git a/.github/workflows/sql_catalog_test.yml b/.github/workflows/sql_catalog_test.yml index 905b4a06c4..654a1abd76 100644 --- a/.github/workflows/sql_catalog_test.yml +++ b/.github/workflows/sql_catalog_test.yml @@ -20,8 +20,7 @@ name: SQL Catalog Tests on: push: branches: - - '**' - - '!dependabot/**' + - main tags: - '**' pull_request: diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 23fcc2d1c3..66222c6539 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -20,8 +20,7 @@ name: Test on: push: branches: - - '**' - - '!dependabot/**' + - main tags: - '**' pull_request: diff --git a/.github/workflows/zizmor.yml b/.github/workflows/zizmor.yml index 9b57dad34e..8201964700 100644 --- a/.github/workflows/zizmor.yml +++ b/.github/workflows/zizmor.yml @@ -21,10 +21,10 @@ name: GitHub Actions Security Analysis with zizmor 🌈 on: push: - branches: ["main"] + branches: + - main pull_request: types: [opened, synchronize, reopened, ready_for_review] - branches: ["**"] concurrency: group: ${{ github.repository }}-${{ github.head_ref || github.sha }}-${{ github.workflow }} From b7b047082004e4c12402b51df59ae5382a7ac79b Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 6 Jul 2026 11:09:06 +0800 Subject: [PATCH 02/23] chore(deps): bump actions/setup-python from 6.2.0 to 6.3.0 (#813) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Bumps [actions/setup-python](https://github.com/actions/setup-python) from 6.2.0 to 6.3.0.
Release notes

Sourced from actions/setup-python's releases.

v6.3.0

What's Changed

Enhancement

Dependency update

Documentation

New Contributors

Full Changelog: https://github.com/actions/setup-python/compare/v6...v6.3.0

Commits

[![Dependabot compatibility score](https://dependabot-badges.githubapp.com/badges/compatibility_score?dependency-name=actions/setup-python&package-manager=github_actions&previous-version=6.2.0&new-version=6.3.0)](https://docs.github.com/en/github/managing-security-vulnerabilities/about-dependabot-security-updates#about-compatibility-scores) Dependabot will resolve any conflicts with this PR as long as you don't alter it yourself. You can also trigger a rebase manually by commenting `@dependabot rebase`. [//]: # (dependabot-automerge-start) [//]: # (dependabot-automerge-end) ---
Dependabot commands and options
You can trigger Dependabot actions by commenting on this PR: - `@dependabot rebase` will rebase this PR - `@dependabot recreate` will recreate this PR, overwriting any edits that have been made to it - `@dependabot show ignore conditions` will show all of the ignore conditions of the specified dependency - `@dependabot ignore this major version` will close this PR and stop Dependabot creating any more for this major version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this minor version` will close this PR and stop Dependabot creating any more for this minor version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this dependency` will close this PR and stop Dependabot creating any more for this dependency (unless you reopen the PR or upgrade to it yourself)
Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- .github/workflows/aws_test.yml | 2 +- .github/workflows/docs.yml | 2 +- .github/workflows/pre-commit.yml | 2 +- .github/workflows/test.yml | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/aws_test.yml b/.github/workflows/aws_test.yml index 7726b94a7a..95f86c5f66 100644 --- a/.github/workflows/aws_test.yml +++ b/.github/workflows/aws_test.yml @@ -149,7 +149,7 @@ jobs: SCCACHE_DIR: ${{ github.workspace }}/.sccache SCCACHE_CACHE_SIZE: "2G" steps: - - uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0 + - uses: actions/setup-python@ece7cb06caefa5fff74198d8649806c4678c61a1 # v6.3.0 with: python-version: '3.x' - name: Checkout iceberg-cpp diff --git a/.github/workflows/docs.yml b/.github/workflows/docs.yml index 729d574c5c..083461083f 100644 --- a/.github/workflows/docs.yml +++ b/.github/workflows/docs.yml @@ -42,7 +42,7 @@ jobs: fetch-depth: 1 persist-credentials: false - - uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0 + - uses: actions/setup-python@ece7cb06caefa5fff74198d8649806c4678c61a1 # v6.3.0 with: python-version: '3.x' diff --git a/.github/workflows/pre-commit.yml b/.github/workflows/pre-commit.yml index b79c9808aa..209ac6918b 100644 --- a/.github/workflows/pre-commit.yml +++ b/.github/workflows/pre-commit.yml @@ -40,5 +40,5 @@ jobs: - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 with: persist-credentials: false - - uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0 + - uses: actions/setup-python@ece7cb06caefa5fff74198d8649806c4678c61a1 # v6.3.0 - uses: pre-commit/action@2c7b3805fd2a0fd8c1884dcaebf91fc102a13ecd # v3.0.1 diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 66222c6539..24f66318f6 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -223,7 +223,7 @@ jobs: - title: AArch64 macOS 26 runs-on: macos-26 steps: - - uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0 + - uses: actions/setup-python@ece7cb06caefa5fff74198d8649806c4678c61a1 # v6.3.0 with: python-version: '3.x' - name: Checkout iceberg-cpp From d068a813fcd01eb596a1366a628bc8276dba01da Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 6 Jul 2026 11:09:25 +0800 Subject: [PATCH 03/23] chore(deps): bump actions/cache from 5.0.5 to 6.1.0 (#814) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Bumps [actions/cache](https://github.com/actions/cache) from 5.0.5 to 6.1.0.
Release notes

Sourced from actions/cache's releases.

v6.1.0

What's Changed

Full Changelog: https://github.com/actions/cache/compare/v6...v6.1.0

v6.0.0

What's Changed

Full Changelog: https://github.com/actions/cache/compare/v5...v6.0.0

v5.1.0

What's Changed

Full Changelog: https://github.com/actions/cache/compare/v5...v5.1.0

Changelog

Sourced from actions/cache's changelog.

Releases

How to prepare a release

[!NOTE] Relevant for maintainers with write access only.

  1. Switch to a new branch from main.
  2. Run npm test to ensure all tests are passing.
  3. Update the version in https://github.com/actions/cache/blob/main/package.json.
  4. Run npm run build to update the compiled files.
  5. Update this https://github.com/actions/cache/blob/main/RELEASES.md with the new version and changes in the ## Changelog section.
  6. Run licensed cache to update the license report.
  7. Run licensed status and resolve any warnings by updating the https://github.com/actions/cache/blob/main/.licensed.yml file with the exceptions.
  8. Commit your changes and push your branch upstream.
  9. Open a pull request against main and get it reviewed and merged.
  10. Draft a new release https://github.com/actions/cache/releases use the same version number used in package.json
    1. Create a new tag with the version number.
    2. Auto generate release notes and update them to match the changes you made in RELEASES.md.
    3. Toggle the set as the latest release option.
    4. Publish the release.
  11. Navigate to https://github.com/actions/cache/actions/workflows/release-new-action-version.yml
    1. There should be a workflow run queued with the same version number.
    2. Approve the run to publish the new version and update the major tags for this action.

Changelog

6.1.0

6.0.0

  • Updated @actions/cache to ^6.0.1, @actions/core to ^3.0.1, @actions/exec to ^3.0.0, @actions/io to ^3.0.2
  • Migrated to ESM module system
  • Upgraded Jest to v30 and test infrastructure to be ESM compatible

5.0.4

  • Bump minimatch to v3.1.5 (fixes ReDoS via globstar patterns)
  • Bump undici to v6.24.1 (WebSocket decompression bomb protection, header validation fixes)
  • Bump fast-xml-parser to v5.5.6

5.0.3

5.0.2

... (truncated)

Commits
  • 55cc834 Merge pull request #1768 from jasongin/readonly-cache
  • d8cd72f Bump @​actions/cache to v6.1.0 - handle cache write error due to RO token
  • 2c8a9bd Merge pull request #1760 from actions/samirat/esm_migration_and_package_update
  • e9b91fd Prettier fixes
  • e4884b8 Rebuild dist
  • 10baf01 Fixed licenses
  • e39b386 Fix test mock return order
  • b692820 PR feedback
  • 6074912 Rebuild dist bundles as ESM to match type:module
  • 5a912e8 Fix lint and jest issues
  • Additional commits viewable in compare view

[![Dependabot compatibility score](https://dependabot-badges.githubapp.com/badges/compatibility_score?dependency-name=actions/cache&package-manager=github_actions&previous-version=5.0.5&new-version=6.1.0)](https://docs.github.com/en/github/managing-security-vulnerabilities/about-dependabot-security-updates#about-compatibility-scores) Dependabot will resolve any conflicts with this PR as long as you don't alter it yourself. You can also trigger a rebase manually by commenting `@dependabot rebase`. [//]: # (dependabot-automerge-start) [//]: # (dependabot-automerge-end) ---
Dependabot commands and options
You can trigger Dependabot actions by commenting on this PR: - `@dependabot rebase` will rebase this PR - `@dependabot recreate` will recreate this PR, overwriting any edits that have been made to it - `@dependabot show ignore conditions` will show all of the ignore conditions of the specified dependency - `@dependabot ignore this major version` will close this PR and stop Dependabot creating any more for this major version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this minor version` will close this PR and stop Dependabot creating any more for this minor version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this dependency` will close this PR and stop Dependabot creating any more for this dependency (unless you reopen the PR or upgrade to it yourself)
Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- .github/workflows/aws_test.yml | 4 ++-- .github/workflows/sql_catalog_test.yml | 2 +- .github/workflows/test.yml | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/aws_test.yml b/.github/workflows/aws_test.yml index 95f86c5f66..2fda2e667f 100644 --- a/.github/workflows/aws_test.yml +++ b/.github/workflows/aws_test.yml @@ -86,7 +86,7 @@ jobs: run: sudo apt-get update && sudo apt-get install -y libcurl4-openssl-dev libjitterentropy3-dev - name: Cache vcpkg packages if: ${{ startsWith(matrix.runs-on, 'ubuntu') && matrix.bundle_awssdk == 'OFF' }} - uses: actions/cache@27d5ce7f107fe9357f9df03efb73ab90386fccae # v5.0.5 + uses: actions/cache@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0 id: vcpkg-cache with: path: /usr/local/share/vcpkg/installed @@ -163,7 +163,7 @@ jobs: python3 -m pip install --upgrade pip python3 -m pip install -r requirements.txt - name: Cache vcpkg packages - uses: actions/cache@27d5ce7f107fe9357f9df03efb73ab90386fccae # v5.0.5 + uses: actions/cache@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0 id: vcpkg-cache with: path: /usr/local/share/vcpkg/installed diff --git a/.github/workflows/sql_catalog_test.yml b/.github/workflows/sql_catalog_test.yml index 654a1abd76..6ce3d083e3 100644 --- a/.github/workflows/sql_catalog_test.yml +++ b/.github/workflows/sql_catalog_test.yml @@ -104,7 +104,7 @@ jobs: echo "CXX=${{ matrix.CXX }}" >> $GITHUB_ENV - name: Cache vcpkg packages if: ${{ startsWith(matrix.runs-on, 'windows') }} - uses: actions/cache@27d5ce7f107fe9357f9df03efb73ab90386fccae # v5.0.5 + uses: actions/cache@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0 id: vcpkg-cache with: path: C:/vcpkg/installed diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 24f66318f6..d8cc85cd9f 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -162,7 +162,7 @@ jobs: Add-Content -Path $env:GITHUB_ENV -Value "MSVC_VER=$msvc" Write-Host "Resolved MSVC_VER=$msvc" - name: Cache vcpkg packages - uses: actions/cache@27d5ce7f107fe9357f9df03efb73ab90386fccae # v5.0.5 + uses: actions/cache@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0 id: vcpkg-cache with: path: C:/vcpkg/installed From c3bc6461a7cbc139b72d33230046b089d2f243da Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 6 Jul 2026 11:09:50 +0800 Subject: [PATCH 04/23] chore(deps): bump actions/cache/restore from 5.0.5 to 6.1.0 (#815) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Bumps [actions/cache/restore](https://github.com/actions/cache) from 5.0.5 to 6.1.0.
Release notes

Sourced from actions/cache/restore's releases.

v6.1.0

What's Changed

Full Changelog: https://github.com/actions/cache/compare/v6...v6.1.0

v6.0.0

What's Changed

Full Changelog: https://github.com/actions/cache/compare/v5...v6.0.0

v5.1.0

What's Changed

Full Changelog: https://github.com/actions/cache/compare/v5...v5.1.0

Changelog

Sourced from actions/cache/restore's changelog.

Releases

How to prepare a release

[!NOTE] Relevant for maintainers with write access only.

  1. Switch to a new branch from main.
  2. Run npm test to ensure all tests are passing.
  3. Update the version in https://github.com/actions/cache/blob/main/package.json.
  4. Run npm run build to update the compiled files.
  5. Update this https://github.com/actions/cache/blob/main/RELEASES.md with the new version and changes in the ## Changelog section.
  6. Run licensed cache to update the license report.
  7. Run licensed status and resolve any warnings by updating the https://github.com/actions/cache/blob/main/.licensed.yml file with the exceptions.
  8. Commit your changes and push your branch upstream.
  9. Open a pull request against main and get it reviewed and merged.
  10. Draft a new release https://github.com/actions/cache/releases use the same version number used in package.json
    1. Create a new tag with the version number.
    2. Auto generate release notes and update them to match the changes you made in RELEASES.md.
    3. Toggle the set as the latest release option.
    4. Publish the release.
  11. Navigate to https://github.com/actions/cache/actions/workflows/release-new-action-version.yml
    1. There should be a workflow run queued with the same version number.
    2. Approve the run to publish the new version and update the major tags for this action.

Changelog

6.1.0

6.0.0

  • Updated @actions/cache to ^6.0.1, @actions/core to ^3.0.1, @actions/exec to ^3.0.0, @actions/io to ^3.0.2
  • Migrated to ESM module system
  • Upgraded Jest to v30 and test infrastructure to be ESM compatible

5.0.4

  • Bump minimatch to v3.1.5 (fixes ReDoS via globstar patterns)
  • Bump undici to v6.24.1 (WebSocket decompression bomb protection, header validation fixes)
  • Bump fast-xml-parser to v5.5.6

5.0.3

5.0.2

... (truncated)

Commits
  • 55cc834 Merge pull request #1768 from jasongin/readonly-cache
  • d8cd72f Bump @​actions/cache to v6.1.0 - handle cache write error due to RO token
  • 2c8a9bd Merge pull request #1760 from actions/samirat/esm_migration_and_package_update
  • e9b91fd Prettier fixes
  • e4884b8 Rebuild dist
  • 10baf01 Fixed licenses
  • e39b386 Fix test mock return order
  • b692820 PR feedback
  • 6074912 Rebuild dist bundles as ESM to match type:module
  • 5a912e8 Fix lint and jest issues
  • Additional commits viewable in compare view

[![Dependabot compatibility score](https://dependabot-badges.githubapp.com/badges/compatibility_score?dependency-name=actions/cache/restore&package-manager=github_actions&previous-version=5.0.5&new-version=6.1.0)](https://docs.github.com/en/github/managing-security-vulnerabilities/about-dependabot-security-updates#about-compatibility-scores) Dependabot will resolve any conflicts with this PR as long as you don't alter it yourself. You can also trigger a rebase manually by commenting `@dependabot rebase`. [//]: # (dependabot-automerge-start) [//]: # (dependabot-automerge-end) ---
Dependabot commands and options
You can trigger Dependabot actions by commenting on this PR: - `@dependabot rebase` will rebase this PR - `@dependabot recreate` will recreate this PR, overwriting any edits that have been made to it - `@dependabot show ignore conditions` will show all of the ignore conditions of the specified dependency - `@dependabot ignore this major version` will close this PR and stop Dependabot creating any more for this major version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this minor version` will close this PR and stop Dependabot creating any more for this minor version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this dependency` will close this PR and stop Dependabot creating any more for this dependency (unless you reopen the PR or upgrade to it yourself)
Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- .github/workflows/aws_test.yml | 4 ++-- .github/workflows/cpp-linter.yml | 2 +- .github/workflows/sanitizer_test.yml | 2 +- .github/workflows/sql_catalog_test.yml | 2 +- .github/workflows/test.yml | 8 ++++---- 5 files changed, 9 insertions(+), 9 deletions(-) diff --git a/.github/workflows/aws_test.yml b/.github/workflows/aws_test.yml index 2fda2e667f..466f528738 100644 --- a/.github/workflows/aws_test.yml +++ b/.github/workflows/aws_test.yml @@ -115,7 +115,7 @@ jobs: shell: bash run: bash ci/scripts/start_minio.sh - name: Restore sccache cache - uses: actions/cache/restore@27d5ce7f107fe9357f9df03efb73ab90386fccae # v5.0.5 + uses: actions/cache/restore@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0 with: path: ${{ github.workspace }}/.sccache key: sccache-aws-${{ matrix.runs-on }}-bundle${{ matrix.bundle_awssdk }}-s3${{ matrix.s3 }}-sigv4${{ matrix.sigv4 }}-${{ github.run_id }} @@ -189,7 +189,7 @@ jobs: echo "CC=sccache gcc-14" >> $GITHUB_ENV echo "CXX=sccache g++-14" >> $GITHUB_ENV - name: Restore sccache cache - uses: actions/cache/restore@27d5ce7f107fe9357f9df03efb73ab90386fccae # v5.0.5 + uses: actions/cache/restore@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0 with: path: ${{ github.workspace }}/.sccache key: sccache-meson-sigv4-${{ github.run_id }} diff --git a/.github/workflows/cpp-linter.yml b/.github/workflows/cpp-linter.yml index 49b056afee..7a40cb110b 100644 --- a/.github/workflows/cpp-linter.yml +++ b/.github/workflows/cpp-linter.yml @@ -59,7 +59,7 @@ jobs: sudo apt-get update sudo apt-get install -y libcurl4-openssl-dev libsqlite3-dev libpq-dev default-libmysqlclient-dev - name: Restore sccache cache - uses: actions/cache/restore@27d5ce7f107fe9357f9df03efb73ab90386fccae # v5.0.5 + uses: actions/cache/restore@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0 with: path: ${{ github.workspace }}/.sccache key: sccache-cpp-linter-ubuntu-${{ github.run_id }} diff --git a/.github/workflows/sanitizer_test.yml b/.github/workflows/sanitizer_test.yml index f5eea55b90..7fea206b50 100644 --- a/.github/workflows/sanitizer_test.yml +++ b/.github/workflows/sanitizer_test.yml @@ -50,7 +50,7 @@ jobs: shell: bash run: sudo apt-get update && sudo apt-get install -y libcurl4-openssl-dev - name: Restore sccache cache - uses: actions/cache/restore@27d5ce7f107fe9357f9df03efb73ab90386fccae # v5.0.5 + uses: actions/cache/restore@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0 with: path: ${{ github.workspace }}/.sccache key: sccache-sanitizer-ubuntu-${{ github.run_id }} diff --git a/.github/workflows/sql_catalog_test.yml b/.github/workflows/sql_catalog_test.yml index 6ce3d083e3..4258a7e31a 100644 --- a/.github/workflows/sql_catalog_test.yml +++ b/.github/workflows/sql_catalog_test.yml @@ -115,7 +115,7 @@ jobs: run: | vcpkg install zlib:x64-windows nlohmann-json:x64-windows nanoarrow:x64-windows roaring:x64-windows sqlite3:x64-windows - name: Restore sccache cache - uses: actions/cache/restore@27d5ce7f107fe9357f9df03efb73ab90386fccae # v5.0.5 + uses: actions/cache/restore@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0 with: path: ${{ github.workspace }}/.sccache key: sccache-sqlcatalog-${{ matrix.runs-on }}${{ env.SCCACHE_KEY_SUFFIX }}-${{ github.run_id }} diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index d8cc85cd9f..700557952f 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -56,7 +56,7 @@ jobs: shell: bash run: sudo apt-get update && sudo apt-get install -y libcurl4-openssl-dev - name: Restore sccache cache - uses: actions/cache/restore@27d5ce7f107fe9357f9df03efb73ab90386fccae # v5.0.5 + uses: actions/cache/restore@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0 with: path: ${{ github.workspace }}/.sccache key: sccache-test-ubuntu-${{ github.run_id }} @@ -101,7 +101,7 @@ jobs: with: persist-credentials: false - name: Restore sccache cache - uses: actions/cache/restore@27d5ce7f107fe9357f9df03efb73ab90386fccae # v5.0.5 + uses: actions/cache/restore@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0 with: path: ${{ github.workspace }}/.sccache key: sccache-test-macos-${{ github.run_id }} @@ -173,7 +173,7 @@ jobs: run: | vcpkg install zlib:x64-windows nlohmann-json:x64-windows nanoarrow:x64-windows roaring:x64-windows cpr:x64-windows - name: Restore sccache cache - uses: actions/cache/restore@27d5ce7f107fe9357f9df03efb73ab90386fccae # v5.0.5 + uses: actions/cache/restore@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0 with: path: ${{ github.workspace }}/.sccache key: sccache-test-windows-${{ env.MSVC_VER }}-${{ github.run_id }} @@ -243,7 +243,7 @@ jobs: echo "CC=sccache ${{ matrix.CC }}" >> $GITHUB_ENV echo "CXX=sccache ${{ matrix.CXX }}" >> $GITHUB_ENV - name: Restore sccache cache - uses: actions/cache/restore@27d5ce7f107fe9357f9df03efb73ab90386fccae # v5.0.5 + uses: actions/cache/restore@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0 with: path: ${{ github.workspace }}/.sccache key: sccache-meson-${{ matrix.runs-on }}-${{ github.run_id }} From fce18ea53d16cea2ecfb26255320050b0ce6f6d8 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 6 Jul 2026 11:10:00 +0800 Subject: [PATCH 05/23] chore(deps): bump actions/cache/save from 5.0.5 to 6.1.0 (#816) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Bumps [actions/cache/save](https://github.com/actions/cache) from 5.0.5 to 6.1.0.
Release notes

Sourced from actions/cache/save's releases.

v6.1.0

What's Changed

Full Changelog: https://github.com/actions/cache/compare/v6...v6.1.0

v6.0.0

What's Changed

Full Changelog: https://github.com/actions/cache/compare/v5...v6.0.0

v5.1.0

What's Changed

Full Changelog: https://github.com/actions/cache/compare/v5...v5.1.0

Changelog

Sourced from actions/cache/save's changelog.

Releases

How to prepare a release

[!NOTE] Relevant for maintainers with write access only.

  1. Switch to a new branch from main.
  2. Run npm test to ensure all tests are passing.
  3. Update the version in https://github.com/actions/cache/blob/main/package.json.
  4. Run npm run build to update the compiled files.
  5. Update this https://github.com/actions/cache/blob/main/RELEASES.md with the new version and changes in the ## Changelog section.
  6. Run licensed cache to update the license report.
  7. Run licensed status and resolve any warnings by updating the https://github.com/actions/cache/blob/main/.licensed.yml file with the exceptions.
  8. Commit your changes and push your branch upstream.
  9. Open a pull request against main and get it reviewed and merged.
  10. Draft a new release https://github.com/actions/cache/releases use the same version number used in package.json
    1. Create a new tag with the version number.
    2. Auto generate release notes and update them to match the changes you made in RELEASES.md.
    3. Toggle the set as the latest release option.
    4. Publish the release.
  11. Navigate to https://github.com/actions/cache/actions/workflows/release-new-action-version.yml
    1. There should be a workflow run queued with the same version number.
    2. Approve the run to publish the new version and update the major tags for this action.

Changelog

6.1.0

6.0.0

  • Updated @actions/cache to ^6.0.1, @actions/core to ^3.0.1, @actions/exec to ^3.0.0, @actions/io to ^3.0.2
  • Migrated to ESM module system
  • Upgraded Jest to v30 and test infrastructure to be ESM compatible

5.0.4

  • Bump minimatch to v3.1.5 (fixes ReDoS via globstar patterns)
  • Bump undici to v6.24.1 (WebSocket decompression bomb protection, header validation fixes)
  • Bump fast-xml-parser to v5.5.6

5.0.3

5.0.2

... (truncated)

Commits
  • 55cc834 Merge pull request #1768 from jasongin/readonly-cache
  • d8cd72f Bump @​actions/cache to v6.1.0 - handle cache write error due to RO token
  • 2c8a9bd Merge pull request #1760 from actions/samirat/esm_migration_and_package_update
  • e9b91fd Prettier fixes
  • e4884b8 Rebuild dist
  • 10baf01 Fixed licenses
  • e39b386 Fix test mock return order
  • b692820 PR feedback
  • 6074912 Rebuild dist bundles as ESM to match type:module
  • 5a912e8 Fix lint and jest issues
  • Additional commits viewable in compare view

[![Dependabot compatibility score](https://dependabot-badges.githubapp.com/badges/compatibility_score?dependency-name=actions/cache/save&package-manager=github_actions&previous-version=5.0.5&new-version=6.1.0)](https://docs.github.com/en/github/managing-security-vulnerabilities/about-dependabot-security-updates#about-compatibility-scores) Dependabot will resolve any conflicts with this PR as long as you don't alter it yourself. You can also trigger a rebase manually by commenting `@dependabot rebase`. [//]: # (dependabot-automerge-start) [//]: # (dependabot-automerge-end) ---
Dependabot commands and options
You can trigger Dependabot actions by commenting on this PR: - `@dependabot rebase` will rebase this PR - `@dependabot recreate` will recreate this PR, overwriting any edits that have been made to it - `@dependabot show ignore conditions` will show all of the ignore conditions of the specified dependency - `@dependabot ignore this major version` will close this PR and stop Dependabot creating any more for this major version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this minor version` will close this PR and stop Dependabot creating any more for this minor version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this dependency` will close this PR and stop Dependabot creating any more for this dependency (unless you reopen the PR or upgrade to it yourself)
Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- .github/workflows/aws_test.yml | 4 ++-- .github/workflows/cpp-linter.yml | 2 +- .github/workflows/sanitizer_test.yml | 2 +- .github/workflows/sql_catalog_test.yml | 2 +- .github/workflows/test.yml | 8 ++++---- 5 files changed, 9 insertions(+), 9 deletions(-) diff --git a/.github/workflows/aws_test.yml b/.github/workflows/aws_test.yml index 466f528738..3e955d84f0 100644 --- a/.github/workflows/aws_test.yml +++ b/.github/workflows/aws_test.yml @@ -133,7 +133,7 @@ jobs: run: sccache --show-stats - name: Save sccache cache if: github.ref == 'refs/heads/main' - uses: actions/cache/save@27d5ce7f107fe9357f9df03efb73ab90386fccae # v5.0.5 + uses: actions/cache/save@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0 with: path: ${{ github.workspace }}/.sccache key: sccache-aws-${{ matrix.runs-on }}-bundle${{ matrix.bundle_awssdk }}-s3${{ matrix.s3 }}-sigv4${{ matrix.sigv4 }}-${{ github.run_id }} @@ -208,7 +208,7 @@ jobs: meson test -C builddir --timeout-multiplier 0 --print-errorlogs - name: Save sccache cache if: github.ref == 'refs/heads/main' - uses: actions/cache/save@27d5ce7f107fe9357f9df03efb73ab90386fccae # v5.0.5 + uses: actions/cache/save@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0 with: path: ${{ github.workspace }}/.sccache key: sccache-meson-sigv4-${{ github.run_id }} diff --git a/.github/workflows/cpp-linter.yml b/.github/workflows/cpp-linter.yml index 7a40cb110b..3dcf7ba4f3 100644 --- a/.github/workflows/cpp-linter.yml +++ b/.github/workflows/cpp-linter.yml @@ -86,7 +86,7 @@ jobs: run: sccache --show-stats - name: Save sccache cache if: github.ref == 'refs/heads/main' - uses: actions/cache/save@27d5ce7f107fe9357f9df03efb73ab90386fccae # v5.0.5 + uses: actions/cache/save@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0 with: path: ${{ github.workspace }}/.sccache key: sccache-cpp-linter-ubuntu-${{ github.run_id }} diff --git a/.github/workflows/sanitizer_test.yml b/.github/workflows/sanitizer_test.yml index 7fea206b50..cd9797b9fa 100644 --- a/.github/workflows/sanitizer_test.yml +++ b/.github/workflows/sanitizer_test.yml @@ -72,7 +72,7 @@ jobs: run: sccache --show-stats - name: Save sccache cache if: github.ref == 'refs/heads/main' - uses: actions/cache/save@27d5ce7f107fe9357f9df03efb73ab90386fccae # v5.0.5 + uses: actions/cache/save@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0 with: path: ${{ github.workspace }}/.sccache key: sccache-sanitizer-ubuntu-${{ github.run_id }} diff --git a/.github/workflows/sql_catalog_test.yml b/.github/workflows/sql_catalog_test.yml index 4258a7e31a..28bd331731 100644 --- a/.github/workflows/sql_catalog_test.yml +++ b/.github/workflows/sql_catalog_test.yml @@ -145,7 +145,7 @@ jobs: run: sccache --show-stats - name: Save sccache cache if: github.ref == 'refs/heads/main' - uses: actions/cache/save@27d5ce7f107fe9357f9df03efb73ab90386fccae # v5.0.5 + uses: actions/cache/save@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0 with: path: ${{ github.workspace }}/.sccache key: sccache-sqlcatalog-${{ matrix.runs-on }}${{ env.SCCACHE_KEY_SUFFIX }}-${{ github.run_id }} diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 700557952f..b46bb924cc 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -75,7 +75,7 @@ jobs: run: sccache --show-stats - name: Save sccache cache if: github.ref == 'refs/heads/main' - uses: actions/cache/save@27d5ce7f107fe9357f9df03efb73ab90386fccae # v5.0.5 + uses: actions/cache/save@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0 with: path: ${{ github.workspace }}/.sccache key: sccache-test-ubuntu-${{ github.run_id }} @@ -117,7 +117,7 @@ jobs: run: sccache --show-stats - name: Save sccache cache if: github.ref == 'refs/heads/main' - uses: actions/cache/save@27d5ce7f107fe9357f9df03efb73ab90386fccae # v5.0.5 + uses: actions/cache/save@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0 with: path: ${{ github.workspace }}/.sccache key: sccache-test-macos-${{ github.run_id }} @@ -190,7 +190,7 @@ jobs: sccache --show-stats - name: Save sccache cache if: github.ref == 'refs/heads/main' - uses: actions/cache/save@27d5ce7f107fe9357f9df03efb73ab90386fccae # v5.0.5 + uses: actions/cache/save@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0 with: path: ${{ github.workspace }}/.sccache key: sccache-test-windows-${{ env.MSVC_VER }}-${{ github.run_id }} @@ -259,7 +259,7 @@ jobs: run: sccache --show-stats - name: Save sccache cache if: github.ref == 'refs/heads/main' - uses: actions/cache/save@27d5ce7f107fe9357f9df03efb73ab90386fccae # v5.0.5 + uses: actions/cache/save@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0 with: path: ${{ github.workspace }}/.sccache key: sccache-meson-${{ matrix.runs-on }}-${{ github.run_id }} From 5d31dee04533b8f46709a157e100d084605a5354 Mon Sep 17 00:00:00 2001 From: wzhuo Date: Tue, 7 Jul 2026 13:44:19 +0800 Subject: [PATCH 06/23] refactor: move typed arrow Append helpers to ArrowRowBuilder (#821) Move AppendUInt, AppendDouble, AppendBytes, AppendIntList (2 overloads), AppendIntMap, and AppendBinaryMap from the anonymous namespace in manifest_adapter.cc to ArrowRowBuilder, the central home for typed nanoarrow append helpers. Remove the duplicate implementations and update all call sites in ManifestEntryAdapter, ManifestFileAdapter, and their nested methods to use the ArrowRowBuilder helpers. --- src/iceberg/arrow_row_builder.cc | 74 ++++++- src/iceberg/arrow_row_builder_internal.h | 28 +++ src/iceberg/manifest/manifest_adapter.cc | 249 ++++++++--------------- 3 files changed, 183 insertions(+), 168 deletions(-) diff --git a/src/iceberg/arrow_row_builder.cc b/src/iceberg/arrow_row_builder.cc index b53c881bb0..26e7cb4a2d 100644 --- a/src/iceberg/arrow_row_builder.cc +++ b/src/iceberg/arrow_row_builder.cc @@ -17,7 +17,9 @@ * under the License. */ -#include +#include +#include +#include #include @@ -109,12 +111,49 @@ Status AppendInt(ArrowArray* array, int64_t value) { return {}; } +Status AppendUInt(ArrowArray* array, uint64_t value) { + ICEBERG_NANOARROW_RETURN_UNEXPECTED(ArrowArrayAppendUInt(array, value)); + return {}; +} + +Status AppendDouble(ArrowArray* array, double value) { + ICEBERG_NANOARROW_RETURN_UNEXPECTED(ArrowArrayAppendDouble(array, value)); + return {}; +} + Status AppendString(ArrowArray* array, std::string_view value) { ArrowStringView view(value.data(), static_cast(value.size())); ICEBERG_NANOARROW_RETURN_UNEXPECTED(ArrowArrayAppendString(array, view)); return {}; } +Status AppendBytes(ArrowArray* array, std::span value) { + ArrowBufferViewData data; + data.as_char = reinterpret_cast(value.data()); + ArrowBufferView view(data, static_cast(value.size())); + ICEBERG_NANOARROW_RETURN_UNEXPECTED(ArrowArrayAppendBytes(array, view)); + return {}; +} + +Status AppendIntList(ArrowArray* array, const std::vector& values) { + auto list_array = array->children[0]; + for (const auto& value : values) { + ICEBERG_NANOARROW_RETURN_UNEXPECTED( + ArrowArrayAppendInt(list_array, static_cast(value))); + } + ICEBERG_NANOARROW_RETURN_UNEXPECTED(ArrowArrayFinishElement(array)); + return {}; +} + +Status AppendIntList(ArrowArray* array, const std::vector& values) { + auto list_array = array->children[0]; + for (const auto& value : values) { + ICEBERG_NANOARROW_RETURN_UNEXPECTED(ArrowArrayAppendInt(list_array, value)); + } + ICEBERG_NANOARROW_RETURN_UNEXPECTED(ArrowArrayFinishElement(array)); + return {}; +} + Status AppendStringMap(ArrowArray* array, const std::unordered_map& entries) { // A nanoarrow map array is a list of struct. children[0] is the @@ -134,4 +173,37 @@ Status AppendStringMap(ArrowArray* array, return {}; } +Status AppendIntMap(ArrowArray* array, const std::map& entries) { + auto map_array = array->children[0]; + if (map_array->n_children != 2) { + return InvalidArrowData("Map array must have exactly 2 children."); + } + for (const auto& [key, value] : entries) { + auto key_array = map_array->children[0]; + auto value_array = map_array->children[1]; + ICEBERG_RETURN_UNEXPECTED(AppendInt(key_array, static_cast(key))); + ICEBERG_RETURN_UNEXPECTED(AppendInt(value_array, value)); + ICEBERG_NANOARROW_RETURN_UNEXPECTED(ArrowArrayFinishElement(map_array)); + } + ICEBERG_NANOARROW_RETURN_UNEXPECTED(ArrowArrayFinishElement(array)); + return {}; +} + +Status AppendBinaryMap(ArrowArray* array, + const std::map>& entries) { + auto map_array = array->children[0]; + if (map_array->n_children != 2) { + return InvalidArrowData("Map array must have exactly 2 children."); + } + for (const auto& [key, value] : entries) { + auto key_array = map_array->children[0]; + auto value_array = map_array->children[1]; + ICEBERG_RETURN_UNEXPECTED(AppendInt(key_array, static_cast(key))); + ICEBERG_RETURN_UNEXPECTED(AppendBytes(value_array, value)); + ICEBERG_NANOARROW_RETURN_UNEXPECTED(ArrowArrayFinishElement(map_array)); + } + ICEBERG_NANOARROW_RETURN_UNEXPECTED(ArrowArrayFinishElement(array)); + return {}; +} + } // namespace iceberg diff --git a/src/iceberg/arrow_row_builder_internal.h b/src/iceberg/arrow_row_builder_internal.h index f3fd2b499d..db1b66f633 100644 --- a/src/iceberg/arrow_row_builder_internal.h +++ b/src/iceberg/arrow_row_builder_internal.h @@ -30,8 +30,11 @@ /// boilerplate. #include +#include +#include #include #include +#include #include "iceberg/arrow_c_data.h" #include "iceberg/iceberg_export.h" @@ -113,9 +116,26 @@ ICEBERG_EXPORT Status AppendBoolean(ArrowArray* array, bool value); /// Works for int32/int64/timestamp columns, which nanoarrow stores as int64. ICEBERG_EXPORT Status AppendInt(ArrowArray* array, int64_t value); +/// \brief Append an unsigned integer value to a nanoarrow array builder. +ICEBERG_EXPORT Status AppendUInt(ArrowArray* array, uint64_t value); + +/// \brief Append a double value to a nanoarrow array builder. +ICEBERG_EXPORT Status AppendDouble(ArrowArray* array, double value); + /// \brief Append a string value to a nanoarrow array builder. ICEBERG_EXPORT Status AppendString(ArrowArray* array, std::string_view value); +/// \brief Append a binary value to a nanoarrow array builder. +ICEBERG_EXPORT Status AppendBytes(ArrowArray* array, std::span value); + +/// \brief Append a list of int32 values to a nanoarrow list array builder. +ICEBERG_EXPORT Status AppendIntList(ArrowArray* array, + const std::vector& values); + +/// \brief Append a list of int64 values to a nanoarrow list array builder. +ICEBERG_EXPORT Status AppendIntList(ArrowArray* array, + const std::vector& values); + /// \brief Append a map value to a nanoarrow map array builder. /// /// Appends one (possibly empty) map element. The iteration order of the @@ -123,4 +143,12 @@ ICEBERG_EXPORT Status AppendString(ArrowArray* array, std::string_view value); ICEBERG_EXPORT Status AppendStringMap( ArrowArray* array, const std::unordered_map& entries); +/// \brief Append a map to a nanoarrow map array builder. +ICEBERG_EXPORT Status AppendIntMap(ArrowArray* array, + const std::map& entries); + +/// \brief Append a map to a nanoarrow map array builder. +ICEBERG_EXPORT Status AppendBinaryMap( + ArrowArray* array, const std::map>& entries); + } // namespace iceberg diff --git a/src/iceberg/manifest/manifest_adapter.cc b/src/iceberg/manifest/manifest_adapter.cc index 526d312716..4b6846ae3b 100644 --- a/src/iceberg/manifest/manifest_adapter.cc +++ b/src/iceberg/manifest/manifest_adapter.cc @@ -23,6 +23,7 @@ #include #include "iceberg/arrow/nanoarrow_status_internal.h" +#include "iceberg/arrow_row_builder_internal.h" #include "iceberg/manifest/manifest_adapter_internal.h" #include "iceberg/manifest/manifest_entry.h" #include "iceberg/manifest/manifest_list.h" @@ -38,87 +39,6 @@ namespace { constexpr int64_t kBlockSizeInBytesV1 = 64 * 1024 * 1024L; constexpr int32_t kBlockSizeInBytesFieldId = 105; -Status AppendField(ArrowArray* array, int64_t value) { - ICEBERG_NANOARROW_RETURN_UNEXPECTED(ArrowArrayAppendInt(array, value)); - return {}; -} - -Status AppendField(ArrowArray* array, uint64_t value) { - ICEBERG_NANOARROW_RETURN_UNEXPECTED(ArrowArrayAppendUInt(array, value)); - return {}; -} - -Status AppendField(ArrowArray* array, double value) { - ICEBERG_NANOARROW_RETURN_UNEXPECTED(ArrowArrayAppendDouble(array, value)); - return {}; -} - -Status AppendField(ArrowArray* array, std::string_view value) { - ArrowStringView view(value.data(), value.size()); - ICEBERG_NANOARROW_RETURN_UNEXPECTED(ArrowArrayAppendString(array, view)); - return {}; -} - -Status AppendField(ArrowArray* array, std::span value) { - ArrowBufferViewData data; - data.as_char = reinterpret_cast(value.data()); - ArrowBufferView view(data, value.size()); - ICEBERG_NANOARROW_RETURN_UNEXPECTED(ArrowArrayAppendBytes(array, view)); - return {}; -} - -Status AppendList(ArrowArray* array, const std::vector& list_value) { - auto list_array = array->children[0]; - for (const auto& value : list_value) { - ICEBERG_NANOARROW_RETURN_UNEXPECTED( - ArrowArrayAppendInt(list_array, static_cast(value))); - } - ICEBERG_NANOARROW_RETURN_UNEXPECTED(ArrowArrayFinishElement(array)); - return {}; -} - -Status AppendList(ArrowArray* array, const std::vector& list_value) { - auto list_array = array->children[0]; - for (const auto& value : list_value) { - ICEBERG_NANOARROW_RETURN_UNEXPECTED(ArrowArrayAppendInt(list_array, value)); - } - ICEBERG_NANOARROW_RETURN_UNEXPECTED(ArrowArrayFinishElement(array)); - return {}; -} - -Status AppendMap(ArrowArray* array, const std::map& map_value) { - auto map_array = array->children[0]; - if (map_array->n_children != 2) { - return InvalidArrowData("Map array must have exactly 2 children."); - } - for (const auto& [key, value] : map_value) { - auto key_array = map_array->children[0]; - auto value_array = map_array->children[1]; - ICEBERG_RETURN_UNEXPECTED(AppendField(key_array, static_cast(key))); - ICEBERG_RETURN_UNEXPECTED(AppendField(value_array, value)); - ICEBERG_NANOARROW_RETURN_UNEXPECTED(ArrowArrayFinishElement(map_array)); - } - ICEBERG_NANOARROW_RETURN_UNEXPECTED(ArrowArrayFinishElement(array)); - return {}; -} - -Status AppendMap(ArrowArray* array, - const std::map>& map_value) { - auto map_array = array->children[0]; - if (map_array->n_children != 2) { - return InvalidArrowData("Map array must have exactly 2 children."); - } - for (const auto& [key, value] : map_value) { - auto key_array = map_array->children[0]; - auto value_array = map_array->children[1]; - ICEBERG_RETURN_UNEXPECTED(AppendField(key_array, static_cast(key))); - ICEBERG_RETURN_UNEXPECTED(AppendField(value_array, value)); - ICEBERG_NANOARROW_RETURN_UNEXPECTED(ArrowArrayFinishElement(map_array)); - } - ICEBERG_NANOARROW_RETURN_UNEXPECTED(ArrowArrayFinishElement(array)); - return {}; -} - } // namespace Status ManifestAdapter::StartAppending() { @@ -179,45 +99,44 @@ Status ManifestEntryAdapter::AppendPartitionValues( const auto& partition_field = fields[i]; auto child_array = array->children[i]; if (partition_value.IsNull()) { - ICEBERG_NANOARROW_RETURN_UNEXPECTED(ArrowArrayAppendNull(child_array, 1)); + ICEBERG_RETURN_UNEXPECTED(AppendNull(child_array)); continue; } switch (partition_field.type()->type_id()) { case TypeId::kBoolean: - ICEBERG_RETURN_UNEXPECTED(AppendField( - child_array, static_cast( - std::get(partition_value.value()) == true ? 1L : 0L))); + ICEBERG_RETURN_UNEXPECTED( + AppendBoolean(child_array, std::get(partition_value.value()))); break; case TypeId::kInt: - ICEBERG_RETURN_UNEXPECTED(AppendField( - child_array, - static_cast(std::get(partition_value.value())))); + ICEBERG_RETURN_UNEXPECTED( + AppendInt(child_array, + static_cast(std::get(partition_value.value())))); break; case TypeId::kLong: ICEBERG_RETURN_UNEXPECTED( - AppendField(child_array, std::get(partition_value.value()))); + AppendInt(child_array, std::get(partition_value.value()))); break; case TypeId::kFloat: - ICEBERG_RETURN_UNEXPECTED(AppendField( + ICEBERG_RETURN_UNEXPECTED(AppendDouble( child_array, static_cast(std::get(partition_value.value())))); break; case TypeId::kDouble: ICEBERG_RETURN_UNEXPECTED( - AppendField(child_array, std::get(partition_value.value()))); + AppendDouble(child_array, std::get(partition_value.value()))); break; case TypeId::kString: ICEBERG_RETURN_UNEXPECTED( - AppendField(child_array, std::get(partition_value.value()))); + AppendString(child_array, std::get(partition_value.value()))); break; case TypeId::kFixed: case TypeId::kBinary: - ICEBERG_RETURN_UNEXPECTED(AppendField( + ICEBERG_RETURN_UNEXPECTED(AppendBytes( child_array, std::get>(partition_value.value()))); break; case TypeId::kDate: - ICEBERG_RETURN_UNEXPECTED(AppendField( - child_array, - static_cast(std::get(partition_value.value())))); + ICEBERG_RETURN_UNEXPECTED( + AppendInt(child_array, + static_cast(std::get(partition_value.value())))); break; case TypeId::kTime: case TypeId::kTimestamp: @@ -225,15 +144,15 @@ Status ManifestEntryAdapter::AppendPartitionValues( case TypeId::kTimestampNs: case TypeId::kTimestampTzNs: ICEBERG_RETURN_UNEXPECTED( - AppendField(child_array, std::get(partition_value.value()))); + AppendInt(child_array, std::get(partition_value.value()))); break; case TypeId::kDecimal: - ICEBERG_RETURN_UNEXPECTED(AppendField( + ICEBERG_RETURN_UNEXPECTED(AppendBytes( child_array, std::get(partition_value.value()).ToBytes())); break; case TypeId::kUuid: ICEBERG_RETURN_UNEXPECTED( - AppendField(child_array, std::get(partition_value.value()).bytes())); + AppendBytes(child_array, std::get(partition_value.value()).bytes())); break; case TypeId::kStruct: case TypeId::kList: @@ -259,13 +178,13 @@ Status ManifestEntryAdapter::AppendDataFile( switch (field.field_id()) { case DataFile::kContentFieldId: // optional int32 ICEBERG_RETURN_UNEXPECTED( - AppendField(child_array, static_cast(file.content))); + AppendInt(child_array, static_cast(file.content))); break; case DataFile::kFilePathFieldId: // required string - ICEBERG_RETURN_UNEXPECTED(AppendField(child_array, file.file_path)); + ICEBERG_RETURN_UNEXPECTED(AppendString(child_array, file.file_path)); break; case DataFile::kFileFormatFieldId: // required string - ICEBERG_RETURN_UNEXPECTED(AppendField(child_array, ToString(file.file_format))); + ICEBERG_RETURN_UNEXPECTED(AppendString(child_array, ToString(file.file_format))); break; case DataFile::kPartitionFieldId: { // required struct auto partition_type = internal::checked_pointer_cast(field.type()); @@ -273,85 +192,84 @@ Status ManifestEntryAdapter::AppendDataFile( AppendPartitionValues(child_array, partition_type, file.partition)); } break; case DataFile::kRecordCountFieldId: // required int64 - ICEBERG_RETURN_UNEXPECTED(AppendField(child_array, file.record_count)); + ICEBERG_RETURN_UNEXPECTED(AppendInt(child_array, file.record_count)); break; case DataFile::kFileSizeFieldId: // required int64 - ICEBERG_RETURN_UNEXPECTED(AppendField(child_array, file.file_size_in_bytes)); + ICEBERG_RETURN_UNEXPECTED(AppendInt(child_array, file.file_size_in_bytes)); break; case kBlockSizeInBytesFieldId: // compatible with v1 // always 64MB for v1 - ICEBERG_RETURN_UNEXPECTED(AppendField(child_array, kBlockSizeInBytesV1)); + ICEBERG_RETURN_UNEXPECTED(AppendInt(child_array, kBlockSizeInBytesV1)); break; case DataFile::kColumnSizesFieldId: // optional map - ICEBERG_RETURN_UNEXPECTED(AppendMap(child_array, file.column_sizes)); + ICEBERG_RETURN_UNEXPECTED(AppendIntMap(child_array, file.column_sizes)); break; case DataFile::kValueCountsFieldId: // optional map - ICEBERG_RETURN_UNEXPECTED(AppendMap(child_array, file.value_counts)); + ICEBERG_RETURN_UNEXPECTED(AppendIntMap(child_array, file.value_counts)); break; case DataFile::kNullValueCountsFieldId: // optional map - ICEBERG_RETURN_UNEXPECTED(AppendMap(child_array, file.null_value_counts)); + ICEBERG_RETURN_UNEXPECTED(AppendIntMap(child_array, file.null_value_counts)); break; case DataFile::kNanValueCountsFieldId: // optional map - ICEBERG_RETURN_UNEXPECTED(AppendMap(child_array, file.nan_value_counts)); + ICEBERG_RETURN_UNEXPECTED(AppendIntMap(child_array, file.nan_value_counts)); break; case DataFile::kLowerBoundsFieldId: // optional map - ICEBERG_RETURN_UNEXPECTED(AppendMap(child_array, file.lower_bounds)); + ICEBERG_RETURN_UNEXPECTED(AppendBinaryMap(child_array, file.lower_bounds)); break; case DataFile::kUpperBoundsFieldId: // optional map - ICEBERG_RETURN_UNEXPECTED(AppendMap(child_array, file.upper_bounds)); + ICEBERG_RETURN_UNEXPECTED(AppendBinaryMap(child_array, file.upper_bounds)); break; case DataFile::kKeyMetadataFieldId: // optional binary if (!file.key_metadata.empty()) { - ICEBERG_RETURN_UNEXPECTED(AppendField(child_array, file.key_metadata)); + ICEBERG_RETURN_UNEXPECTED(AppendBytes(child_array, file.key_metadata)); } else { - ICEBERG_NANOARROW_RETURN_UNEXPECTED(ArrowArrayAppendNull(child_array, 1)); + ICEBERG_RETURN_UNEXPECTED(AppendNull(child_array)); } break; case DataFile::kSplitOffsetsFieldId: // optional list - ICEBERG_RETURN_UNEXPECTED(AppendList(child_array, file.split_offsets)); + ICEBERG_RETURN_UNEXPECTED(AppendIntList(child_array, file.split_offsets)); break; case DataFile::kEqualityIdsFieldId: // optional list - ICEBERG_RETURN_UNEXPECTED(AppendList(child_array, file.equality_ids)); + ICEBERG_RETURN_UNEXPECTED(AppendIntList(child_array, file.equality_ids)); break; case DataFile::kSortOrderIdFieldId: // optional int32 if (file.sort_order_id.has_value()) { ICEBERG_RETURN_UNEXPECTED( - AppendField(child_array, static_cast(file.sort_order_id.value()))); + AppendInt(child_array, static_cast(file.sort_order_id.value()))); } else { - ICEBERG_NANOARROW_RETURN_UNEXPECTED(ArrowArrayAppendNull(child_array, 1)); + ICEBERG_RETURN_UNEXPECTED(AppendNull(child_array)); } break; case DataFile::kFirstRowIdFieldId: // optional int64 if (file.first_row_id.has_value()) { - ICEBERG_RETURN_UNEXPECTED(AppendField(child_array, file.first_row_id.value())); + ICEBERG_RETURN_UNEXPECTED(AppendInt(child_array, file.first_row_id.value())); } else { - ICEBERG_NANOARROW_RETURN_UNEXPECTED(ArrowArrayAppendNull(child_array, 1)); + ICEBERG_RETURN_UNEXPECTED(AppendNull(child_array)); } break; case DataFile::kReferencedDataFileFieldId: { // optional string ICEBERG_ASSIGN_OR_RAISE(auto referenced_data_file, GetReferenceDataFile(file)); if (referenced_data_file.has_value()) { ICEBERG_RETURN_UNEXPECTED( - AppendField(child_array, referenced_data_file.value())); + AppendString(child_array, referenced_data_file.value())); } else { - ICEBERG_NANOARROW_RETURN_UNEXPECTED(ArrowArrayAppendNull(child_array, 1)); + ICEBERG_RETURN_UNEXPECTED(AppendNull(child_array)); } break; } case DataFile::kContentOffsetFieldId: // optional int64 if (file.content_offset.has_value()) { - ICEBERG_RETURN_UNEXPECTED( - AppendField(child_array, file.content_offset.value())); + ICEBERG_RETURN_UNEXPECTED(AppendInt(child_array, file.content_offset.value())); } else { - ICEBERG_NANOARROW_RETURN_UNEXPECTED(ArrowArrayAppendNull(child_array, 1)); + ICEBERG_RETURN_UNEXPECTED(AppendNull(child_array)); } break; case DataFile::kContentSizeFieldId: // optional int64 if (file.content_size_in_bytes.has_value()) { ICEBERG_RETURN_UNEXPECTED( - AppendField(child_array, file.content_size_in_bytes.value())); + AppendInt(child_array, file.content_size_in_bytes.value())); } else { - ICEBERG_NANOARROW_RETURN_UNEXPECTED(ArrowArrayAppendNull(child_array, 1)); + ICEBERG_RETURN_UNEXPECTED(AppendNull(child_array)); } break; default: @@ -400,13 +318,13 @@ Status ManifestEntryAdapter::AppendInternal(const ManifestEntry& entry) { switch (field.field_id()) { case ManifestEntry::kStatusFieldId: // required int32 ICEBERG_RETURN_UNEXPECTED( - AppendField(array, static_cast(static_cast(entry.status)))); + AppendInt(array, static_cast(static_cast(entry.status)))); break; case ManifestEntry::kSnapshotIdFieldId: // optional int64 if (entry.snapshot_id.has_value()) { - ICEBERG_RETURN_UNEXPECTED(AppendField(array, entry.snapshot_id.value())); + ICEBERG_RETURN_UNEXPECTED(AppendInt(array, entry.snapshot_id.value())); } else { - ICEBERG_NANOARROW_RETURN_UNEXPECTED(ArrowArrayAppendNull(array, 1)); + ICEBERG_RETURN_UNEXPECTED(AppendNull(array)); } break; case ManifestEntry::kDataFileFieldId: // required struct @@ -422,18 +340,17 @@ Status ManifestEntryAdapter::AppendInternal(const ManifestEntry& entry) { case ManifestEntry::kSequenceNumberFieldId: { // optional int64 ICEBERG_ASSIGN_OR_RAISE(auto sequence_num, GetSequenceNumber(entry)); if (sequence_num.has_value()) { - ICEBERG_RETURN_UNEXPECTED(AppendField(array, sequence_num.value())); + ICEBERG_RETURN_UNEXPECTED(AppendInt(array, sequence_num.value())); } else { - ICEBERG_NANOARROW_RETURN_UNEXPECTED(ArrowArrayAppendNull(array, 1)); + ICEBERG_RETURN_UNEXPECTED(AppendNull(array)); } break; } case ManifestEntry::kFileSequenceNumberFieldId: // optional int64 if (entry.file_sequence_number.has_value()) { - ICEBERG_RETURN_UNEXPECTED( - AppendField(array, entry.file_sequence_number.value())); + ICEBERG_RETURN_UNEXPECTED(AppendInt(array, entry.file_sequence_number.value())); } else { - ICEBERG_NANOARROW_RETURN_UNEXPECTED(ArrowArrayAppendNull(array, 1)); + ICEBERG_RETURN_UNEXPECTED(AppendNull(array)); } break; default: @@ -470,18 +387,16 @@ Status ManifestFileAdapter::AppendPartitionSummary( switch (summary_field.field_id()) { case 509: // contains_null (required bool) ICEBERG_RETURN_UNEXPECTED( - AppendField(summary_array->children[0], - static_cast(summary.contains_null ? 1 : 0))); + AppendBoolean(summary_array->children[0], summary.contains_null)); break; case 518: { // contains_nan (optional bool) auto field_array = summary_array->children[1]; if (summary.contains_nan.has_value()) { ICEBERG_RETURN_UNEXPECTED( - AppendField(field_array, - static_cast(summary.contains_nan.value() ? 1 : 0))); + AppendBoolean(field_array, summary.contains_nan.value())); } else { - ICEBERG_NANOARROW_RETURN_UNEXPECTED(ArrowArrayAppendNull(field_array, 1)); + ICEBERG_RETURN_UNEXPECTED(AppendNull(field_array)); } break; } @@ -490,9 +405,9 @@ Status ManifestFileAdapter::AppendPartitionSummary( auto field_array = summary_array->children[2]; if (summary.lower_bound.has_value() && !summary.lower_bound->empty()) { ICEBERG_RETURN_UNEXPECTED( - AppendField(field_array, summary.lower_bound.value())); + AppendBytes(field_array, summary.lower_bound.value())); } else { - ICEBERG_NANOARROW_RETURN_UNEXPECTED(ArrowArrayAppendNull(field_array, 1)); + ICEBERG_RETURN_UNEXPECTED(AppendNull(field_array)); } break; } @@ -501,9 +416,9 @@ Status ManifestFileAdapter::AppendPartitionSummary( auto field_array = summary_array->children[3]; if (summary.upper_bound.has_value() && !summary.upper_bound->empty()) { ICEBERG_RETURN_UNEXPECTED( - AppendField(field_array, summary.upper_bound.value())); + AppendBytes(field_array, summary.upper_bound.value())); } else { - ICEBERG_NANOARROW_RETURN_UNEXPECTED(ArrowArrayAppendNull(field_array, 1)); + ICEBERG_RETURN_UNEXPECTED(AppendNull(field_array)); } break; } @@ -539,81 +454,81 @@ Status ManifestFileAdapter::AppendInternal(const ManifestFile& file) { auto array = array_.children[i]; switch (field.field_id()) { case ManifestFile::kManifestPathFieldId: - ICEBERG_RETURN_UNEXPECTED(AppendField(array, file.manifest_path)); + ICEBERG_RETURN_UNEXPECTED(AppendString(array, file.manifest_path)); break; case ManifestFile::kManifestLengthFieldId: - ICEBERG_RETURN_UNEXPECTED(AppendField(array, file.manifest_length)); + ICEBERG_RETURN_UNEXPECTED(AppendInt(array, file.manifest_length)); break; case ManifestFile::kPartitionSpecIdFieldId: ICEBERG_RETURN_UNEXPECTED( - AppendField(array, static_cast(file.partition_spec_id))); + AppendInt(array, static_cast(file.partition_spec_id))); break; case ManifestFile::kContentFieldId: ICEBERG_RETURN_UNEXPECTED( - AppendField(array, static_cast(static_cast(file.content)))); + AppendInt(array, static_cast(static_cast(file.content)))); break; case ManifestFile::kSequenceNumberFieldId: { ICEBERG_ASSIGN_OR_RAISE(auto sequence_num, GetSequenceNumber(file)); - ICEBERG_RETURN_UNEXPECTED(AppendField(array, sequence_num)); + ICEBERG_RETURN_UNEXPECTED(AppendInt(array, sequence_num)); break; } case ManifestFile::kMinSequenceNumberFieldId: { ICEBERG_ASSIGN_OR_RAISE(auto min_sequence_num, GetMinSequenceNumber(file)); - ICEBERG_RETURN_UNEXPECTED(AppendField(array, min_sequence_num)); + ICEBERG_RETURN_UNEXPECTED(AppendInt(array, min_sequence_num)); break; } case ManifestFile::kAddedSnapshotIdFieldId: - ICEBERG_RETURN_UNEXPECTED(AppendField(array, file.added_snapshot_id)); + ICEBERG_RETURN_UNEXPECTED(AppendInt(array, file.added_snapshot_id)); break; case ManifestFile::kAddedFilesCountFieldId: if (file.added_files_count.has_value()) { ICEBERG_RETURN_UNEXPECTED( - AppendField(array, static_cast(file.added_files_count.value()))); + AppendInt(array, static_cast(file.added_files_count.value()))); } else { // Append null for optional field - ICEBERG_NANOARROW_RETURN_UNEXPECTED(ArrowArrayAppendNull(array, 1)); + ICEBERG_RETURN_UNEXPECTED(AppendNull(array)); } break; case ManifestFile::kExistingFilesCountFieldId: if (file.existing_files_count.has_value()) { - ICEBERG_RETURN_UNEXPECTED(AppendField( - array, static_cast(file.existing_files_count.value()))); + ICEBERG_RETURN_UNEXPECTED( + AppendInt(array, static_cast(file.existing_files_count.value()))); } else { // Append null for optional field - ICEBERG_NANOARROW_RETURN_UNEXPECTED(ArrowArrayAppendNull(array, 1)); + ICEBERG_RETURN_UNEXPECTED(AppendNull(array)); } break; case ManifestFile::kDeletedFilesCountFieldId: if (file.deleted_files_count.has_value()) { ICEBERG_RETURN_UNEXPECTED( - AppendField(array, static_cast(file.deleted_files_count.value()))); + AppendInt(array, static_cast(file.deleted_files_count.value()))); } else { // Append null for optional field - ICEBERG_NANOARROW_RETURN_UNEXPECTED(ArrowArrayAppendNull(array, 1)); + ICEBERG_RETURN_UNEXPECTED(AppendNull(array)); } break; case ManifestFile::kAddedRowsCountFieldId: if (file.added_rows_count.has_value()) { - ICEBERG_RETURN_UNEXPECTED(AppendField(array, file.added_rows_count.value())); + ICEBERG_RETURN_UNEXPECTED(AppendInt(array, file.added_rows_count.value())); } else { // Append null for optional field - ICEBERG_NANOARROW_RETURN_UNEXPECTED(ArrowArrayAppendNull(array, 1)); + ICEBERG_RETURN_UNEXPECTED(AppendNull(array)); } break; case ManifestFile::kExistingRowsCountFieldId: if (file.existing_rows_count.has_value()) { - ICEBERG_RETURN_UNEXPECTED(AppendField(array, file.existing_rows_count.value())); + ICEBERG_RETURN_UNEXPECTED(AppendInt(array, file.existing_rows_count.value())); } else { // Append null for optional field - ICEBERG_NANOARROW_RETURN_UNEXPECTED(ArrowArrayAppendNull(array, 1)); + ICEBERG_RETURN_UNEXPECTED(AppendNull(array)); } break; case ManifestFile::kDeletedRowsCountFieldId: if (file.deleted_rows_count.has_value()) { - ICEBERG_RETURN_UNEXPECTED(AppendField(array, file.deleted_rows_count.value())); + ICEBERG_RETURN_UNEXPECTED(AppendInt(array, file.deleted_rows_count.value())); } else { // Append null for optional field - ICEBERG_NANOARROW_RETURN_UNEXPECTED(ArrowArrayAppendNull(array, 1)); + ICEBERG_RETURN_UNEXPECTED(AppendNull(array)); } break; case ManifestFile::kPartitionSummaryFieldId: @@ -623,18 +538,18 @@ Status ManifestFileAdapter::AppendInternal(const ManifestFile& file) { break; case ManifestFile::kKeyMetadataFieldId: if (!file.key_metadata.empty()) { - ICEBERG_RETURN_UNEXPECTED(AppendField(array, file.key_metadata)); + ICEBERG_RETURN_UNEXPECTED(AppendBytes(array, file.key_metadata)); } else { - ICEBERG_NANOARROW_RETURN_UNEXPECTED(ArrowArrayAppendNull(array, 1)); + ICEBERG_RETURN_UNEXPECTED(AppendNull(array)); } break; case ManifestFile::kFirstRowIdFieldId: { ICEBERG_ASSIGN_OR_RAISE(auto first_row_id, GetFirstRowId(file)); if (first_row_id.has_value()) { - ICEBERG_RETURN_UNEXPECTED(AppendField(array, first_row_id.value())); + ICEBERG_RETURN_UNEXPECTED(AppendInt(array, first_row_id.value())); } else { // Append null for optional field - ICEBERG_NANOARROW_RETURN_UNEXPECTED(ArrowArrayAppendNull(array, 1)); + ICEBERG_RETURN_UNEXPECTED(AppendNull(array)); } break; } From f4f6359fa9a95bde4bb2627bd8a5d45273644294 Mon Sep 17 00:00:00 2001 From: Anupam Yadav Date: Mon, 6 Jul 2026 23:13:50 -0700 Subject: [PATCH 07/23] build: scope warnings-as-error to project targets, not FetchContent subprojects (#819) ## What Fixes #246. `-Werror` (via a global `CMAKE_COMPILE_WARNING_AS_ERROR`) leaked into FetchContent-built dependencies, so a warning in a third-party subproject (e.g. Arrow) could fail the whole build. As noted on the issue, the global flag does not reliably isolate subprojects. ## How - Removed the global `CMAKE_COMPILE_WARNING_AS_ERROR ON`. - Set `COMPILE_WARNING_AS_ERROR ON` as a per-target property on Iceberg's own shared/static libraries, so the project stays strict. - Set `BUILD_WARNING_LEVEL PRODUCTION` for Arrow to suppress its independent `-Werror` injection. - Kept a defensive guard in `prepare_fetchcontent()` for the case where `-DCMAKE_COMPILE_WARNING_AS_ERROR=ON` is passed on the command line. ## Verified Iceberg targets still compile with `-Werror`; FetchContent subprojects do not. Build + tests green (18/18). --- This contribution was authored with assistance from Claude Opus 4.8, in line with the Iceberg guidelines for AI-assisted contributions (https://iceberg.apache.org/contribute/#guidelines-for-ai-assisted-contributions). All changes were reviewed and tested by the author. --- CMakeLists.txt | 2 -- cmake_modules/IcebergBuildUtils.cmake | 2 ++ cmake_modules/IcebergThirdpartyToolchain.cmake | 4 ++++ src/iceberg/test/CMakeLists.txt | 2 -- 4 files changed, 6 insertions(+), 4 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index eb13912388..5f88235679 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -37,8 +37,6 @@ set(CMAKE_CXX_STANDARD 23) set(CMAKE_CXX_STANDARD_REQUIRED ON) set(CMAKE_CXX_EXTENSIONS OFF) set(CMAKE_EXPORT_COMPILE_COMMANDS ON) -set(CMAKE_COMPILE_WARNING_AS_ERROR ON) - option(ICEBERG_BUILD_STATIC "Build static library" ON) option(ICEBERG_BUILD_SHARED "Build shared library" OFF) option(ICEBERG_BUILD_TESTS "Build tests" ON) diff --git a/cmake_modules/IcebergBuildUtils.cmake b/cmake_modules/IcebergBuildUtils.cmake index 74b0297088..da23f37a26 100644 --- a/cmake_modules/IcebergBuildUtils.cmake +++ b/cmake_modules/IcebergBuildUtils.cmake @@ -113,6 +113,7 @@ function(add_iceberg_lib LIB_NAME) if(BUILD_SHARED) add_library(${LIB_NAME}_shared SHARED) + set_target_properties(${LIB_NAME}_shared PROPERTIES COMPILE_WARNING_AS_ERROR ON) if(LIB_DEPS) target_sources(${LIB_NAME}_shared PRIVATE ${LIB_DEPS}) @@ -172,6 +173,7 @@ function(add_iceberg_lib LIB_NAME) if(BUILD_STATIC) add_library(${LIB_NAME}_static STATIC) + set_target_properties(${LIB_NAME}_static PROPERTIES COMPILE_WARNING_AS_ERROR ON) if(LIB_DEPS) target_sources(${LIB_NAME}_static PRIVATE ${LIB_DEPS}) diff --git a/cmake_modules/IcebergThirdpartyToolchain.cmake b/cmake_modules/IcebergThirdpartyToolchain.cmake index 825a5a0548..34ad6d9105 100644 --- a/cmake_modules/IcebergThirdpartyToolchain.cmake +++ b/cmake_modules/IcebergThirdpartyToolchain.cmake @@ -151,6 +151,10 @@ endmacro() function(resolve_arrow_dependency) prepare_fetchcontent() + # Prevent Arrow from injecting -Werror into CMAKE_CXX_FLAGS_DEBUG via + # arrow_add_werror_if_debug(). PRODUCTION level only adds standard warnings. + set(BUILD_WARNING_LEVEL PRODUCTION) + set(ARROW_BUILD_SHARED OFF) set(ARROW_BUILD_STATIC ON) # Work around undefined symbol: arrow::ipc::ReadSchema(arrow::io::InputStream*, arrow::ipc::DictionaryMemo*) diff --git a/src/iceberg/test/CMakeLists.txt b/src/iceberg/test/CMakeLists.txt index b3592d8e2c..7a1b45b70e 100644 --- a/src/iceberg/test/CMakeLists.txt +++ b/src/iceberg/test/CMakeLists.txt @@ -15,8 +15,6 @@ # specific language governing permissions and limitations # under the License. -set(CMAKE_COMPILE_WARNING_AS_ERROR FALSE) - set(INSTALL_GTEST OFF CACHE BOOL "" FORCE) From ebc0e07c98b1771cbe46879507163c0d3cd40256 Mon Sep 17 00:00:00 2001 From: Yuya Ebihara Date: Sat, 11 Jul 2026 11:24:26 +0900 Subject: [PATCH 08/23] Infra: Group github/codeql-action bumps into a single dependabot PR (#826) Thus, dependabot can update these two jobs in a single PR: https://github.com/apache/iceberg-cpp/blob/f4f6359fa9a95bde4bb2627bd8a5d45273644294/.github/workflows/codeql.yml#L56-L62 - https://github.com/apache/infrastructure-actions/pull/1028 --- .github/dependabot.yml | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/.github/dependabot.yml b/.github/dependabot.yml index fc74cd2992..325ea24247 100644 --- a/.github/dependabot.yml +++ b/.github/dependabot.yml @@ -26,3 +26,10 @@ updates: day: "sunday" cooldown: default-days: 7 + groups: + # The codeql-action init/autobuild/analyze steps must all run the + # same version - split PRs cause a version mismatch that fails the + # Analyze jobs. Group them so a single PR bumps all of them together. + codeql-action: + patterns: + - "github/codeql-action*" From 4c906d1d631fbc9907b3b0d0c7dbf0bebb90e41b Mon Sep 17 00:00:00 2001 From: Rahul Goel Date: Fri, 10 Jul 2026 22:30:38 -0400 Subject: [PATCH 09/23] feat: make ToLower Unicode-aware via utf8proc (#760) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Replaces the ASCII-only `StringUtils::ToLower` with a Unicode-aware implementation backed by [utf8proc](https://github.com/JuliaStrings/utf8proc), aligning case-insensitive name handling with Iceberg Java's `toLowerCase(Locale.ROOT)` for simple (1:1) case mappings. The few code points where simple and full case mapping differ (e.g. `İ` U+0130) are tracked in #808. - `ToLower` lower-cases UTF-8 using utf8proc simple (1:1) case mapping (e.g. `CAFÉ` → `café`, `GROẞE` → `große`). Pure-ASCII input takes a byte-wise fast path and never touches utf8proc. - Invalid UTF-8 is handled as a correctness guarantee (raised in review): `ToLower` is total — a byte that does not begin a valid UTF-8 sequence is passed through unchanged and decoding resumes at the next byte, so the valid code points around it are still lower-cased. The comparisons below therefore compare invalid bytes verbatim. - `EqualsIgnoreCase` and `StartsWithIgnoreCase` fold through `ToLower`, so they are case-insensitive for non-ASCII letters too. Both take an allocation-free ASCII fast path and fall back to `ToLower` only when a non-ASCII byte is present. - `StartsWithIgnoreCase` no longer assumes case mapping preserves byte length — the old byte-slice guard mishandled length-changing maps (e.g. `İ` U+0130 → `i`). Now `"İx"` starts with `"i"`, and `"i"` starts with `"İ"`. - `ToUpper` is intentionally left ASCII-only — it only normalizes ASCII enum/codec strings, and simple case mapping would be wrong for some letters (e.g. `ß` would stay unchanged instead of becoming `SS`). - utf8proc is pinned to 2.10.0 and wired into both the CMake (vendored via FetchContent / system package) and Meson (`subprojects/utf8proc.wrap`) builds, with matching source hashes and correct static-vs-shared linkage on Windows. ## Testing - `string_util_test.cc`: `ToLowerUnicode` (incl. invalid / truncated / stray-continuation bytes), `ToUpperAsciiOnly`, and Unicode + invalid-UTF-8 cases for `EqualsIgnoreCase` / `StartsWithIgnoreCase`. - `IgnoreCaseAgreesWithToLowerOracle` exhaustively checks both ASCII fast paths against the `ToLower` oracle over all short strings drawn from an alphabet with length-changing maps (`İ`→`i`, `K`→`k`) and invalid UTF-8. ## Remaining work (tracked in #808) - Full case-mapping parity with Java for the code points where simple ≠ full mapping (e.g. `İ` → `i̇`). - A streaming (allocation-free) non-ASCII path for `EqualsIgnoreCase` / `StartsWithIgnoreCase`. Part of #613 — does not fully fix it; #808 holds the design and remaining plan. Follow-up to #748. --- LICENSE | 92 ++++++++++ README.md | 1 + .../IcebergThirdpartyToolchain.cmake | 71 ++++++++ mkdocs/docs/getting-started.md | 1 + src/iceberg/CMakeLists.txt | 8 +- src/iceberg/meson.build | 20 ++- src/iceberg/test/string_util_test.cc | 161 ++++++++++++++++-- src/iceberg/util/string_util.cc | 44 +++++ src/iceberg/util/string_util.h | 93 +++++++--- subprojects/utf8proc.wrap | 30 ++++ 10 files changed, 484 insertions(+), 37 deletions(-) create mode 100644 subprojects/utf8proc.wrap diff --git a/LICENSE b/LICENSE index 374b7fc58f..8d8d5ff4e6 100644 --- a/LICENSE +++ b/LICENSE @@ -228,3 +228,95 @@ Home page: https://arrow.apache.org/ License: https://www.apache.org/licenses/LICENSE-2.0 -------------------------------------------------------------------------------- + +This product bundles utf8proc, which is available under the MIT License: + +utf8proc is a software package originally developed by Jan Behrens and the rest +of the Public Software Group, now maintained by the Julia-language developers. +All new work on the utf8proc library is licensed under the MIT "expat" license: + +Copyright (c) 2014-2021 by Steven G. Johnson, Jiahao Chen, Tony Kelman, Jonas +Fonseca, and other contributors listed in the git history. + +Permission is hereby granted, free of charge, to any person obtaining a +copy of this software and associated documentation files (the "Software"), +to deal in the Software without restriction, including without limitation +the rights to use, copy, modify, merge, publish, distribute, sublicense, +and/or sell copies of the Software, and to permit persons to whom the +Software is furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in +all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +DEALINGS IN THE SOFTWARE. + +The original utf8proc is licensed under the same MIT "expat" license: + +Copyright (c) 2009, 2013 Public Software Group e. V., Berlin, Germany + +Permission is hereby granted, free of charge, to any person obtaining a +copy of this software and associated documentation files (the "Software"), +to deal in the Software without restriction, including without limitation +the rights to use, copy, modify, merge, publish, distribute, sublicense, +and/or sell copies of the Software, and to permit persons to whom the +Software is furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in +all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +DEALINGS IN THE SOFTWARE. + +utf8proc also contains data derived from the Unicode data files. The following +license applies to that data: + +COPYRIGHT AND PERMISSION NOTICE + +Copyright (c) 1991-2007 Unicode, Inc. All rights reserved. Distributed under +the Terms of Use in http://www.unicode.org/copyright.html. + +Permission is hereby granted, free of charge, to any person obtaining a +copy of the Unicode data files and any associated documentation (the "Data +Files") or Unicode software and any associated documentation (the +"Software") to deal in the Data Files or Software without restriction, +including without limitation the rights to use, copy, modify, merge, +publish, distribute, and/or sell copies of the Data Files or Software, and +to permit persons to whom the Data Files or Software are furnished to do +so, provided that (a) the above copyright notice(s) and this permission +notice appear with all copies of the Data Files or Software, (b) both the +above copyright notice(s) and this permission notice appear in associated +documentation, and (c) there is clear notice in each modified Data File or +in the Software as well as in the documentation associated with the Data +File(s) or Software that the data or software has been modified. + +THE DATA FILES AND SOFTWARE ARE PROVIDED "AS IS", WITHOUT WARRANTY OF ANY +KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF +MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT OF +THIRD PARTY RIGHTS. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR HOLDERS +INCLUDED IN THIS NOTICE BE LIABLE FOR ANY CLAIM, OR ANY SPECIAL INDIRECT OR +CONSEQUENTIAL DAMAGES, OR ANY DAMAGES WHATSOEVER RESULTING FROM LOSS OF +USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR OTHER +TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR +PERFORMANCE OF THE DATA FILES OR SOFTWARE. + +Except as contained in this notice, the name of a copyright holder shall +not be used in advertising or otherwise to promote the sale, use or other +dealings in these Data Files or Software without prior written +authorization of the copyright holder. + +Unicode and the Unicode logo are trademarks of Unicode, Inc., and may be +registered in some jurisdictions. All other trademarks and registered +trademarks mentioned herein are the property of their respective owners. + +-------------------------------------------------------------------------------- diff --git a/README.md b/README.md index 7c9a343ec0..1318933d9a 100644 --- a/README.md +++ b/README.md @@ -151,6 +151,7 @@ If you experience network issues when downloading dependencies, you can customiz - `ICEBERG_AVRO_GIT_URL`: Apache Avro git repository URL - `ICEBERG_NANOARROW_URL`: Nanoarrow tarball URL - `ICEBERG_CROARING_URL`: CRoaring tarball URL +- `ICEBERG_UTF8PROC_URL`: utf8proc tarball URL - `ICEBERG_NLOHMANN_JSON_URL`: nlohmann-json tarball URL - `ICEBERG_SPDLOG_URL`: spdlog tarball URL - `ICEBERG_CPR_URL`: cpr tarball URL diff --git a/cmake_modules/IcebergThirdpartyToolchain.cmake b/cmake_modules/IcebergThirdpartyToolchain.cmake index 34ad6d9105..86e8400975 100644 --- a/cmake_modules/IcebergThirdpartyToolchain.cmake +++ b/cmake_modules/IcebergThirdpartyToolchain.cmake @@ -83,6 +83,7 @@ endfunction() # ICEBERG_AVRO_GIT_URL - Apache Avro git repository URL # ICEBERG_NANOARROW_URL - Nanoarrow tarball URL # ICEBERG_CROARING_URL - CRoaring tarball URL +# ICEBERG_UTF8PROC_URL - utf8proc tarball URL # ICEBERG_NLOHMANN_JSON_URL - nlohmann-json tarball URL # ICEBERG_SPDLOG_URL - spdlog tarball URL # ICEBERG_CPR_URL - cpr tarball URL @@ -120,6 +121,20 @@ else() ) endif() +set(ICEBERG_UTF8PROC_BUILD_VERSION "2.10.0") +set(ICEBERG_UTF8PROC_BUILD_SHA256_CHECKSUM + "276a37dc4d1dd24d7896826a579f4439d1e5fe33603add786bb083cab802e23e") + +if(DEFINED ENV{ICEBERG_UTF8PROC_URL}) + set(UTF8PROC_SOURCE_URL "$ENV{ICEBERG_UTF8PROC_URL}") +else() + # Use the release asset (stable bytes, matching subprojects/utf8proc.wrap) rather + # than the auto-generated tag archive, whose contents GitHub does not guarantee. + set(UTF8PROC_SOURCE_URL + "https://github.com/JuliaStrings/utf8proc/releases/download/v${ICEBERG_UTF8PROC_BUILD_VERSION}/utf8proc-${ICEBERG_UTF8PROC_BUILD_VERSION}.tar.gz" + ) +endif() + # ---------------------------------------------------------------------- # FetchContent @@ -436,6 +451,61 @@ function(resolve_croaring_dependency) PARENT_SCOPE) endfunction() +# ---------------------------------------------------------------------- +# utf8proc + +function(resolve_utf8proc_dependency) + prepare_fetchcontent() + + # The vendored build needs no install rules; without this, CMake < 3.28 (where + # FetchContent has no EXCLUDE_FROM_ALL) would install utf8proc's headers and + # pkg-config file into the iceberg install prefix. + set(UTF8PROC_INSTALL OFF) + + fetchcontent_declare(utf8proc + ${FC_DECLARE_COMMON_OPTIONS} + URL ${UTF8PROC_SOURCE_URL} + URL_HASH "SHA256=${ICEBERG_UTF8PROC_BUILD_SHA256_CHECKSUM}" + FIND_PACKAGE_ARGS + NAMES + utf8proc + CONFIG) + fetchcontent_makeavailable(utf8proc) + + if(utf8proc_SOURCE_DIR) + if(NOT TARGET utf8proc::utf8proc) + add_library(utf8proc::utf8proc INTERFACE IMPORTED) + target_link_libraries(utf8proc::utf8proc INTERFACE utf8proc) + target_include_directories(utf8proc::utf8proc INTERFACE ${utf8proc_SOURCE_DIR}) + endif() + + set(UTF8PROC_VENDORED TRUE) + # utf8proc's CMake puts a raw build-tree path in INTERFACE_INCLUDE_DIRECTORIES, which + # install(EXPORT) rejects. Wrap it in BUILD_INTERFACE so the export is valid; utf8proc + # is a private dependency, so installed consumers never need its headers. + set_target_properties(utf8proc + PROPERTIES OUTPUT_NAME "iceberg_vendored_utf8proc" + POSITION_INDEPENDENT_CODE ON + INTERFACE_INCLUDE_DIRECTORIES + "$") + install(TARGETS utf8proc + EXPORT iceberg_targets + RUNTIME DESTINATION "${ICEBERG_INSTALL_BINDIR}" + ARCHIVE DESTINATION "${ICEBERG_INSTALL_LIBDIR}" + LIBRARY DESTINATION "${ICEBERG_INSTALL_LIBDIR}") + else() + set(UTF8PROC_VENDORED FALSE) + list(APPEND ICEBERG_SYSTEM_DEPENDENCIES utf8proc) + endif() + + set(ICEBERG_SYSTEM_DEPENDENCIES + ${ICEBERG_SYSTEM_DEPENDENCIES} + PARENT_SCOPE) + set(UTF8PROC_VENDORED + ${UTF8PROC_VENDORED} + PARENT_SCOPE) +endfunction() + # ---------------------------------------------------------------------- # nlohmann-json @@ -734,6 +804,7 @@ endfunction() resolve_zlib_dependency() resolve_nanoarrow_dependency() resolve_croaring_dependency() +resolve_utf8proc_dependency() resolve_nlohmann_json_dependency() resolve_spdlog_dependency() diff --git a/mkdocs/docs/getting-started.md b/mkdocs/docs/getting-started.md index 10a1a5c907..52a04e635c 100644 --- a/mkdocs/docs/getting-started.md +++ b/mkdocs/docs/getting-started.md @@ -143,6 +143,7 @@ If you experience network issues when downloading dependencies, you can override | `ICEBERG_AVRO_GIT_URL` | Apache Avro git repository | | `ICEBERG_NANOARROW_URL` | Nanoarrow tarball | | `ICEBERG_CROARING_URL` | CRoaring tarball | +| `ICEBERG_UTF8PROC_URL` | utf8proc tarball | | `ICEBERG_NLOHMANN_JSON_URL` | nlohmann-json tarball | | `ICEBERG_CPR_URL` | cpr tarball | diff --git a/src/iceberg/CMakeLists.txt b/src/iceberg/CMakeLists.txt index 999532c1c5..da820c26be 100644 --- a/src/iceberg/CMakeLists.txt +++ b/src/iceberg/CMakeLists.txt @@ -154,23 +154,27 @@ list(APPEND "$,nanoarrow::nanoarrow_static,$,nanoarrow::nanoarrow_static,nanoarrow::nanoarrow_shared>>" nlohmann_json::nlohmann_json spdlog::spdlog + utf8proc::utf8proc ZLIB::ZLIB) list(APPEND ICEBERG_SHARED_BUILD_INTERFACE_LIBS "$,nanoarrow::nanoarrow_static,$,nanoarrow::nanoarrow_shared,nanoarrow::nanoarrow_static>>" nlohmann_json::nlohmann_json spdlog::spdlog + utf8proc::utf8proc ZLIB::ZLIB) list(APPEND ICEBERG_STATIC_INSTALL_INTERFACE_LIBS "$,iceberg::nanoarrow_static,$,nanoarrow::nanoarrow_static,nanoarrow::nanoarrow_shared>>" "$,iceberg::nlohmann_json,$,nlohmann_json::nlohmann_json,nlohmann_json::nlohmann_json>>" - "$,iceberg::spdlog,spdlog::spdlog>") + "$,iceberg::spdlog,spdlog::spdlog>" + "$,iceberg::utf8proc,utf8proc::utf8proc>") list(APPEND ICEBERG_SHARED_INSTALL_INTERFACE_LIBS "$,iceberg::nanoarrow_static,$,nanoarrow::nanoarrow_shared,nanoarrow::nanoarrow_static>>" "$,iceberg::nlohmann_json,$,nlohmann_json::nlohmann_json,nlohmann_json::nlohmann_json>>" - "$,iceberg::spdlog,spdlog::spdlog>") + "$,iceberg::spdlog,spdlog::spdlog>" + "$,iceberg::utf8proc,utf8proc::utf8proc>") add_iceberg_lib(iceberg SOURCES diff --git a/src/iceberg/meson.build b/src/iceberg/meson.build index 4c7960ae63..e5158a4b93 100644 --- a/src/iceberg/meson.build +++ b/src/iceberg/meson.build @@ -198,9 +198,27 @@ croaring_dep = dependency('croaring', static: croaring_needs_static) nanoarrow_dep = dependency('nanoarrow') nlohmann_json_dep = dependency('nlohmann_json') spdlog_dep = dependency('spdlog') +# utf8proc's header declares its functions __declspec(dllimport) on Windows unless +# UTF8PROC_STATIC is defined, and the wrap does not propagate that define to consumers. +# Define it whenever utf8proc is linked statically, so the header's declarations match +# how it is linked. Harmless on other platforms. +utf8proc_needs_static = get_option('default_library') == 'static' +utf8proc_dep = dependency('libutf8proc', static: utf8proc_needs_static) +if utf8proc_needs_static + utf8proc_dep = declare_dependency( + compile_args: ['-DUTF8PROC_STATIC'], + dependencies: utf8proc_dep, + ) +endif zlib_dep = dependency('zlib') -iceberg_deps = [nanoarrow_dep, nlohmann_json_dep, spdlog_dep, zlib_dep] +iceberg_deps = [ + nanoarrow_dep, + nlohmann_json_dep, + spdlog_dep, + utf8proc_dep, + zlib_dep, +] iceberg_lib = library( 'iceberg', diff --git a/src/iceberg/test/string_util_test.cc b/src/iceberg/test/string_util_test.cc index a3fd03760e..65d5c23cc5 100644 --- a/src/iceberg/test/string_util_test.cc +++ b/src/iceberg/test/string_util_test.cc @@ -41,19 +41,53 @@ TEST(StringUtilsTest, ToUpper) { ASSERT_EQ(StringUtils::ToUpper("123"), "123"); } -// Non-ASCII (multibyte UTF-8) bytes have the high bit set, i.e. are negative when stored -// in a signed char. Only ASCII letters are converted; multibyte bytes pass through -// unchanged. The non-ASCII strings are written as explicit UTF-8 byte escapes so the test -// does not depend on the source-file encoding. See -// https://github.com/apache/iceberg-cpp/issues/613. -TEST(StringUtilsTest, NonAsciiPassThrough) { - // "Naïve" -> "naïve" (ï = U+00EF = 0xC3 0xAF; only the ASCII letters change). - ASSERT_EQ(StringUtils::ToLower("Na\xC3\xAFve"), "na\xC3\xAFve"); - // "café" -> "CAFé" (é = U+00E9 = 0xC3 0xA9 stays unchanged). - ASSERT_EQ(StringUtils::ToUpper("caf\xC3\xA9"), "CAF\xC3\xA9"); - // "日本語" (0xE6 0x97 0xA5 0xE6 0x9C 0xAC 0xE8 0xAA 0x9E) is returned verbatim. +// Non-ASCII strings are written as explicit UTF-8 byte escapes so the test does not +// depend on the source-file encoding. An escape is split before a following hex digit +// (e.g. "...\x9E" "E") so the \x does not absorb it. +// See https://github.com/apache/iceberg-cpp/issues/613. +TEST(StringUtilsTest, ToLowerUnicode) { + // "CAFÉ" -> "café" (É U+00C9 = 0xC3 0x89 -> é U+00E9 = 0xC3 0xA9). + ASSERT_EQ(StringUtils::ToLower("CAF\xC3\x89"), "caf\xC3\xA9"); + // "GROẞE" -> "große": capital sharp S (ẞ U+1E9E) lower-cases to ß (U+00DF), not "ss" + // as casefolding would produce. + ASSERT_EQ(StringUtils::ToLower("GRO\xE1\xBA\x9E" + "E"), + "gro\xC3\x9F" + "e"); + // "日本語" has no case mapping and is returned verbatim. ASSERT_EQ(StringUtils::ToLower("\xE6\x97\xA5\xE6\x9C\xAC\xE8\xAA\x9E"), "\xE6\x97\xA5\xE6\x9C\xAC\xE8\xAA\x9E"); + // ASCII prefix before the first non-ASCII byte takes the fast path; the rest goes + // through utf8proc. "ABÉ" -> "abé". + ASSERT_EQ(StringUtils::ToLower("AB\xC3\x89"), "ab\xC3\xA9"); + // An invalid UTF-8 byte (a lone 0xFF) passes through unchanged rather than erroring. + ASSERT_EQ(StringUtils::ToLower("\xFF"), "\xFF"); + // An invalid byte only passes through itself; the valid code points around it are + // still lower-cased ("AB" 0xFF "CÉ" -> "ab" 0xFF "cé"). + ASSERT_EQ(StringUtils::ToLower("AB\xFF" + "C\xC3\x89"), + "ab\xFF" + "c\xC3\xA9"); + // The invalid byte can abut a multi-byte code point with no ASCII between them; 0xFF + // passes through and the adjacent "É" still lower-cases to "é" (0xFF "É" -> 0xFF "é"). + ASSERT_EQ(StringUtils::ToLower("\xFF\xC3\x89"), "\xFF\xC3\xA9"); + // A truncated multi-byte sequence (0xC3 with no continuation byte) passes through + // without consuming the bytes after it. + ASSERT_EQ(StringUtils::ToLower("\xC3" + "AB"), + "\xC3" + "ab"); + // A stray continuation byte (0x80) behaves the same way. + ASSERT_EQ(StringUtils::ToLower("A\x80" + "B"), + "a\x80" + "b"); +} + +// ToUpper is intentionally ASCII-only; non-ASCII (multibyte UTF-8) bytes pass through. +TEST(StringUtilsTest, ToUpperAsciiOnly) { + // "café" -> "CAFé" (é stays unchanged). + ASSERT_EQ(StringUtils::ToUpper("caf\xC3\xA9"), "CAF\xC3\xA9"); ASSERT_EQ(StringUtils::ToUpper("\xE6\x97\xA5\xE6\x9C\xAC\xE8\xAA\x9E"), "\xE6\x97\xA5\xE6\x9C\xAC\xE8\xAA\x9E"); } @@ -63,10 +97,109 @@ TEST(StringUtilsTest, EqualsIgnoreCase) { ASSERT_TRUE(StringUtils::EqualsIgnoreCase("", "")); ASSERT_FALSE(StringUtils::EqualsIgnoreCase("abc", "abcd")); ASSERT_FALSE(StringUtils::EqualsIgnoreCase("abc", "abd")); - // ASCII case is folded; non-ASCII bytes are compared as-is. ("Café" vs "café") - ASSERT_TRUE(StringUtils::EqualsIgnoreCase("Caf\xC3\xA9", "caf\xC3\xA9")); - // "café" vs "cafe": the multibyte é differs from ASCII 'e'. + // Unicode-aware: "CAFÉ" matches "café". + ASSERT_TRUE(StringUtils::EqualsIgnoreCase("CAF\xC3\x89", "caf\xC3\xA9")); + // "GROẞE" matches "große" under lowercasing (ẞ -> ß). + ASSERT_TRUE( + StringUtils::EqualsIgnoreCase("GRO\xE1\xBA\x9E" + "E", + "gro\xC3\x9F" + "e")); + // Different letters still differ ("café" vs "cafe"). ASSERT_FALSE(StringUtils::EqualsIgnoreCase("caf\xC3\xA9", "cafe")); + // Fallback correctness: an ASCII operand can equal a non-ASCII one once lower-cased, + // even though their raw byte lengths differ. "İ" (U+0130 = 0xC4 0xB0, two bytes) + // lower-cases to one-byte "i", so it must compare equal to "i" and "I". + ASSERT_TRUE(StringUtils::EqualsIgnoreCase("i", "\xC4\xB0")); + ASSERT_TRUE(StringUtils::EqualsIgnoreCase("\xC4\xB0", "I")); + // The non-ASCII byte can appear after a matching ASCII prefix ("abi" vs "abİ"). + ASSERT_TRUE(StringUtils::EqualsIgnoreCase("abi", "ab\xC4\xB0")); + // Pure-ASCII operands that share a prefix but differ in length are not equal. + ASSERT_FALSE(StringUtils::EqualsIgnoreCase("abc", "ab")); + // Operands containing invalid UTF-8 are still compared case-insensitively on their + // valid parts; the invalid bytes themselves compare verbatim. + ASSERT_TRUE( + StringUtils::EqualsIgnoreCase("AB\xFF" + "C", + "ab\xFF" + "c")); + ASSERT_FALSE(StringUtils::EqualsIgnoreCase("\xFF", "\xFE")); +} + +TEST(StringUtilsTest, StartsWithIgnoreCase) { + ASSERT_TRUE(StringUtils::StartsWithIgnoreCase("AbCdef", "abc")); + ASSERT_TRUE(StringUtils::StartsWithIgnoreCase("abc", "ABC")); + ASSERT_FALSE(StringUtils::StartsWithIgnoreCase("abc", "abd")); + // Empty prefix always matches; a prefix longer than the string does not. + ASSERT_TRUE(StringUtils::StartsWithIgnoreCase("abc", "")); + ASSERT_FALSE(StringUtils::StartsWithIgnoreCase("ab", "abcd")); + // Regression (#760): lower-casing can change byte length, so the prefix must not be + // matched by byte-slicing. "İ" (U+0130 = 0xC4 0xB0) lower-cases to "i", so "İx" + // starts with "i" ... + ASSERT_TRUE(StringUtils::StartsWithIgnoreCase("\xC4\xB0x", "i")); + // ... and "i" starts with "İ" (both lower-case to "i"), which the old byte-length + // guard wrongly rejected. + ASSERT_TRUE(StringUtils::StartsWithIgnoreCase("i", "\xC4\xB0")); + // A matching Unicode prefix: "CAFÉbar" starts with "café". + ASSERT_TRUE( + StringUtils::StartsWithIgnoreCase("CAF\xC3\x89" + "bar", + "caf\xC3\xA9")); + // Invalid UTF-8 bytes compare verbatim in the prefix as well. + ASSERT_TRUE( + StringUtils::StartsWithIgnoreCase("AB\xFF" + "x", + "ab\xFF")); + ASSERT_FALSE(StringUtils::StartsWithIgnoreCase("ab\xFE", "ab\xFF")); +} + +// The ASCII fast paths in EqualsIgnoreCase / StartsWithIgnoreCase must agree with their +// documented ToLower-based semantics for every input, including length-changing case +// mappings and invalid UTF-8. Rather than enumerate cases by hand, exhaustively compare +// both functions against the ToLower oracle over all short strings built from a small +// alphabet that straddles those boundaries. This is the mechanical form of the #760 +// regression, where a fast path disagreed with ToLower on a length-changing mapping. +TEST(StringUtilsTest, IgnoreCaseAgreesWithToLowerOracle) { + // Atoms mix ASCII (upper/lower, including the lowercase targets of the multi-byte + // mappings) with a 2-byte code point that lower-cases to one byte ("İ" U+0130 -> "i"), + // a 3-byte one that also shrinks to one byte ("K" U+212A -> "k"), an ordinary 2-byte + // cased letter ("É"), and an invalid UTF-8 byte. + const std::vector atoms = { + "a", "I", "i", "k", "\xC4\xB0", "\xE2\x84\xAA", "\xC3\x89", "\xFF"}; + + // Build every string of 0..3 atoms, one generation (length) at a time. + std::vector inputs = {""}; + size_t generation_begin = 0; + for (int len = 0; len < 3; ++len) { + const size_t generation_end = inputs.size(); + for (size_t i = generation_begin; i < generation_end; ++i) { + for (const auto& atom : atoms) { + inputs.push_back(inputs[i] + atom); + } + } + generation_begin = generation_end; + } + + // Precompute the oracle so the O(n^2) comparison below does not re-lower each string. + std::vector lowered; + lowered.reserve(inputs.size()); + for (const auto& s : inputs) { + lowered.push_back(StringUtils::ToLower(s)); + } + + for (size_t i = 0; i < inputs.size(); ++i) { + for (size_t j = 0; j < inputs.size(); ++j) { + EXPECT_EQ(StringUtils::EqualsIgnoreCase(inputs[i], inputs[j]), + lowered[i] == lowered[j]) + << "EqualsIgnoreCase disagreed for a=" << testing::PrintToString(inputs[i]) + << " b=" << testing::PrintToString(inputs[j]); + EXPECT_EQ(StringUtils::StartsWithIgnoreCase(inputs[i], inputs[j]), + lowered[i].starts_with(lowered[j])) + << "StartsWithIgnoreCase disagreed for str=" + << testing::PrintToString(inputs[i]) + << " prefix=" << testing::PrintToString(inputs[j]); + } + } } } // namespace iceberg diff --git a/src/iceberg/util/string_util.cc b/src/iceberg/util/string_util.cc index 0454a62b58..8b98397db1 100644 --- a/src/iceberg/util/string_util.cc +++ b/src/iceberg/util/string_util.cc @@ -19,10 +19,54 @@ #include "iceberg/util/string_util.h" +#include + +#include + #include "iceberg/util/macros.h" namespace iceberg { +std::string StringUtils::ToLower(std::string_view str) { + std::string result; + result.reserve(str.size()); + + // Lower-case ASCII bytes directly; hand non-ASCII bytes to utf8proc. The common inputs + // (modes, UUIDs, header/property names, enum-like strings) are pure ASCII and never + // touch utf8proc. utf8proc has no string-level helper, so each non-ASCII code point is + // decoded, mapped with utf8proc_tolower (simple 1:1 mapping, not casefolding), and + // re-encoded. + const auto* data = reinterpret_cast(str.data()); + const auto size = static_cast(str.size()); + utf8proc_ssize_t offset = 0; + while (offset < size) { + // An ASCII byte is a complete 1-byte code point (never a UTF-8 continuation byte), + // and utf8proc_tolower agrees with ToLowerAscii on it, so handle it without utf8proc. + if (IsAsciiByte(str[offset])) { + result.push_back(ToLowerAscii(str[offset])); + ++offset; + continue; + } + utf8proc_int32_t code_point = 0; + utf8proc_ssize_t consumed = + utf8proc_iterate(data + offset, size - offset, &code_point); + if (consumed < 0) { + // Invalid UTF-8: pass the offending byte through unchanged and resume decoding at + // the next byte, so the valid code points around it are still lower-cased. + result.push_back(str[offset]); + ++offset; + continue; + } + const utf8proc_int32_t lowered = utf8proc_tolower(code_point); + std::array encoded{}; + const utf8proc_ssize_t written = utf8proc_encode_char(lowered, encoded.data()); + result.append(reinterpret_cast(encoded.data()), + static_cast(written)); + offset += consumed; + } + return result; +} + Result> StringUtils::HexStringToBytes(std::string_view hex) { if (hex.size() % 2 != 0) [[unlikely]] { return InvalidArgument("Hex string must have even length, got: {}", hex.size()); diff --git a/src/iceberg/util/string_util.h b/src/iceberg/util/string_util.h index 01b6087b8c..62718a0eaa 100644 --- a/src/iceberg/util/string_util.h +++ b/src/iceberg/util/string_util.h @@ -20,9 +20,9 @@ #pragma once #include -#include #include #include +#include #include #include #include @@ -41,29 +41,61 @@ concept FromChars = requires(const char* p, T& v) { std::from_chars(p, p, v); }; class ICEBERG_EXPORT StringUtils { public: - // NOTE: These convert ASCII letters only; all other bytes, including non-ASCII - // (multibyte UTF-8) bytes, are passed through unchanged. - // See https://github.com/apache/iceberg-cpp/issues/613. - static std::string ToLower(std::string_view str) { - return str | std::ranges::views::transform(ToLowerAscii) | - std::ranges::to(); - } - + /// \brief Lower-case a UTF-8 string using Unicode simple (1:1) case mapping. + /// + /// Intended for case-insensitive name matching, similar to Iceberg Java's + /// toLowerCase(Locale.ROOT). The mapping is locale-independent, matching the intent + /// of Locale.ROOT. It uses simple (1:1) case mapping rather than Java's full case + /// mapping, so results differ for a few code points; e.g. U+0130 (capital I with dot + /// above) maps to U+0069 ("i") here, but to U+0069 U+0307 ("i" + combining dot above) + /// in Java. For ASCII and the large majority of letters the two agree. + /// + /// Pure-ASCII input takes a byte-wise fast path; utf8proc is only invoked when a + /// non-ASCII byte (>= 0x80) is present. The function is total: it never fails, and + /// input need not be valid UTF-8. A byte that does not begin a valid UTF-8 sequence + /// is copied through unchanged and decoding resumes at the next byte, so the valid + /// code points around it are still lower-cased. + /// See https://github.com/apache/iceberg-cpp/issues/613. + static std::string ToLower(std::string_view str); + + /// \brief Upper-case the ASCII letters (a-z) in a string; all other bytes, including + /// multi-byte UTF-8 sequences, are left unchanged. + /// + /// Deliberately ASCII-only and, unlike ToLower, not Unicode-aware. It is only used to + /// normalize ASCII enum/codec strings (e.g. "gzip" -> "GZIP", "all" -> "ALL") for + /// case-insensitive comparison. A Unicode upper-case is intentionally not provided: + /// simple case mapping would be wrong for some letters (e.g. "ß" (U+00DF) would stay + /// unchanged instead of becoming "SS"), and no caller needs it. static std::string ToUpper(std::string_view str) { return str | std::ranges::views::transform(ToUpperAscii) | std::ranges::to(); } + /// \brief Case-insensitive equality using Unicode simple (1:1) case mapping. + /// + /// Equal when the ToLower forms of both operands are equal, so folding follows + /// ToLower's rules (e.g. "İ" (U+0130) folds to "i"). Defined for any byte sequence: + /// ToLower passes invalid UTF-8 bytes through unchanged, so they compare verbatim. static bool EqualsIgnoreCase(std::string_view lhs, std::string_view rhs) { - return std::ranges::equal( - lhs, rhs, [](char lc, char rc) { return ToLowerAscii(lc) == ToLowerAscii(rc); }); + const std::optional fast = AsciiEqualsIgnoreCase(lhs, rhs); + return fast.has_value() ? *fast : (ToLower(lhs) == ToLower(rhs)); } + /// \brief Case-insensitive prefix test using Unicode simple (1:1) case mapping. + /// + /// True when the ToLower form of str starts with the ToLower form of prefix, so folding + /// follows ToLower's rules (e.g. "İ" (U+0130) folds to "i"). Defined for any byte + /// sequence: ToLower passes invalid UTF-8 bytes through unchanged, so they compare + /// verbatim. static bool StartsWithIgnoreCase(std::string_view str, std::string_view prefix) { - if (str.size() < prefix.size()) { - return false; + if (prefix.size() <= str.size()) { + const std::optional fast = + AsciiEqualsIgnoreCase(str.substr(0, prefix.size()), prefix); + if (fast.has_value()) { + return *fast; + } } - return EqualsIgnoreCase(str.substr(0, prefix.size()), prefix); + return ToLower(str).starts_with(ToLower(prefix)); } /// \brief Count the number of code points in a UTF-8 string. @@ -134,16 +166,37 @@ class ICEBERG_EXPORT StringUtils { } private: - // ASCII-only case conversion using explicit range checks rather than - // std::tolower/std::toupper. This is independent of the current C locale and never - // touches non-ASCII (high-bit) bytes, so multibyte UTF-8 sequences are preserved. It - // also sidesteps the undefined behavior of passing a negative char to . + // ASCII-only case mappings. These avoid std::toupper/std::tolower, which are + // locale-dependent and have undefined behavior for negative char values. + static constexpr char ToUpperAscii(char c) noexcept { + return (c >= 'a' && c <= 'z') ? static_cast(c - 'a' + 'A') : c; + } static constexpr char ToLowerAscii(char c) noexcept { return (c >= 'A' && c <= 'Z') ? static_cast(c - 'A' + 'a') : c; } - static constexpr char ToUpperAscii(char c) noexcept { - return (c >= 'a' && c <= 'z') ? static_cast(c - 'a' + 'A') : c; + // True if c is a 7-bit ASCII byte (< 0x80). The cast is required because char may be + // signed, which would make bytes >= 0x80 compare as negative. + static constexpr bool IsAsciiByte(char c) noexcept { + return (static_cast(c) & 0x80) == 0; + } + + // Case-insensitive equality decided in a single byte-wise pass, without allocating. + // Returns nullopt once a byte of either operand is non-ASCII, because folding can then + // be non-ASCII and length-changing (e.g. "İ" (U+0130) -> "i"), which only ToLower + // knows. + static std::optional AsciiEqualsIgnoreCase(std::string_view a, + std::string_view b) { + const size_t n = std::min(a.size(), b.size()); + for (size_t i = 0; i < n; ++i) { + if (!IsAsciiByte(a[i]) || !IsAsciiByte(b[i])) { + return std::nullopt; + } + if (ToLowerAscii(a[i]) != ToLowerAscii(b[i])) { + return false; + } + } + return a.size() == b.size(); } }; diff --git a/subprojects/utf8proc.wrap b/subprojects/utf8proc.wrap new file mode 100644 index 0000000000..9b33b3beaf --- /dev/null +++ b/subprojects/utf8proc.wrap @@ -0,0 +1,30 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +[wrap-file] +directory = utf8proc-2.10.0 +source_url = https://github.com/JuliaStrings/utf8proc/releases/download/v2.10.0/utf8proc-2.10.0.tar.gz +source_filename = utf8proc-2.10.0.tar.gz +source_hash = 276a37dc4d1dd24d7896826a579f4439d1e5fe33603add786bb083cab802e23e +patch_filename = utf8proc_2.10.0-1_patch.zip +patch_url = https://wrapdb.mesonbuild.com/v2/utf8proc_2.10.0-1/get_patch +patch_hash = be16c4514603e922f9636045699fe1a6f844d340b9b7c14b809e47253b06a844 +source_fallback_url = https://github.com/mesonbuild/wrapdb/releases/download/utf8proc_2.10.0-1/utf8proc-2.10.0.tar.gz +wrapdb_version = 2.10.0-1 + +[provide] +libutf8proc = utf8proc_dep From b4737dfd9da028fd82f3a5f2ff6aa66ded0660f7 Mon Sep 17 00:00:00 2001 From: Junwang Zhao Date: Sat, 11 Jul 2026 10:33:13 +0800 Subject: [PATCH 10/23] fix: generate public API docs correctly (#804) --- Makefile | 1 + mkdocs/Doxyfile | 59 ++----------------- mkdocs/build-docs.sh | 1 + mkdocs/namespace_docs.dox | 54 +++++++++++++++++ src/iceberg/arrow/arrow_io_util.h | 3 + src/iceberg/arrow/s3/s3_properties.h | 3 + src/iceberg/avro/avro_constants.h | 3 + src/iceberg/avro/avro_metrics.h | 3 + src/iceberg/avro/avro_reader.h | 3 + src/iceberg/avro/avro_writer.h | 3 + src/iceberg/catalog.h | 3 + .../catalog/hive/iceberg_hive_export.h | 3 + .../catalog/memory/in_memory_catalog.h | 3 + src/iceberg/catalog/rest/http_request.h | 3 + .../catalog/rest/iceberg_rest_export.h | 3 + src/iceberg/catalog/rest/rest_file_io.h | 3 + src/iceberg/catalog/rest/rest_util.h | 3 + src/iceberg/catalog/session_catalog.h | 3 + src/iceberg/catalog/session_context.h | 3 + .../catalog/sql/iceberg_sql_catalog_export.h | 3 + .../deletes/position_delete_range_consumer.h | 3 + src/iceberg/expression/literal.h | 3 + src/iceberg/file_io.h | 3 + src/iceberg/file_io_registry.h | 3 + src/iceberg/iceberg_bundle_export.h | 3 + src/iceberg/iceberg_data_export.h | 3 + src/iceberg/iceberg_export.h | 3 + src/iceberg/inspect/history_table.h | 3 + src/iceberg/inspect/metadata_table.h | 3 + src/iceberg/inspect/snapshots_table.h | 3 + src/iceberg/location_provider.h | 3 + src/iceberg/metrics/commit_report.h | 3 + src/iceberg/metrics/counter.h | 3 + src/iceberg/metrics/metrics_reporter.h | 3 + src/iceberg/metrics/scan_report.h | 3 + src/iceberg/metrics/timer.h | 3 + src/iceberg/name_mapping.h | 3 + src/iceberg/parquet/parquet_reader.h | 3 + src/iceberg/parquet/parquet_writer.h | 3 + src/iceberg/result.h | 3 + src/iceberg/row/manifest_wrapper.h | 2 +- src/iceberg/schema_util.h | 3 + src/iceberg/snapshot.h | 3 + src/iceberg/sort_order.h | 3 + src/iceberg/storage_credential.h | 3 + src/iceberg/table.h | 3 + src/iceberg/table_properties.h | 3 + src/iceberg/table_scan.h | 3 + src/iceberg/transaction.h | 3 + src/iceberg/update/snapshot_update.h | 3 + src/iceberg/util/bucket_util.h | 3 + src/iceberg/util/config.h | 3 + src/iceberg/util/executor.h | 3 + src/iceberg/util/functional.h | 3 + src/iceberg/util/location_util.h | 3 + src/iceberg/util/macros.h | 3 + src/iceberg/util/partition_value_util.h | 2 +- src/iceberg/util/property_util.h | 3 + src/iceberg/util/retry_util.h | 3 + src/iceberg/util/string_util.h | 3 + src/iceberg/util/task_group.h | 3 + src/iceberg/util/temporal_util.h | 3 + src/iceberg/util/timepoint.h | 3 + src/iceberg/util/transform_util.h | 3 + src/iceberg/util/truncate_util.h | 3 + src/iceberg/util/visitor_generate.h | 3 + 66 files changed, 244 insertions(+), 55 deletions(-) create mode 100644 mkdocs/namespace_docs.dox diff --git a/Makefile b/Makefile index 1000695453..99ca5bbaa3 100644 --- a/Makefile +++ b/Makefile @@ -33,6 +33,7 @@ install-deps: build-api-docs: cd mkdocs && \ + rm -rf docs/api && \ mkdir -p docs/api && \ doxygen Doxyfile && \ echo "Doxygen output created in docs/api/" diff --git a/mkdocs/Doxyfile b/mkdocs/Doxyfile index 75e5ee232b..0237eb4795 100644 --- a/mkdocs/Doxyfile +++ b/mkdocs/Doxyfile @@ -966,7 +966,7 @@ WARN_LOGFILE = # spaces. See also FILE_PATTERNS and EXTENSION_MAPPING # Note: If this tag is empty the current directory is searched. -INPUT = ../src ../README.md +INPUT = ../src ../README.md namespace_docs.dox # This tag can be used to specify the character encoding of the source files # that doxygen parses. Internally doxygen uses the UTF-8 encoding. Doxygen uses @@ -1006,57 +1006,9 @@ INPUT_FILE_ENCODING = # be provided as doxygen C comment), *.py, *.pyw, *.f90, *.f95, *.f03, *.f08, # *.f18, *.f, *.for, *.vhd, *.vhdl, *.ucf, *.qsf and *.ice. -FILE_PATTERNS = *.c \ - *.cc \ - *.cxx \ - *.cxxm \ - *.cpp \ - *.cppm \ - *.ccm \ - *.c++ \ - *.c++m \ - *.java \ - *.ii \ - *.ixx \ - *.ipp \ - *.i++ \ - *.inl \ - *.idl \ - *.ddl \ - *.odl \ - *.h \ - *.hh \ - *.hxx \ - *.hpp \ - *.h++ \ - *.ixx \ - *.l \ - *.cs \ - *.d \ - *.php \ - *.php4 \ - *.php5 \ - *.phtml \ - *.inc \ - *.m \ - *.markdown \ +FILE_PATTERNS = *.h \ *.md \ - *.mm \ - *.dox \ - *.py \ - *.pyw \ - *.f90 \ - *.f95 \ - *.f03 \ - *.f08 \ - *.f18 \ - *.f \ - *.for \ - *.vhd \ - *.vhdl \ - *.ucf \ - *.qsf \ - *.ice + *.dox # The RECURSIVE tag can be used to specify whether or not subdirectories should # be searched for input files as well. @@ -1071,7 +1023,8 @@ RECURSIVE = YES # Note that relative paths are relative to the directory from which doxygen is # run. -EXCLUDE = +EXCLUDE = ../src/iceberg/test \ + ../src/iceberg/catalog/hive/gen-cpp # The EXCLUDE_SYMLINKS tag can be used to select whether or not files or # directories that are symbolic links (a Unix file system feature) are excluded @@ -1087,7 +1040,7 @@ EXCLUDE_SYMLINKS = NO # Note that the wildcards are matched against the file with absolute path, so to # exclude all test directories for example use the pattern */test/* -EXCLUDE_PATTERNS = +EXCLUDE_PATTERNS = *_internal.h # The EXCLUDE_SYMBOLS tag can be used to specify one or more symbol names # (namespaces, classes, functions, etc.) that should be excluded from the diff --git a/mkdocs/build-docs.sh b/mkdocs/build-docs.sh index 6917d339ea..70f4bf8f7b 100755 --- a/mkdocs/build-docs.sh +++ b/mkdocs/build-docs.sh @@ -41,6 +41,7 @@ if ! command -v doxygen &> /dev/null; then else echo "Building API documentation with Doxygen..." cd mkdocs + rm -rf docs/api mkdir -p docs/api doxygen Doxyfile cd .. diff --git a/mkdocs/namespace_docs.dox b/mkdocs/namespace_docs.dox new file mode 100644 index 0000000000..9faa714d68 --- /dev/null +++ b/mkdocs/namespace_docs.dox @@ -0,0 +1,54 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +/// \namespace iceberg +/// \brief Core Apache Iceberg C++ APIs. + +/// \namespace iceberg::arrow +/// \brief Arrow-backed IO and data conversion APIs. + +/// \namespace iceberg::avro +/// \brief Avro reader, writer, and metadata APIs. + +/// \namespace iceberg::hive +/// \brief Hive catalog APIs. + +/// \namespace iceberg::parquet +/// \brief Parquet reader, writer, and registration APIs. + +/// \namespace iceberg::puffin +/// \brief Puffin file metadata, reader, and writer APIs. + +/// \namespace iceberg::rest +/// \brief REST catalog APIs. + +/// \namespace iceberg::rest::auth +/// \brief REST catalog authentication APIs. + +/// \namespace iceberg::retry +/// \brief Retry policy APIs. + +/// \namespace iceberg::sql +/// \brief SQL catalog APIs. + +/// \namespace iceberg::table +/// \brief Table update requirement APIs. + +/// \namespace iceberg::util +/// \brief Utility APIs. diff --git a/src/iceberg/arrow/arrow_io_util.h b/src/iceberg/arrow/arrow_io_util.h index e33826dc69..925ee12a27 100644 --- a/src/iceberg/arrow/arrow_io_util.h +++ b/src/iceberg/arrow/arrow_io_util.h @@ -19,6 +19,9 @@ #pragma once +/// \file iceberg/arrow/arrow_io_util.h +/// \brief Provide helpers to create Arrow-backed FileIO implementations. + #include #include #include diff --git a/src/iceberg/arrow/s3/s3_properties.h b/src/iceberg/arrow/s3/s3_properties.h index 7b76968a66..03d1492ade 100644 --- a/src/iceberg/arrow/s3/s3_properties.h +++ b/src/iceberg/arrow/s3/s3_properties.h @@ -19,6 +19,9 @@ #pragma once +/// \file iceberg/arrow/s3/s3_properties.h +/// \brief Define S3 configuration property keys. + #include namespace iceberg::arrow { diff --git a/src/iceberg/avro/avro_constants.h b/src/iceberg/avro/avro_constants.h index 6fdfdc5edb..f3f175c1e2 100644 --- a/src/iceberg/avro/avro_constants.h +++ b/src/iceberg/avro/avro_constants.h @@ -19,6 +19,9 @@ #pragma once +/// \file iceberg/avro/avro_constants.h +/// \brief Define Avro metadata constants used by Iceberg. + #include namespace iceberg::avro { diff --git a/src/iceberg/avro/avro_metrics.h b/src/iceberg/avro/avro_metrics.h index 799c00ee39..3f055c23ba 100644 --- a/src/iceberg/avro/avro_metrics.h +++ b/src/iceberg/avro/avro_metrics.h @@ -19,6 +19,9 @@ #pragma once +/// \file iceberg/avro/avro_metrics.h +/// \brief Compute Iceberg metrics for Avro data files. + #include "iceberg/iceberg_bundle_export.h" #include "iceberg/metrics.h" diff --git a/src/iceberg/avro/avro_reader.h b/src/iceberg/avro/avro_reader.h index 24f95f5d43..c193d00c24 100644 --- a/src/iceberg/avro/avro_reader.h +++ b/src/iceberg/avro/avro_reader.h @@ -19,6 +19,9 @@ #pragma once +/// \file iceberg/avro/avro_reader.h +/// \brief Provide the Avro file reader interface. + #include "iceberg/file_reader.h" #include "iceberg/iceberg_bundle_export.h" diff --git a/src/iceberg/avro/avro_writer.h b/src/iceberg/avro/avro_writer.h index 3bd8110d93..af68fac440 100644 --- a/src/iceberg/avro/avro_writer.h +++ b/src/iceberg/avro/avro_writer.h @@ -19,6 +19,9 @@ #pragma once +/// \file iceberg/avro/avro_writer.h +/// \brief Provide the Avro file writer interface. + #include "iceberg/file_writer.h" #include "iceberg/iceberg_bundle_export.h" diff --git a/src/iceberg/catalog.h b/src/iceberg/catalog.h index 81c9ddd112..79c901fd0b 100644 --- a/src/iceberg/catalog.h +++ b/src/iceberg/catalog.h @@ -19,6 +19,9 @@ #pragma once +/// \file iceberg/catalog.h +/// \brief Define the catalog API for table and namespace operations. + #include #include #include diff --git a/src/iceberg/catalog/hive/iceberg_hive_export.h b/src/iceberg/catalog/hive/iceberg_hive_export.h index 2293045535..3863297534 100644 --- a/src/iceberg/catalog/hive/iceberg_hive_export.h +++ b/src/iceberg/catalog/hive/iceberg_hive_export.h @@ -19,6 +19,9 @@ #pragma once +/// \file iceberg/catalog/hive/iceberg_hive_export.h +/// \brief Define symbol visibility macros for the Hive catalog library. + #if defined(_WIN32) || defined(__CYGWIN__) # ifdef ICEBERG_HIVE_STATIC # define ICEBERG_HIVE_EXPORT diff --git a/src/iceberg/catalog/memory/in_memory_catalog.h b/src/iceberg/catalog/memory/in_memory_catalog.h index 22a596c107..548fd7afc5 100644 --- a/src/iceberg/catalog/memory/in_memory_catalog.h +++ b/src/iceberg/catalog/memory/in_memory_catalog.h @@ -19,6 +19,9 @@ #pragma once +/// \file iceberg/catalog/memory/in_memory_catalog.h +/// \brief Provide an in-memory catalog implementation. + #include #include "iceberg/catalog.h" diff --git a/src/iceberg/catalog/rest/http_request.h b/src/iceberg/catalog/rest/http_request.h index 47419c3610..4016df66f7 100644 --- a/src/iceberg/catalog/rest/http_request.h +++ b/src/iceberg/catalog/rest/http_request.h @@ -19,6 +19,9 @@ #pragma once +/// \file iceberg/catalog/rest/http_request.h +/// \brief Define REST HTTP request types. + #include #include #include diff --git a/src/iceberg/catalog/rest/iceberg_rest_export.h b/src/iceberg/catalog/rest/iceberg_rest_export.h index 7e8836f8fb..f8b322660c 100644 --- a/src/iceberg/catalog/rest/iceberg_rest_export.h +++ b/src/iceberg/catalog/rest/iceberg_rest_export.h @@ -19,6 +19,9 @@ #pragma once +/// \file iceberg/catalog/rest/iceberg_rest_export.h +/// \brief Define symbol visibility macros for the REST catalog library. + #if defined(_WIN32) || defined(__CYGWIN__) # ifdef ICEBERG_REST_STATIC # define ICEBERG_REST_EXPORT diff --git a/src/iceberg/catalog/rest/rest_file_io.h b/src/iceberg/catalog/rest/rest_file_io.h index 9e5f7a0a9c..9bd0a826e4 100644 --- a/src/iceberg/catalog/rest/rest_file_io.h +++ b/src/iceberg/catalog/rest/rest_file_io.h @@ -19,6 +19,9 @@ #pragma once +/// \file iceberg/catalog/rest/rest_file_io.h +/// \brief Provide helpers to create FileIO instances for REST catalog responses. + #include #include #include diff --git a/src/iceberg/catalog/rest/rest_util.h b/src/iceberg/catalog/rest/rest_util.h index 51a708994f..556b2557ad 100644 --- a/src/iceberg/catalog/rest/rest_util.h +++ b/src/iceberg/catalog/rest/rest_util.h @@ -19,6 +19,9 @@ #pragma once +/// \file iceberg/catalog/rest/rest_util.h +/// \brief Provide utilities for REST catalog path and config handling. + #include #include #include diff --git a/src/iceberg/catalog/session_catalog.h b/src/iceberg/catalog/session_catalog.h index 2d7aeae255..08ac048a51 100644 --- a/src/iceberg/catalog/session_catalog.h +++ b/src/iceberg/catalog/session_catalog.h @@ -19,6 +19,9 @@ #pragma once +/// \file iceberg/catalog/session_catalog.h +/// \brief Define session catalog routing APIs. + #include #include diff --git a/src/iceberg/catalog/session_context.h b/src/iceberg/catalog/session_context.h index 7dee828e1e..19eb8fc5f8 100644 --- a/src/iceberg/catalog/session_context.h +++ b/src/iceberg/catalog/session_context.h @@ -19,6 +19,9 @@ #pragma once +/// \file iceberg/catalog/session_context.h +/// \brief Define session context metadata. + #include #include diff --git a/src/iceberg/catalog/sql/iceberg_sql_catalog_export.h b/src/iceberg/catalog/sql/iceberg_sql_catalog_export.h index 88921779bd..d609e56752 100644 --- a/src/iceberg/catalog/sql/iceberg_sql_catalog_export.h +++ b/src/iceberg/catalog/sql/iceberg_sql_catalog_export.h @@ -19,6 +19,9 @@ #pragma once +/// \file iceberg/catalog/sql/iceberg_sql_catalog_export.h +/// \brief Define symbol visibility macros for the SQL catalog library. + #if defined(_WIN32) || defined(__CYGWIN__) # ifdef ICEBERG_SQL_CATALOG_STATIC # define ICEBERG_SQL_CATALOG_EXPORT diff --git a/src/iceberg/deletes/position_delete_range_consumer.h b/src/iceberg/deletes/position_delete_range_consumer.h index 69d1c33d7c..37f6caeac5 100644 --- a/src/iceberg/deletes/position_delete_range_consumer.h +++ b/src/iceberg/deletes/position_delete_range_consumer.h @@ -19,6 +19,9 @@ #pragma once +/// \file iceberg/deletes/position_delete_range_consumer.h +/// \brief Provide bulk application of position deletes. + #include #include #include diff --git a/src/iceberg/expression/literal.h b/src/iceberg/expression/literal.h index dbcabb5210..d88ea9248a 100644 --- a/src/iceberg/expression/literal.h +++ b/src/iceberg/expression/literal.h @@ -19,6 +19,9 @@ #pragma once +/// \file iceberg/expression/literal.h +/// \brief Define typed literal values used by expressions. + #include #include #include diff --git a/src/iceberg/file_io.h b/src/iceberg/file_io.h index 7d7b31113f..3ea4afa499 100644 --- a/src/iceberg/file_io.h +++ b/src/iceberg/file_io.h @@ -19,6 +19,9 @@ #pragma once +/// \file iceberg/file_io.h +/// \brief Define the FileIO abstraction and file stream interfaces. + #include #include #include diff --git a/src/iceberg/file_io_registry.h b/src/iceberg/file_io_registry.h index a0be1b5632..1643bf457e 100644 --- a/src/iceberg/file_io_registry.h +++ b/src/iceberg/file_io_registry.h @@ -19,6 +19,9 @@ #pragma once +/// \file iceberg/file_io_registry.h +/// \brief Define the FileIO registry. + #include #include #include diff --git a/src/iceberg/iceberg_bundle_export.h b/src/iceberg/iceberg_bundle_export.h index c63f8811dc..49407a34ae 100644 --- a/src/iceberg/iceberg_bundle_export.h +++ b/src/iceberg/iceberg_bundle_export.h @@ -19,6 +19,9 @@ #pragma once +/// \file iceberg/iceberg_bundle_export.h +/// \brief Define symbol visibility macros for bundled format integrations. + #if defined(_WIN32) || defined(__CYGWIN__) # ifdef ICEBERG_BUNDLE_STATIC # define ICEBERG_BUNDLE_EXPORT diff --git a/src/iceberg/iceberg_data_export.h b/src/iceberg/iceberg_data_export.h index 652de97b6d..5b710e9cc8 100644 --- a/src/iceberg/iceberg_data_export.h +++ b/src/iceberg/iceberg_data_export.h @@ -19,6 +19,9 @@ #pragma once +/// \file iceberg/iceberg_data_export.h +/// \brief Define symbol visibility macros for data APIs. + #if defined(_WIN32) || defined(__CYGWIN__) # ifdef ICEBERG_DATA_STATIC # define ICEBERG_DATA_EXPORT diff --git a/src/iceberg/iceberg_export.h b/src/iceberg/iceberg_export.h index 2463fbeaf5..5dcfa2f668 100644 --- a/src/iceberg/iceberg_export.h +++ b/src/iceberg/iceberg_export.h @@ -19,6 +19,9 @@ #pragma once +/// \file iceberg/iceberg_export.h +/// \brief Define symbol visibility macros for core Iceberg APIs. + #if defined(_WIN32) || defined(__CYGWIN__) # ifdef ICEBERG_STATIC # define ICEBERG_EXPORT diff --git a/src/iceberg/inspect/history_table.h b/src/iceberg/inspect/history_table.h index 94a26368ff..21f1f80027 100644 --- a/src/iceberg/inspect/history_table.h +++ b/src/iceberg/inspect/history_table.h @@ -19,6 +19,9 @@ #pragma once +/// \file iceberg/inspect/history_table.h +/// \brief Define the history metadata table. + #include #include "iceberg/iceberg_export.h" diff --git a/src/iceberg/inspect/metadata_table.h b/src/iceberg/inspect/metadata_table.h index 7d0ac22dad..51c5f7920e 100644 --- a/src/iceberg/inspect/metadata_table.h +++ b/src/iceberg/inspect/metadata_table.h @@ -19,6 +19,9 @@ #pragma once +/// \file iceberg/inspect/metadata_table.h +/// \brief Define base APIs for metadata tables. + #include #include "iceberg/iceberg_export.h" diff --git a/src/iceberg/inspect/snapshots_table.h b/src/iceberg/inspect/snapshots_table.h index 50017796fa..9af1bcacb4 100644 --- a/src/iceberg/inspect/snapshots_table.h +++ b/src/iceberg/inspect/snapshots_table.h @@ -19,6 +19,9 @@ #pragma once +/// \file iceberg/inspect/snapshots_table.h +/// \brief Define the snapshots metadata table. + #include #include "iceberg/iceberg_export.h" diff --git a/src/iceberg/location_provider.h b/src/iceberg/location_provider.h index 6b6fa4475a..8541b7c117 100644 --- a/src/iceberg/location_provider.h +++ b/src/iceberg/location_provider.h @@ -19,6 +19,9 @@ #pragma once +/// \file iceberg/location_provider.h +/// \brief Define table data location providers. + #include #include #include diff --git a/src/iceberg/metrics/commit_report.h b/src/iceberg/metrics/commit_report.h index 63867c8e0c..a644d0b07e 100644 --- a/src/iceberg/metrics/commit_report.h +++ b/src/iceberg/metrics/commit_report.h @@ -19,6 +19,9 @@ #pragma once +/// \file iceberg/metrics/commit_report.h +/// \brief Define commit metrics and reports. + #include #include #include diff --git a/src/iceberg/metrics/counter.h b/src/iceberg/metrics/counter.h index 4542b90f0b..b44cf844b8 100644 --- a/src/iceberg/metrics/counter.h +++ b/src/iceberg/metrics/counter.h @@ -19,6 +19,9 @@ #pragma once +/// \file iceberg/metrics/counter.h +/// \brief Define metrics counters. + #include #include #include diff --git a/src/iceberg/metrics/metrics_reporter.h b/src/iceberg/metrics/metrics_reporter.h index b255a95e81..a45754dc0c 100644 --- a/src/iceberg/metrics/metrics_reporter.h +++ b/src/iceberg/metrics/metrics_reporter.h @@ -19,6 +19,9 @@ #pragma once +/// \file iceberg/metrics/metrics_reporter.h +/// \brief Define metrics report dispatch APIs. + #include #include #include diff --git a/src/iceberg/metrics/scan_report.h b/src/iceberg/metrics/scan_report.h index 2d81e7e8fe..1ac74c05fb 100644 --- a/src/iceberg/metrics/scan_report.h +++ b/src/iceberg/metrics/scan_report.h @@ -19,6 +19,9 @@ #pragma once +/// \file iceberg/metrics/scan_report.h +/// \brief Define scan metrics and reports. + #include #include #include diff --git a/src/iceberg/metrics/timer.h b/src/iceberg/metrics/timer.h index 08bbf8e34c..e15c623f34 100644 --- a/src/iceberg/metrics/timer.h +++ b/src/iceberg/metrics/timer.h @@ -19,6 +19,9 @@ #pragma once +/// \file iceberg/metrics/timer.h +/// \brief Define metrics timers. + #include #include #include diff --git a/src/iceberg/name_mapping.h b/src/iceberg/name_mapping.h index 392b573e34..7fd140c0d7 100644 --- a/src/iceberg/name_mapping.h +++ b/src/iceberg/name_mapping.h @@ -19,6 +19,9 @@ #pragma once +/// \file iceberg/name_mapping.h +/// \brief Define schema name mapping APIs. + #include #include #include diff --git a/src/iceberg/parquet/parquet_reader.h b/src/iceberg/parquet/parquet_reader.h index 0604230c8f..3a8e57cf3a 100644 --- a/src/iceberg/parquet/parquet_reader.h +++ b/src/iceberg/parquet/parquet_reader.h @@ -19,6 +19,9 @@ #pragma once +/// \file iceberg/parquet/parquet_reader.h +/// \brief Provide the Parquet file reader interface. + #include "iceberg/file_reader.h" #include "iceberg/iceberg_bundle_export.h" diff --git a/src/iceberg/parquet/parquet_writer.h b/src/iceberg/parquet/parquet_writer.h index 5abe45db95..562ffb8013 100644 --- a/src/iceberg/parquet/parquet_writer.h +++ b/src/iceberg/parquet/parquet_writer.h @@ -19,6 +19,9 @@ #pragma once +/// \file iceberg/parquet/parquet_writer.h +/// \brief Provide the Parquet file writer interface. + #include "iceberg/file_writer.h" #include "iceberg/iceberg_bundle_export.h" diff --git a/src/iceberg/result.h b/src/iceberg/result.h index 4b4d0e7fb7..88b13c51ea 100644 --- a/src/iceberg/result.h +++ b/src/iceberg/result.h @@ -19,6 +19,9 @@ #pragma once +/// \file iceberg/result.h +/// \brief Define Result, Status, and error helpers. + #include #include #include diff --git a/src/iceberg/row/manifest_wrapper.h b/src/iceberg/row/manifest_wrapper.h index 20c2165b27..bd4fc4df68 100644 --- a/src/iceberg/row/manifest_wrapper.h +++ b/src/iceberg/row/manifest_wrapper.h @@ -19,7 +19,7 @@ #pragma once -/// \file iceberg/manifest_wrapper.h +/// \file iceberg/row/manifest_wrapper.h /// Wrapper classes for manifest-related data structures that implement /// StructLike, ArrayLike, and MapLike interfaces for unified data access. diff --git a/src/iceberg/schema_util.h b/src/iceberg/schema_util.h index 9df4023365..421d00e0ad 100644 --- a/src/iceberg/schema_util.h +++ b/src/iceberg/schema_util.h @@ -19,6 +19,9 @@ #pragma once +/// \file iceberg/schema_util.h +/// \brief Provide schema projection utilities. + #include #include #include diff --git a/src/iceberg/snapshot.h b/src/iceberg/snapshot.h index c2fd9b8a29..e32f3a715e 100644 --- a/src/iceberg/snapshot.h +++ b/src/iceberg/snapshot.h @@ -19,6 +19,9 @@ #pragma once +/// \file iceberg/snapshot.h +/// \brief Define snapshot metadata types. + #include #include #include diff --git a/src/iceberg/sort_order.h b/src/iceberg/sort_order.h index 7c9b799fbe..75cabeca7e 100644 --- a/src/iceberg/sort_order.h +++ b/src/iceberg/sort_order.h @@ -19,6 +19,9 @@ #pragma once +/// \file iceberg/sort_order.h +/// \brief Define table sort order metadata. + #include #include #include diff --git a/src/iceberg/storage_credential.h b/src/iceberg/storage_credential.h index 604e8ddce4..746bad086c 100644 --- a/src/iceberg/storage_credential.h +++ b/src/iceberg/storage_credential.h @@ -19,6 +19,9 @@ #pragma once +/// \file iceberg/storage_credential.h +/// \brief Define storage credential metadata. + #include #include diff --git a/src/iceberg/table.h b/src/iceberg/table.h index b52b5d29d0..f9b72302a2 100644 --- a/src/iceberg/table.h +++ b/src/iceberg/table.h @@ -19,6 +19,9 @@ #pragma once +/// \file iceberg/table.h +/// \brief Define table abstractions and update entry points. + #include #include #include diff --git a/src/iceberg/table_properties.h b/src/iceberg/table_properties.h index 6b92ea1fff..f334272e65 100644 --- a/src/iceberg/table_properties.h +++ b/src/iceberg/table_properties.h @@ -19,6 +19,9 @@ #pragma once +/// \file iceberg/table_properties.h +/// \brief Define table property helpers. + #include #include #include diff --git a/src/iceberg/table_scan.h b/src/iceberg/table_scan.h index e55330f307..f3e2f244d6 100644 --- a/src/iceberg/table_scan.h +++ b/src/iceberg/table_scan.h @@ -19,6 +19,9 @@ #pragma once +/// \file iceberg/table_scan.h +/// \brief Define table scan APIs and scan task types. + #include #include #include diff --git a/src/iceberg/transaction.h b/src/iceberg/transaction.h index 47714ba70e..284222351e 100644 --- a/src/iceberg/transaction.h +++ b/src/iceberg/transaction.h @@ -20,6 +20,9 @@ #pragma once +/// \file iceberg/transaction.h +/// \brief Define transaction APIs. + #include #include #include diff --git a/src/iceberg/update/snapshot_update.h b/src/iceberg/update/snapshot_update.h index 10bb16fbcb..e29d96d579 100644 --- a/src/iceberg/update/snapshot_update.h +++ b/src/iceberg/update/snapshot_update.h @@ -19,6 +19,9 @@ #pragma once +/// \file iceberg/update/snapshot_update.h +/// \brief Define base APIs for snapshot updates. + #include #include #include diff --git a/src/iceberg/util/bucket_util.h b/src/iceberg/util/bucket_util.h index 31a574b480..459f487bb6 100644 --- a/src/iceberg/util/bucket_util.h +++ b/src/iceberg/util/bucket_util.h @@ -19,6 +19,9 @@ #pragma once +/// \file iceberg/util/bucket_util.h +/// \brief Provide bucket transform utilities. + #include #include diff --git a/src/iceberg/util/config.h b/src/iceberg/util/config.h index 089c78f5d8..ed847eb631 100644 --- a/src/iceberg/util/config.h +++ b/src/iceberg/util/config.h @@ -18,6 +18,9 @@ */ #pragma once +/// \file iceberg/util/config.h +/// \brief Provide typed configuration helpers. + #include #include #include diff --git a/src/iceberg/util/executor.h b/src/iceberg/util/executor.h index 01aa83d7a7..749502b179 100644 --- a/src/iceberg/util/executor.h +++ b/src/iceberg/util/executor.h @@ -19,6 +19,9 @@ #pragma once +/// \file iceberg/util/executor.h +/// \brief Define task executor interfaces. + #include #include diff --git a/src/iceberg/util/functional.h b/src/iceberg/util/functional.h index c5ab2164b0..dbb35a3761 100644 --- a/src/iceberg/util/functional.h +++ b/src/iceberg/util/functional.h @@ -22,6 +22,9 @@ #pragma once +/// \file iceberg/util/functional.h +/// \brief Provide move-only function wrappers. + #include #include #include diff --git a/src/iceberg/util/location_util.h b/src/iceberg/util/location_util.h index eb78dece3b..c213a3bf78 100644 --- a/src/iceberg/util/location_util.h +++ b/src/iceberg/util/location_util.h @@ -19,6 +19,9 @@ #pragma once +/// \file iceberg/util/location_util.h +/// \brief Provide location path utilities. + #include #include "iceberg/iceberg_export.h" diff --git a/src/iceberg/util/macros.h b/src/iceberg/util/macros.h index e391d2bfd1..400573d8b1 100644 --- a/src/iceberg/util/macros.h +++ b/src/iceberg/util/macros.h @@ -19,6 +19,9 @@ #pragma once +/// \file iceberg/util/macros.h +/// \brief Define common Iceberg utility macros. + #include #include "iceberg/exception.h" diff --git a/src/iceberg/util/partition_value_util.h b/src/iceberg/util/partition_value_util.h index 0113623ba7..abfc1ca873 100644 --- a/src/iceberg/util/partition_value_util.h +++ b/src/iceberg/util/partition_value_util.h @@ -19,7 +19,7 @@ #pragma once -/// \file iceberg/row/partition_values.h +/// \file iceberg/util/partition_value_util.h /// Wrapper classes for partition value related data structures. #include diff --git a/src/iceberg/util/property_util.h b/src/iceberg/util/property_util.h index 4e3e9b1259..c46d3a6abd 100644 --- a/src/iceberg/util/property_util.h +++ b/src/iceberg/util/property_util.h @@ -19,6 +19,9 @@ #pragma once +/// \file iceberg/util/property_util.h +/// \brief Provide property conversion helpers. + #include #include diff --git a/src/iceberg/util/retry_util.h b/src/iceberg/util/retry_util.h index 08e3fb81d6..8d2ed2a183 100644 --- a/src/iceberg/util/retry_util.h +++ b/src/iceberg/util/retry_util.h @@ -19,6 +19,9 @@ #pragma once +/// \file iceberg/util/retry_util.h +/// \brief Provide retry policy and runner utilities. + #include #include #include diff --git a/src/iceberg/util/string_util.h b/src/iceberg/util/string_util.h index 62718a0eaa..1231673244 100644 --- a/src/iceberg/util/string_util.h +++ b/src/iceberg/util/string_util.h @@ -19,6 +19,9 @@ #pragma once +/// \file iceberg/util/string_util.h +/// \brief Provide string utility helpers. + #include #include #include diff --git a/src/iceberg/util/task_group.h b/src/iceberg/util/task_group.h index 5263816e35..3d345c240d 100644 --- a/src/iceberg/util/task_group.h +++ b/src/iceberg/util/task_group.h @@ -19,6 +19,9 @@ #pragma once +/// \file iceberg/util/task_group.h +/// \brief Provide grouped task execution utilities. + #include #include #include diff --git a/src/iceberg/util/temporal_util.h b/src/iceberg/util/temporal_util.h index acb1f0e73b..4a0642daa6 100644 --- a/src/iceberg/util/temporal_util.h +++ b/src/iceberg/util/temporal_util.h @@ -19,6 +19,9 @@ #pragma once +/// \file iceberg/util/temporal_util.h +/// \brief Provide date and timestamp conversion utilities. + #include #include #include diff --git a/src/iceberg/util/timepoint.h b/src/iceberg/util/timepoint.h index 34ffedfc79..3a12ccd227 100644 --- a/src/iceberg/util/timepoint.h +++ b/src/iceberg/util/timepoint.h @@ -19,6 +19,9 @@ #pragma once +/// \file iceberg/util/timepoint.h +/// \brief Provide time point conversion helpers. + #include #include "iceberg/iceberg_export.h" diff --git a/src/iceberg/util/transform_util.h b/src/iceberg/util/transform_util.h index e0918dcf30..77197d6ab7 100644 --- a/src/iceberg/util/transform_util.h +++ b/src/iceberg/util/transform_util.h @@ -19,6 +19,9 @@ #pragma once +/// \file iceberg/util/transform_util.h +/// \brief Provide partition transform utility helpers. + #include #include diff --git a/src/iceberg/util/truncate_util.h b/src/iceberg/util/truncate_util.h index 7fee86eba3..0ed6865463 100644 --- a/src/iceberg/util/truncate_util.h +++ b/src/iceberg/util/truncate_util.h @@ -19,6 +19,9 @@ #pragma once +/// \file iceberg/util/truncate_util.h +/// \brief Provide truncate transform utility helpers. + #include #include #include diff --git a/src/iceberg/util/visitor_generate.h b/src/iceberg/util/visitor_generate.h index ad6f5eb2d1..37a9bb6c1a 100644 --- a/src/iceberg/util/visitor_generate.h +++ b/src/iceberg/util/visitor_generate.h @@ -19,6 +19,9 @@ #pragma once +/// \file iceberg/util/visitor_generate.h +/// \brief Define macros for generating type visitors. + namespace iceberg { #define ICEBERG_GENERATE_FOR_ALL_TYPES(ACTION) \ From a45bce6326101bc13f80c89912d86b2e8e0b6621 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Sat, 11 Jul 2026 14:46:30 +0800 Subject: [PATCH 11/23] chore(deps): bump the codeql-action group with 2 updates (#827) Bumps the codeql-action group with 2 updates: [github/codeql-action/init](https://github.com/github/codeql-action) and [github/codeql-action/analyze](https://github.com/github/codeql-action). Updates `github/codeql-action/init` from 4.36.2 to 4.36.3
Release notes

Sourced from github/codeql-action/init's releases.

v4.36.3

No user facing changes.

Changelog

Sourced from github/codeql-action/init's changelog.

CodeQL Action Changelog

See the releases page for the relevant changes to the CodeQL CLI and language packs.

[UNRELEASED]

  • Upcoming breaking change: Add a deprecation warning for customers using CodeQL version 2.20.6 and earlier. These versions of CodeQL were discontinued on 1 July 2026 alongside GitHub Enterprise Server 3.16, and will be unsupported by the next minor release of the CodeQL Action. #3956

4.37.0 - 08 Jul 2026

  • Update default CodeQL bundle version to 2.26.0. #3995
  • In addition to the existing input format, the config-file input for the codeql-action/init step will soon support a new [owner/]repo[@ref][:path] format. All components except the repository name are optional. If omitted, owner defaults to the same owner as the repository the analysis is running for, ref to main, and path to .github/codeql-action.yaml. Support for this format ships in this version of the CodeQL Action, but will only be enabled over the coming weeks. #3973

4.36.3 - 01 Jul 2026

No user facing changes.

4.36.2 - 04 Jun 2026

  • Cache CodeQL CLI version information across Actions steps. #3943
  • Reduce requests while waiting for analysis processing by using exponential backoff when polling SARIF processing status. #3937
  • Update default CodeQL bundle version to 2.25.6. #3948

4.36.1 - 02 Jun 2026

No user facing changes.

4.36.0 - 22 May 2026

  • Breaking change: Bump the minimum required CodeQL bundle version to 2.19.4. #3894
  • Add support for SHA-256 Git object IDs. #3893
  • Update default CodeQL bundle version to 2.25.5. #3926

4.35.5 - 15 May 2026

  • We have improved how the JavaScript bundles for the CodeQL Action are generated to avoid duplication across bundles and reduce the size of the repository by around 70%. This should have no effect on the runtime behaviour of the CodeQL Action. #3899
  • For performance and accuracy reasons, improved incremental analysis will now only be enabled on a pull request when diff-informed analysis is also enabled for that run. If diff-informed analysis is unavailable (for example, because the PR diff ranges could not be computed), the action will fall back to a full analysis. #3791
  • If multiple inputs are provided for the GitHub-internal analysis-kinds input, only code-scanning will be enabled. The analysis-kinds input is experimental, for GitHub-internal use only, and may change without notice at any time. #3892
  • Added an experimental change which, when running a Code Scanning analysis for a PR with improved incremental analysis enabled, prefers CodeQL CLI versions that have a cached overlay-base database for the configured languages. This speeds up analysis for a repository when there is not yet a cached overlay-base database for the latest CLI version. We expect to roll this change out to everyone in May. #3880

4.35.4 - 07 May 2026

  • Update default CodeQL bundle version to 2.25.4. #3881

4.35.3 - 01 May 2026

  • Upcoming breaking change: Add a deprecation warning for customers using CodeQL version 2.19.3 and earlier. These versions of CodeQL were discontinued on 9 April 2026 alongside GitHub Enterprise Server 3.15, and will be unsupported by the next minor release of the CodeQL Action. #3837
  • Configurations for private registries that use Cloudsmith or GCP OIDC are now accepted. #3850
  • Best-effort connection tests for private registries now use GET requests instead of HEAD for better compatibility with various registry implementations. For NuGet feeds, the test is now always performed against the service index. #3853
  • Fixed a bug where two diagnostics produced within the same millisecond could overwrite each other on disk, causing one of them to be lost. #3852

... (truncated)

Commits
  • 54f647b Merge pull request #3984 from github/update-v4.36.3-1f34ec164
  • e78819e Trigger checks
  • 2c9d3d6 Update changelog for v4.36.3
  • 1f34ec1 Merge pull request #3983 from github/mbg/repo-props/ff-for-config-file-prop
  • d5f0145 Log when repository property has a value but is ignored
  • f27f563 Add test for when the FF is off
  • 0025d0f Use FF
  • f7fa18f Add FF for config file repo property
  • 628fc3f Merge pull request #3979 from github/henrymercer/overlay-db-cleanup-size-tele...
  • 9cfb67b Add clarifying comments
  • Additional commits viewable in compare view

Updates `github/codeql-action/analyze` from 4.36.2 to 4.36.3
Release notes

Sourced from github/codeql-action/analyze's releases.

v4.36.3

No user facing changes.

Changelog

Sourced from github/codeql-action/analyze's changelog.

CodeQL Action Changelog

See the releases page for the relevant changes to the CodeQL CLI and language packs.

[UNRELEASED]

  • Upcoming breaking change: Add a deprecation warning for customers using CodeQL version 2.20.6 and earlier. These versions of CodeQL were discontinued on 1 July 2026 alongside GitHub Enterprise Server 3.16, and will be unsupported by the next minor release of the CodeQL Action. #3956

4.37.0 - 08 Jul 2026

  • Update default CodeQL bundle version to 2.26.0. #3995
  • In addition to the existing input format, the config-file input for the codeql-action/init step will soon support a new [owner/]repo[@ref][:path] format. All components except the repository name are optional. If omitted, owner defaults to the same owner as the repository the analysis is running for, ref to main, and path to .github/codeql-action.yaml. Support for this format ships in this version of the CodeQL Action, but will only be enabled over the coming weeks. #3973

4.36.3 - 01 Jul 2026

No user facing changes.

4.36.2 - 04 Jun 2026

  • Cache CodeQL CLI version information across Actions steps. #3943
  • Reduce requests while waiting for analysis processing by using exponential backoff when polling SARIF processing status. #3937
  • Update default CodeQL bundle version to 2.25.6. #3948

4.36.1 - 02 Jun 2026

No user facing changes.

4.36.0 - 22 May 2026

  • Breaking change: Bump the minimum required CodeQL bundle version to 2.19.4. #3894
  • Add support for SHA-256 Git object IDs. #3893
  • Update default CodeQL bundle version to 2.25.5. #3926

4.35.5 - 15 May 2026

  • We have improved how the JavaScript bundles for the CodeQL Action are generated to avoid duplication across bundles and reduce the size of the repository by around 70%. This should have no effect on the runtime behaviour of the CodeQL Action. #3899
  • For performance and accuracy reasons, improved incremental analysis will now only be enabled on a pull request when diff-informed analysis is also enabled for that run. If diff-informed analysis is unavailable (for example, because the PR diff ranges could not be computed), the action will fall back to a full analysis. #3791
  • If multiple inputs are provided for the GitHub-internal analysis-kinds input, only code-scanning will be enabled. The analysis-kinds input is experimental, for GitHub-internal use only, and may change without notice at any time. #3892
  • Added an experimental change which, when running a Code Scanning analysis for a PR with improved incremental analysis enabled, prefers CodeQL CLI versions that have a cached overlay-base database for the configured languages. This speeds up analysis for a repository when there is not yet a cached overlay-base database for the latest CLI version. We expect to roll this change out to everyone in May. #3880

4.35.4 - 07 May 2026

  • Update default CodeQL bundle version to 2.25.4. #3881

4.35.3 - 01 May 2026

  • Upcoming breaking change: Add a deprecation warning for customers using CodeQL version 2.19.3 and earlier. These versions of CodeQL were discontinued on 9 April 2026 alongside GitHub Enterprise Server 3.15, and will be unsupported by the next minor release of the CodeQL Action. #3837
  • Configurations for private registries that use Cloudsmith or GCP OIDC are now accepted. #3850
  • Best-effort connection tests for private registries now use GET requests instead of HEAD for better compatibility with various registry implementations. For NuGet feeds, the test is now always performed against the service index. #3853
  • Fixed a bug where two diagnostics produced within the same millisecond could overwrite each other on disk, causing one of them to be lost. #3852

... (truncated)

Commits
  • 54f647b Merge pull request #3984 from github/update-v4.36.3-1f34ec164
  • e78819e Trigger checks
  • 2c9d3d6 Update changelog for v4.36.3
  • 1f34ec1 Merge pull request #3983 from github/mbg/repo-props/ff-for-config-file-prop
  • d5f0145 Log when repository property has a value but is ignored
  • f27f563 Add test for when the FF is off
  • 0025d0f Use FF
  • f7fa18f Add FF for config file repo property
  • 628fc3f Merge pull request #3979 from github/henrymercer/overlay-db-cleanup-size-tele...
  • 9cfb67b Add clarifying comments
  • Additional commits viewable in compare view

Dependabot will resolve any conflicts with this PR as long as you don't alter it yourself. You can also trigger a rebase manually by commenting `@dependabot rebase`. [//]: # (dependabot-automerge-start) [//]: # (dependabot-automerge-end) ---
Dependabot commands and options
You can trigger Dependabot actions by commenting on this PR: - `@dependabot rebase` will rebase this PR - `@dependabot recreate` will recreate this PR, overwriting any edits that have been made to it - `@dependabot show ignore conditions` will show all of the ignore conditions of the specified dependency - `@dependabot ignore major version` will close this group update PR and stop Dependabot creating any more for the specific dependency's major version (unless you unignore this specific dependency's major version or upgrade to it yourself) - `@dependabot ignore minor version` will close this group update PR and stop Dependabot creating any more for the specific dependency's minor version (unless you unignore this specific dependency's minor version or upgrade to it yourself) - `@dependabot ignore ` will close this group update PR and stop Dependabot creating any more for the specific dependency (unless you unignore this specific dependency or upgrade to it yourself) - `@dependabot unignore ` will remove all of the ignore conditions of the specified dependency - `@dependabot unignore ` will remove the ignore condition of the specified dependency and ignore conditions
Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- .github/workflows/codeql.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/codeql.yml b/.github/workflows/codeql.yml index 2c03085ebd..8220e01ea7 100644 --- a/.github/workflows/codeql.yml +++ b/.github/workflows/codeql.yml @@ -54,11 +54,11 @@ jobs: persist-credentials: false - name: Initialize CodeQL - uses: github/codeql-action/init@8aad20d150bbac5944a9f9d289da16a4b0d87c1e # v4.36.2 + uses: github/codeql-action/init@54f647b7e1bb85c95cddabcd46b0c578ec92bc1a # v4.36.3 with: languages: actions - name: Perform CodeQL Analysis - uses: github/codeql-action/analyze@8aad20d150bbac5944a9f9d289da16a4b0d87c1e # v4.36.2 + uses: github/codeql-action/analyze@54f647b7e1bb85c95cddabcd46b0c578ec92bc1a # v4.36.3 with: category: "/language:actions" From 3d5af4751f6b74e3cb588b95c464e69a520b8180 Mon Sep 17 00:00:00 2001 From: wzhuo Date: Sat, 11 Jul 2026 15:31:28 +0800 Subject: [PATCH 12/23] fix: return error for reserved property keys in UpdateProperties::Set() (#817) Previously, reserved table properties (except `format-version`) were silently ignored when set via `UpdateProperties::Set()`. This caused undetected configuration drift compared to Java, which throws exceptions for such attempts. Now attempts to set reserved properties accumulate a validation error via `ICEBERGER_BUILDER_CHECK`, surfaced at `Apply()` time. This matches Java's loud-failure behavior. --- src/iceberg/test/update_properties_test.cc | 47 ++++++++++++++++++++++ src/iceberg/update/update_properties.cc | 8 ++-- src/iceberg/update/update_properties.h | 5 ++- 3 files changed, 54 insertions(+), 6 deletions(-) diff --git a/src/iceberg/test/update_properties_test.cc b/src/iceberg/test/update_properties_test.cc index 886558153f..2a22ff949d 100644 --- a/src/iceberg/test/update_properties_test.cc +++ b/src/iceberg/test/update_properties_test.cc @@ -129,6 +129,53 @@ TEST_F(UpdatePropertiesTest, UpgradeFormatVersionUnsupported) { EXPECT_THAT(result, HasErrorMessage("unsupported format version")); } +TEST_F(UpdatePropertiesTest, SetReservedPropertyUuid) { + ICEBERG_UNWRAP_OR_FAIL(auto update, table_->NewUpdateProperties()); + update->Set("uuid", "some-uuid"); + + auto result = update->Apply(); + EXPECT_THAT(result, IsError(ErrorKind::kValidationFailed)); + EXPECT_THAT(result, HasErrorMessage("Cannot set reserved property")); +} + +TEST_F(UpdatePropertiesTest, SetReservedPropertyCurrentSchema) { + ICEBERG_UNWRAP_OR_FAIL(auto update, table_->NewUpdateProperties()); + update->Set("current-schema", R"({"type": "struct"})"); + + auto result = update->Apply(); + EXPECT_THAT(result, IsError(ErrorKind::kValidationFailed)); + EXPECT_THAT(result, HasErrorMessage("Cannot set reserved property")); +} + +TEST_F(UpdatePropertiesTest, SetReservedPropertyCurrentSnapshotId) { + ICEBERG_UNWRAP_OR_FAIL(auto update, table_->NewUpdateProperties()); + update->Set("current-snapshot-id", "12345"); + + auto result = update->Apply(); + EXPECT_THAT(result, IsError(ErrorKind::kValidationFailed)); + EXPECT_THAT(result, HasErrorMessage("Cannot set reserved property")); +} + +TEST_F(UpdatePropertiesTest, SetFormatVersionStillAllowed) { + ICEBERG_UNWRAP_OR_FAIL(auto update, table_->NewUpdateProperties()); + update->Set("format-version", "3"); + + ICEBERG_UNWRAP_OR_FAIL(auto result, update->Apply()); + EXPECT_TRUE(result.updates.empty()); + ASSERT_TRUE(result.format_version.has_value()); + EXPECT_EQ(result.format_version.value(), 3); +} + +TEST_F(UpdatePropertiesTest, SetValidAndReservedProperties) { + ICEBERG_UNWRAP_OR_FAIL(auto update, table_->NewUpdateProperties()); + update->Set("valid.key", "valid.value"); + update->Set("snapshot-count", "10"); + + auto result = update->Apply(); + EXPECT_THAT(result, IsError(ErrorKind::kValidationFailed)); + EXPECT_THAT(result, HasErrorMessage("Cannot set reserved property")); +} + TEST_F(UpdatePropertiesTest, CommitSuccess) { ICEBERG_UNWRAP_OR_FAIL(auto empty_update, table_->NewUpdateProperties()); EXPECT_THAT(empty_update->Commit(), IsOk()); diff --git a/src/iceberg/update/update_properties.cc b/src/iceberg/update/update_properties.cc index 1837ab009c..d2e75c3012 100644 --- a/src/iceberg/update/update_properties.cc +++ b/src/iceberg/update/update_properties.cc @@ -50,10 +50,10 @@ UpdateProperties& UpdateProperties::Set(const std::string& key, "Cannot set property '{}' that is already marked for removal", key); - if (!TableProperties::reserved_properties().contains(key) || - key == TableProperties::kFormatVersion.key()) { - updates_.insert_or_assign(key, value); - } + ICEBERG_BUILDER_CHECK(!TableProperties::reserved_properties().contains(key) || + key == TableProperties::kFormatVersion.key(), + "Cannot set reserved property: '{}'", key); + updates_.insert_or_assign(key, value); return *this; } diff --git a/src/iceberg/update/update_properties.h b/src/iceberg/update/update_properties.h index dee923410b..18eba427bb 100644 --- a/src/iceberg/update/update_properties.h +++ b/src/iceberg/update/update_properties.h @@ -51,8 +51,9 @@ class ICEBERG_EXPORT UpdateProperties : public PendingUpdate { /// \brief Sets a property key to a specified value. /// - /// The key must not have been previously marked for removal and reserved property keys - /// will be ignored. + /// The key must not have been previously marked for removal and must not be a + /// reserved property key (except `format-version`). Setting a reserved property + /// will result in a validation error at Apply() time. /// /// \param key The property key to set /// \param value The property value to set From 62e1bd8ed5e1ba58b86f8630d10be7cf2ecbb908 Mon Sep 17 00:00:00 2001 From: Xin Huang <42597328+huan233usc@users.noreply.github.com> Date: Sat, 11 Jul 2026 00:32:58 -0700 Subject: [PATCH 13/23] feat: support v3 column default values in UpdateSchema (#793) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ## What Part 3 of 4 of Iceberg v3 column default-value support (POC #731), built on the schema-layer support merged in #746. Independent of the read-path PRs (#792 and the Avro follow-up). Adds default-value handling to `UpdateSchema` (schema evolution). ## Changes - **`AddColumn` / `AddRequiredColumn`** take an optional `default_value`. When provided it is set as both the column's `initial-default` and `write-default`. A non-null default also lets a required column be added without `AllowIncompatibleChanges()` — rows written before the change read the default instead of null. - **`UpdateColumnDefault(name, default)`** (new) sets, or clears with `std::nullopt`, a column's `write-default`; the `initial-default` is fixed when the column is added. - Defaults are **cast to the column type** (rejecting uncastable or out-of-range values) and **preserved** across rename / doc / type-promotion updates and nested field-id reassignment. - `RequireColumn` may now mark a column that was added with a default required. The `SchemaField` constructor stores defaults verbatim (it does not coerce them), so the cast/promotion is performed explicitly at each evolution site — the same effect as Java, where `NestedField`'s constructor runs `castDefault`. Same-scale decimal precision widening is handled directly (the unscaled value is unchanged), since `Literal::CastTo` does not cast between decimal types. ## Tests 13 cases in `update_schema_test.cc`: add optional/required/nested column with a default, mismatched/narrowing rejection, `UpdateColumnDefault` (set / clear / cast-to-type / pre-existing column), require-after-default, and preservation across doc updates and type promotion — including same-scale decimal precision promotion. ## Stack 1. #746 — schema: represent / serialize / validate (merged) 2. #792 — read path: Parquet 3. read path: Avro (follows) 4. **this PR** — schema evolution: `addColumn` / `updateColumnDefault` --- src/iceberg/schema_field.cc | 68 +++- src/iceberg/schema_field.h | 42 ++- src/iceberg/test/meson.build | 1 + src/iceberg/test/schema_field_test.cc | 63 ++++ src/iceberg/test/update_schema_test.cc | 410 ++++++++++++++++++++++++- src/iceberg/type_fwd.h | 1 + src/iceberg/update/update_schema.cc | 150 +++++++-- src/iceberg/update/update_schema.h | 53 +++- src/iceberg/util/type_util.cc | 19 +- 9 files changed, 720 insertions(+), 87 deletions(-) diff --git a/src/iceberg/schema_field.cc b/src/iceberg/schema_field.cc index 6c8d10d970..80ea61bce9 100644 --- a/src/iceberg/schema_field.cc +++ b/src/iceberg/schema_field.cc @@ -22,9 +22,11 @@ #include #include #include +#include #include "iceberg/expression/literal.h" #include "iceberg/type.h" +#include "iceberg/util/checked_cast.h" #include "iceberg/util/formatter.h" // IWYU pragma: keep #include "iceberg/util/macros.h" @@ -32,8 +34,7 @@ namespace iceberg { namespace { -// A null default value is modeled as the absence of a default (matching Java), so it is -// not stored. +// Treat null defaults as absent. std::shared_ptr DropNullDefault(std::shared_ptr value) { if (value != nullptr && value->IsNull()) { return nullptr; @@ -65,6 +66,55 @@ SchemaField SchemaField::MakeRequired(int32_t field_id, std::string_view name, return {field_id, name, std::move(type), false, doc}; } +SchemaField SchemaField::WithName(std::string_view name) const { + return {field_id_, name, type_, optional_, doc_, initial_default_, write_default_}; +} + +SchemaField SchemaField::WithType(std::shared_ptr type) const { + return {field_id_, name_, std::move(type), optional_, doc_, + initial_default_, write_default_}; +} + +SchemaField SchemaField::WithDoc(std::string_view doc) const { + return {field_id_, name_, type_, optional_, doc, initial_default_, write_default_}; +} + +SchemaField SchemaField::WithInitialDefault( + std::shared_ptr initial_default) const { + return {field_id_, name_, type_, optional_, doc_, std::move(initial_default), + write_default_}; +} + +SchemaField SchemaField::WithWriteDefault( + std::shared_ptr write_default) const { + return {field_id_, + name_, + type_, + optional_, + doc_, + initial_default_, + std::move(write_default)}; +} + +Result SchemaField::CastDefaultValue( + const Literal& value, const std::shared_ptr& target_type) { + if (target_type->type_id() == TypeId::kDecimal && + std::holds_alternative(value.value())) { + const auto& source_type = internal::checked_cast(*value.type()); + const auto& decimal_type = internal::checked_cast(*target_type); + if (source_type.scale() == decimal_type.scale()) { + const auto& decimal_value = std::get(value.value()); + if (!decimal_value.FitsInPrecision(decimal_type.precision())) { + return InvalidArgument("Cannot cast default value to {}: {}", *target_type, + value); + } + return Literal::Decimal(decimal_value.value(), decimal_type.precision(), + decimal_type.scale()); + } + } + return value.CastTo(target_type); +} + int32_t SchemaField::field_id() const { return field_id_; } std::string_view SchemaField::name() const { return name_; } @@ -87,8 +137,7 @@ namespace { Status ValidateDefault(const SchemaField& field, const Literal& value, std::string_view kind) { - // A null default is modeled as absence and dropped at construction, so it never reaches - // here; only the out-of-range cast sentinels need rejecting. + // Null defaults are dropped at construction. if (value.IsAboveMax() || value.IsBelowMin()) { return InvalidSchema("Invalid {} value for {}: value is out of range", kind, field.name()); @@ -97,8 +146,7 @@ Status ValidateDefault(const SchemaField& field, const Literal& value, return InvalidSchema("Invalid {} value for {}: field has no type", kind, field.name()); } - // The spec requires unknown/variant/geometry/geography columns to default to null, so a - // non-null default on them is invalid (a null default was already dropped as absence). + // These types can only use null defaults. switch (field.type()->type_id()) { case TypeId::kUnknown: case TypeId::kVariant: @@ -109,17 +157,13 @@ Status ValidateDefault(const SchemaField& field, const Literal& value, default: break; } - // Defaults are otherwise only supported on primitive fields. The spec also permits JSON - // single-value defaults for struct/list/map (e.g. an empty struct `{}` whose sub-field - // defaults live in field metadata); that matches the current Java model's gap and is - // left as a follow-up. + // Nested defaults are not supported yet. if (!field.type()->is_primitive()) { return InvalidSchema( "Invalid {} value for {}: default values are only supported for primitive types", kind, field.name()); } - // Defaults are stored verbatim (no implicit cast), so a default whose literal type does - // not match the field type is invalid. + // Stored defaults must already match the field type. if (*value.type() != *field.type()) { return InvalidSchema("{} of field {} has type {} but expected {}", kind, field.name(), *value.type(), *field.type()); diff --git a/src/iceberg/schema_field.h b/src/iceberg/schema_field.h index 8066a64065..86bfb92b63 100644 --- a/src/iceberg/schema_field.h +++ b/src/iceberg/schema_field.h @@ -46,10 +46,8 @@ class ICEBERG_EXPORT SchemaField : public iceberg::util::Formattable { /// \param[in] type The field type. /// \param[in] optional Whether values of this field are required or nullable. /// \param[in] doc Optional documentation string for the field. - /// \param[in] initial_default The v3 `initial-default` value, or null if absent. The - /// field shares ownership of the (immutable) value. - /// \param[in] write_default The v3 `write-default` value, or null if absent. The field - /// shares ownership of the (immutable) value. + /// \param[in] initial_default The v3 `initial-default`, or null if absent. + /// \param[in] write_default The v3 `write-default`, or null if absent. SchemaField(int32_t field_id, std::string_view name, std::shared_ptr type, bool optional, std::string_view doc = {}, std::shared_ptr initial_default = nullptr, @@ -62,14 +60,33 @@ class ICEBERG_EXPORT SchemaField : public iceberg::util::Formattable { static SchemaField MakeRequired(int32_t field_id, std::string_view name, std::shared_ptr type, std::string_view doc = {}); + /// \brief Return a copy with a new name. + SchemaField WithName(std::string_view name) const; + + /// \brief Return a copy with a new type. + SchemaField WithType(std::shared_ptr type) const; + + /// \brief Return a copy with new documentation. + SchemaField WithDoc(std::string_view doc) const; + + /// \brief Return a copy with a new `initial-default`. + SchemaField WithInitialDefault(std::shared_ptr initial_default) const; + + /// \brief Return a copy with a new `write-default`. + SchemaField WithWriteDefault(std::shared_ptr write_default) const; + + /// \brief Cast a default literal to a field type. + static Result CastDefaultValue( + const Literal& value, const std::shared_ptr& target_type); + /// \brief Get the field ID. - [[nodiscard]] int32_t field_id() const; + int32_t field_id() const; /// \brief Get the field name. - [[nodiscard]] std::string_view name() const; + std::string_view name() const; /// \brief Get the field type. - [[nodiscard]] const std::shared_ptr& type() const; + const std::shared_ptr& type() const; /// \brief Get whether the field is optional. [[nodiscard]] bool optional() const; @@ -77,15 +94,13 @@ class ICEBERG_EXPORT SchemaField : public iceberg::util::Formattable { /// \brief Get the field documentation. std::string_view doc() const; - /// \brief Get the owning pointer to the default value for this field used when reading - /// rows written before the field existed (v3 `initial-default`), or null if absent. + /// \brief Get the v3 `initial-default`, or null if absent. const std::shared_ptr& initial_default() const; - /// \brief Get the owning pointer to the default value for this field used when a writer - /// does not supply a value (v3 `write-default`), or null if absent. + /// \brief Get the v3 `write-default`, or null if absent. const std::shared_ptr& write_default() const; - [[nodiscard]] std::string ToString() const override; + std::string ToString() const override; Status Validate() const; @@ -107,14 +122,13 @@ class ICEBERG_EXPORT SchemaField : public iceberg::util::Formattable { private: /// \brief Compare two fields for equality. - [[nodiscard]] bool Equals(const SchemaField& other) const; + bool Equals(const SchemaField& other) const; int32_t field_id_; std::string name_; std::shared_ptr type_; bool optional_; std::string doc_; - // Immutable default values, shared (not deep-copied) across field copies, like `type_`. std::shared_ptr initial_default_; std::shared_ptr write_default_; }; diff --git a/src/iceberg/test/meson.build b/src/iceberg/test/meson.build index ba0f6230d0..12460237d3 100644 --- a/src/iceberg/test/meson.build +++ b/src/iceberg/test/meson.build @@ -59,6 +59,7 @@ iceberg_tests = { 'table_requirements_test.cc', 'table_test.cc', 'table_update_test.cc', + 'update_schema_test.cc', ), }, 'logging_test': { diff --git a/src/iceberg/test/schema_field_test.cc b/src/iceberg/test/schema_field_test.cc index c2cd6a7584..d757cb0b20 100644 --- a/src/iceberg/test/schema_field_test.cc +++ b/src/iceberg/test/schema_field_test.cc @@ -24,6 +24,7 @@ #include +#include "iceberg/expression/literal.h" #include "iceberg/type.h" #include "iceberg/util/formatter.h" // IWYU pragma: keep @@ -107,4 +108,66 @@ TEST(SchemaFieldTest, WithDoc) { } } +TEST(SchemaFieldTest, WithFieldMetadata) { + auto initial_default = std::make_shared(Literal::Int(1)); + auto write_default = std::make_shared(Literal::Int(2)); + SchemaField field(1, "foo", int32(), false, "doc", initial_default, write_default); + + auto renamed = field.WithName("bar"); + EXPECT_EQ(renamed.name(), "bar"); + EXPECT_EQ(renamed.field_id(), field.field_id()); + EXPECT_EQ(renamed.type(), field.type()); + EXPECT_EQ(renamed.doc(), field.doc()); + EXPECT_EQ(renamed.initial_default(), field.initial_default()); + EXPECT_EQ(renamed.write_default(), field.write_default()); + + auto retyped = field.WithType(int64()); + EXPECT_EQ(retyped.type(), int64()); + EXPECT_EQ(retyped.name(), field.name()); + EXPECT_EQ(retyped.initial_default(), field.initial_default()); + EXPECT_EQ(retyped.write_default(), field.write_default()); + + auto documented = field.WithDoc("new doc"); + EXPECT_EQ(documented.doc(), "new doc"); + EXPECT_EQ(documented.name(), field.name()); + EXPECT_EQ(documented.initial_default(), field.initial_default()); + EXPECT_EQ(documented.write_default(), field.write_default()); + + auto new_initial = std::make_shared(Literal::Int(3)); + auto with_initial = field.WithInitialDefault(new_initial); + EXPECT_EQ(with_initial.initial_default(), new_initial); + EXPECT_EQ(with_initial.write_default(), field.write_default()); + + auto new_write = std::make_shared(Literal::Int(4)); + auto with_write = field.WithWriteDefault(new_write); + EXPECT_EQ(with_write.initial_default(), field.initial_default()); + EXPECT_EQ(with_write.write_default(), new_write); + EXPECT_EQ(field.name(), "foo"); + EXPECT_EQ(*field.write_default(), Literal::Int(2)); +} + +TEST(SchemaFieldTest, CastDefaultValue) { + { + auto result = SchemaField::CastDefaultValue(Literal::Int(5), int64()); + ASSERT_TRUE(result.has_value()); + EXPECT_EQ(result.value(), Literal::Long(5)); + } + { + auto result = + SchemaField::CastDefaultValue(Literal::Decimal(1234, 9, 2), decimal(18, 2)); + ASSERT_TRUE(result.has_value()); + EXPECT_EQ(result.value(), Literal::Decimal(1234, 18, 2)); + } + { + auto result = + SchemaField::CastDefaultValue(Literal::Decimal(1234, 9, 3), decimal(18, 2)); + EXPECT_FALSE(result.has_value()); + } + { + auto result = + SchemaField::CastDefaultValue(Literal::Decimal(1234567, 9, 2), decimal(4, 2)); + EXPECT_FALSE(result.has_value()); + } +} + } // namespace iceberg diff --git a/src/iceberg/test/update_schema_test.cc b/src/iceberg/test/update_schema_test.cc index 07872e69a3..db6b1d02e1 100644 --- a/src/iceberg/test/update_schema_test.cc +++ b/src/iceberg/test/update_schema_test.cc @@ -19,23 +19,88 @@ #include "iceberg/update/update_schema.h" +#include +#include #include +#include +#include +#include #include +#include #include +#include "iceberg/catalog/memory/in_memory_catalog.h" +#include "iceberg/expression/literal.h" #include "iceberg/schema.h" #include "iceberg/schema_field.h" +#include "iceberg/table.h" +#include "iceberg/table_identifier.h" +#include "iceberg/table_metadata.h" #include "iceberg/test/matchers.h" -#include "iceberg/test/update_test_base.h" +#include "iceberg/test/mock_io.h" +#include "iceberg/test/test_resource.h" #include "iceberg/type.h" #include "iceberg/util/checked_cast.h" +#include "iceberg/util/uuid.h" namespace iceberg { using internal::checked_cast; -class UpdateSchemaTest : public UpdateTestBase {}; +class UpdateSchemaTest : public ::testing::Test { + protected: + virtual std::string MetadataResource() const { return "TableMetadataV2Valid.json"; } + virtual std::string TableName() const { return "test_table"; } + + void SetUp() override { + table_ident_ = TableIdentifier{.name = TableName()}; + table_location_ = "/warehouse/" + TableName(); + + InitializeFileIO(); + RegisterTableFromResource(MetadataResource()); + } + + void InitializeFileIO() { + auto mock_file_io = std::make_shared<::testing::NiceMock>(); + auto* mock_file_io_ptr = mock_file_io.get(); + ON_CALL(*mock_file_io, ReadFile(::testing::_, ::testing::_)) + .WillByDefault([mock_file_io_ptr](const std::string& file_location, + std::optional length) { + return mock_file_io_ptr->FileIO::ReadFile(file_location, length); + }); + ON_CALL(*mock_file_io, WriteFile(::testing::_, ::testing::_)) + .WillByDefault([mock_file_io_ptr](const std::string& file_location, + std::string_view content) { + return mock_file_io_ptr->FileIO::WriteFile(file_location, content); + }); + file_io_ = std::move(mock_file_io); + catalog_ = + InMemoryCatalog::Make("test_catalog", file_io_, "/warehouse/", /*properties=*/{}); + } + + void RegisterTable(std::unique_ptr metadata) { + auto metadata_location = std::format("{}/metadata/00001-{}.metadata.json", + table_location_, Uuid::GenerateV7().ToString()); + metadata->location = table_location_; + ASSERT_THAT(TableMetadataUtil::Write(*file_io_, metadata_location, *metadata), + IsOk()); + + ICEBERG_UNWRAP_OR_FAIL(table_, + catalog_->RegisterTable(table_ident_, metadata_location)); + } + + void RegisterTableFromResource(const std::string& resource_name) { + ICEBERG_UNWRAP_OR_FAIL(auto metadata, ReadTableMetadataFromResource(resource_name)); + RegisterTable(std::move(metadata)); + } + + TableIdentifier table_ident_; + std::string table_location_; + std::shared_ptr file_io_; + std::shared_ptr catalog_; + std::shared_ptr table_; +}; TEST_F(UpdateSchemaTest, AddOptionalColumn) { ICEBERG_UNWRAP_OR_FAIL(auto update, table_->NewUpdateSchema()); @@ -82,6 +147,347 @@ TEST_F(UpdateSchemaTest, AddRequiredColumnWithAllowIncompatible) { EXPECT_EQ(new_field.doc(), "A required string column"); } +/// Uses v3 metadata for default-value tests. +class UpdateSchemaDefaultValueTest : public UpdateSchemaTest { + protected: + void SetUp() override { + table_ident_ = TableIdentifier{.name = TableName()}; + table_location_ = "/warehouse/" + TableName(); + + InitializeFileIO(); + + ICEBERG_UNWRAP_OR_FAIL(auto metadata, + ReadTableMetadataFromResource("TableMetadataV2Valid.json")); + metadata->format_version = TableMetadata::kSupportedTableFormatVersion; + metadata->next_row_id = TableMetadata::kInitialRowId; + RegisterTable(std::move(metadata)); + } +}; + +TEST_F(UpdateSchemaTest, AddColumnWithDefaultValueRequiresV3) { + ICEBERG_UNWRAP_OR_FAIL(auto update, table_->NewUpdateSchema()); + update->AddColumn("new_col", int32(), "An integer column", Literal::Int(42)); + + auto result = update->Apply(); + EXPECT_THAT(result, IsError(ErrorKind::kInvalidSchema)); + EXPECT_THAT(result, HasErrorMessage("is not supported until v3")); +} + +TEST_F(UpdateSchemaDefaultValueTest, AddColumnWithDefaultValue) { + ICEBERG_UNWRAP_OR_FAIL(auto update, table_->NewUpdateSchema()); + update->AddColumn("new_col", int32(), "An integer column", Literal::Int(42)); + + ICEBERG_UNWRAP_OR_FAIL(auto result, update->Apply()); + ICEBERG_UNWRAP_OR_FAIL(auto new_field_opt, result.schema->FindFieldByName("new_col")); + ASSERT_TRUE(new_field_opt.has_value()); + + const auto& new_field = new_field_opt->get(); + ASSERT_NE(new_field.initial_default(), nullptr); + EXPECT_EQ(*new_field.initial_default(), Literal::Int(42)); + ASSERT_NE(new_field.write_default(), nullptr); + EXPECT_EQ(*new_field.write_default(), Literal::Int(42)); +} + +TEST_F(UpdateSchemaDefaultValueTest, AddRequiredColumnWithDefaultValue) { + ICEBERG_UNWRAP_OR_FAIL(auto update, table_->NewUpdateSchema()); + update->AddRequiredColumn("required_col", string(), "A required string column", + Literal::String("n/a")); + + ICEBERG_UNWRAP_OR_FAIL(auto result, update->Apply()); + ICEBERG_UNWRAP_OR_FAIL(auto new_field_opt, + result.schema->FindFieldByName("required_col")); + ASSERT_TRUE(new_field_opt.has_value()); + + const auto& new_field = new_field_opt->get(); + EXPECT_FALSE(new_field.optional()); + ASSERT_NE(new_field.initial_default(), nullptr); + EXPECT_EQ(*new_field.initial_default(), Literal::String("n/a")); + ASSERT_NE(new_field.write_default(), nullptr); + EXPECT_EQ(*new_field.write_default(), Literal::String("n/a")); +} + +TEST_F(UpdateSchemaDefaultValueTest, AddColumnWithMismatchedDefaultValueFails) { + ICEBERG_UNWRAP_OR_FAIL(auto update, table_->NewUpdateSchema()); + update->AddColumn("new_col", int32(), "An integer column", Literal::String("oops")); + + auto result = update->Apply(); + EXPECT_THAT(result, IsError(ErrorKind::kValidationFailed)); + EXPECT_THAT(result, HasErrorMessage("Cannot cast default value")); +} + +TEST_F(UpdateSchemaDefaultValueTest, AddColumnWithNarrowingDefaultValueFails) { + ICEBERG_UNWRAP_OR_FAIL(auto update, table_->NewUpdateSchema()); + update->AddColumn("new_col", int32(), "An integer column", + Literal::Long(std::numeric_limits::max())); + + auto result = update->Apply(); + EXPECT_THAT(result, IsError(ErrorKind::kValidationFailed)); + EXPECT_THAT(result, HasErrorMessage("Cannot cast default value")); +} + +TEST_F(UpdateSchemaDefaultValueTest, UpdateColumnDefault) { + ICEBERG_UNWRAP_OR_FAIL(auto update, table_->NewUpdateSchema()); + update->AddColumn("new_col", int32(), "An integer column", Literal::Int(42)) + .UpdateColumnDefault("new_col", Literal::Int(7)); + + ICEBERG_UNWRAP_OR_FAIL(auto result, update->Apply()); + ICEBERG_UNWRAP_OR_FAIL(auto new_field_opt, result.schema->FindFieldByName("new_col")); + ASSERT_TRUE(new_field_opt.has_value()); + + const auto& new_field = new_field_opt->get(); + ASSERT_NE(new_field.initial_default(), nullptr); + EXPECT_EQ(*new_field.initial_default(), Literal::Int(42)); + ASSERT_NE(new_field.write_default(), nullptr); + EXPECT_EQ(*new_field.write_default(), Literal::Int(7)); +} + +TEST_F(UpdateSchemaDefaultValueTest, UpdateColumnDefaultOnExistingColumn) { + ICEBERG_UNWRAP_OR_FAIL(auto update, table_->NewUpdateSchema()); + update->UpdateColumnDefault("x", Literal::Long(0)); + + ICEBERG_UNWRAP_OR_FAIL(auto result, update->Apply()); + ICEBERG_UNWRAP_OR_FAIL(auto field_opt, result.schema->FindFieldByName("x")); + ASSERT_TRUE(field_opt.has_value()); + + const auto& field = field_opt->get(); + EXPECT_EQ(field.initial_default(), nullptr); + ASSERT_NE(field.write_default(), nullptr); + EXPECT_EQ(*field.write_default(), Literal::Long(0)); +} + +TEST_F(UpdateSchemaDefaultValueTest, UpdateColumnDefaultClearsWithNullopt) { + ICEBERG_UNWRAP_OR_FAIL(auto update, table_->NewUpdateSchema()); + update->AddColumn("new_col", int32(), "An integer column", Literal::Int(42)) + .UpdateColumnDefault("new_col", std::nullopt); + + ICEBERG_UNWRAP_OR_FAIL(auto result, update->Apply()); + ICEBERG_UNWRAP_OR_FAIL(auto field_opt, result.schema->FindFieldByName("new_col")); + ASSERT_TRUE(field_opt.has_value()); + + const auto& field = field_opt->get(); + ASSERT_NE(field.initial_default(), nullptr); + EXPECT_EQ(*field.initial_default(), Literal::Int(42)); + EXPECT_EQ(field.write_default(), nullptr); +} + +TEST_F(UpdateSchemaDefaultValueTest, AddNestedColumnPreservesNestedDefaults) { + auto nested_type = std::make_shared(std::vector{ + SchemaField(/*field_id=*/100, "inner", int32(), /*optional=*/false, /*doc=*/{}, + std::make_shared(Literal::Int(5)), + std::make_shared(Literal::Int(9)))}); + + ICEBERG_UNWRAP_OR_FAIL(auto update, table_->NewUpdateSchema()); + update->AddColumn("outer", nested_type, "A nested column"); + + ICEBERG_UNWRAP_OR_FAIL(auto result, update->Apply()); + ICEBERG_UNWRAP_OR_FAIL(auto outer_opt, result.schema->FindFieldByName("outer")); + ASSERT_TRUE(outer_opt.has_value()); + + const auto& outer_struct = + internal::checked_cast(*outer_opt->get().type()); + ASSERT_EQ(outer_struct.fields().size(), 1); + const SchemaField& inner = outer_struct.fields()[0]; + ASSERT_NE(inner.initial_default(), nullptr); + EXPECT_EQ(*inner.initial_default(), Literal::Int(5)); + ASSERT_NE(inner.write_default(), nullptr); + EXPECT_EQ(*inner.write_default(), Literal::Int(9)); +} + +TEST_F(UpdateSchemaDefaultValueTest, UpdateColumnDefaultCastsToColumnType) { + ICEBERG_UNWRAP_OR_FAIL(auto update, table_->NewUpdateSchema()); + update->UpdateColumnDefault("x", Literal::Int(5)); + + ICEBERG_UNWRAP_OR_FAIL(auto result, update->Apply()); + ICEBERG_UNWRAP_OR_FAIL(auto field_opt, result.schema->FindFieldByName("x")); + ASSERT_TRUE(field_opt.has_value()); + + const auto& field = field_opt->get(); + ASSERT_NE(field.write_default(), nullptr); + EXPECT_EQ(*field.write_default(), Literal::Long(5)); +} + +TEST_F(UpdateSchemaDefaultValueTest, RequireColumnAddedWithDefault) { + ICEBERG_UNWRAP_OR_FAIL(auto update, table_->NewUpdateSchema()); + update->AddColumn("new_col", int32(), "An integer column", Literal::Int(42)) + .RequireColumn("new_col"); + + ICEBERG_UNWRAP_OR_FAIL(auto result, update->Apply()); + ICEBERG_UNWRAP_OR_FAIL(auto new_field_opt, result.schema->FindFieldByName("new_col")); + ASSERT_TRUE(new_field_opt.has_value()); + EXPECT_FALSE(new_field_opt->get().optional()); +} + +TEST_F(UpdateSchemaDefaultValueTest, RequireNestedMapListColumnAddedWithDefault) { + // Map/list paths omit value/element. + auto map_key_struct = std::make_shared( + std::vector{SchemaField(20, "address", string(), false)}); + auto map_value_struct = std::make_shared( + std::vector{SchemaField(12, "lat", float32(), false), + SchemaField(13, "long", float32(), false)}); + auto map_type = + std::make_shared(SchemaField(10, "key", map_key_struct, false), + SchemaField(11, "value", map_value_struct, false)); + auto list_element_struct = std::make_shared(std::vector{ + SchemaField(15, "x", int64(), false), SchemaField(16, "y", int64(), false)}); + auto list_type = + std::make_shared(SchemaField(14, "element", list_element_struct, true)); + + ICEBERG_UNWRAP_OR_FAIL(auto setup_update, table_->NewUpdateSchema()); + setup_update->AddColumn("locations", map_type, "map of address to coordinate") + .AddColumn("points", list_type, "2-D cartesian points"); + EXPECT_THAT(setup_update->Commit(), IsOk()); + + ICEBERG_UNWRAP_OR_FAIL(auto reloaded, catalog_->LoadTable(table_ident_)); + ICEBERG_UNWRAP_OR_FAIL(auto update, reloaded->NewUpdateSchema()); + update->AddColumn("locations", "alt", float32(), "altitude", Literal::Float(0.0f)) + .RequireColumn("locations.alt") + .AddColumn("points", "z", int64(), "z coordinate", Literal::Long(0)) + .RequireColumn("points.z"); + + ICEBERG_UNWRAP_OR_FAIL(auto result, update->Apply()); + + ICEBERG_UNWRAP_OR_FAIL(auto locations_opt, result.schema->FindFieldByName("locations")); + ASSERT_TRUE(locations_opt.has_value()); + const auto& map = checked_cast(*locations_opt->get().type()); + const auto& value_struct = checked_cast(*map.value().type()); + ICEBERG_UNWRAP_OR_FAIL(auto alt_opt, value_struct.GetFieldByName("alt")); + ASSERT_TRUE(alt_opt.has_value()); + EXPECT_FALSE(alt_opt->get().optional()); + ASSERT_NE(alt_opt->get().initial_default(), nullptr); + + ICEBERG_UNWRAP_OR_FAIL(auto points_opt, result.schema->FindFieldByName("points")); + ASSERT_TRUE(points_opt.has_value()); + const auto& list = checked_cast(*points_opt->get().type()); + const auto& element_struct = checked_cast(*list.element().type()); + ICEBERG_UNWRAP_OR_FAIL(auto z_opt, element_struct.GetFieldByName("z")); + ASSERT_TRUE(z_opt.has_value()); + EXPECT_FALSE(z_opt->get().optional()); + ASSERT_NE(z_opt->get().initial_default(), nullptr); +} + +TEST_F(UpdateSchemaDefaultValueTest, UpdateColumnDocPreservesDefaultValues) { + ICEBERG_UNWRAP_OR_FAIL(auto update, table_->NewUpdateSchema()); + update->AddColumn("new_col", int32(), "An integer column", Literal::Int(42)) + .UpdateColumnDoc("new_col", "updated doc"); + + ICEBERG_UNWRAP_OR_FAIL(auto result, update->Apply()); + ICEBERG_UNWRAP_OR_FAIL(auto field_opt, result.schema->FindFieldByName("new_col")); + ASSERT_TRUE(field_opt.has_value()); + + const auto& field = field_opt->get(); + EXPECT_EQ(field.doc(), "updated doc"); + ASSERT_NE(field.initial_default(), nullptr); + EXPECT_EQ(*field.initial_default(), Literal::Int(42)); + ASSERT_NE(field.write_default(), nullptr); + EXPECT_EQ(*field.write_default(), Literal::Int(42)); +} + +TEST_F(UpdateSchemaDefaultValueTest, UpdateColumnTypePromotesDefaultValues) { + ICEBERG_UNWRAP_OR_FAIL(auto update, table_->NewUpdateSchema()); + update->AddColumn("new_col", int32(), "An integer column", Literal::Int(42)) + .UpdateColumn("new_col", int64()); + + ICEBERG_UNWRAP_OR_FAIL(auto result, update->Apply()); + ICEBERG_UNWRAP_OR_FAIL(auto field_opt, result.schema->FindFieldByName("new_col")); + ASSERT_TRUE(field_opt.has_value()); + + const auto& field = field_opt->get(); + EXPECT_EQ(field.type(), int64()); + ASSERT_NE(field.initial_default(), nullptr); + EXPECT_EQ(*field.initial_default(), Literal::Long(42)); + ASSERT_NE(field.write_default(), nullptr); + EXPECT_EQ(*field.write_default(), Literal::Long(42)); +} + +TEST_F(UpdateSchemaDefaultValueTest, UpdateColumnTypePromotesDecimalDefault) { + ICEBERG_UNWRAP_OR_FAIL(auto update, table_->NewUpdateSchema()); + update + ->AddColumn("new_col", decimal(9, 2), "A decimal column", + Literal::Decimal(1234, 9, 2)) + .UpdateColumn("new_col", decimal(18, 2)); + + ICEBERG_UNWRAP_OR_FAIL(auto result, update->Apply()); + ICEBERG_UNWRAP_OR_FAIL(auto field_opt, result.schema->FindFieldByName("new_col")); + ASSERT_TRUE(field_opt.has_value()); + + const auto& field = field_opt->get(); + EXPECT_EQ(field.type()->ToString(), decimal(18, 2)->ToString()); + ASSERT_NE(field.initial_default(), nullptr); + EXPECT_EQ(*field.initial_default(), Literal::Decimal(1234, 18, 2)); + ASSERT_NE(field.write_default(), nullptr); + EXPECT_EQ(*field.write_default(), Literal::Decimal(1234, 18, 2)); +} + +TEST_F(UpdateSchemaDefaultValueTest, AddColumnWithWiderPrecisionDecimalDefault) { + ICEBERG_UNWRAP_OR_FAIL(auto update, table_->NewUpdateSchema()); + update->AddColumn("new_col", decimal(18, 2), "A decimal column", + Literal::Decimal(1234, 9, 2)); + + ICEBERG_UNWRAP_OR_FAIL(auto result, update->Apply()); + ICEBERG_UNWRAP_OR_FAIL(auto field_opt, result.schema->FindFieldByName("new_col")); + ASSERT_TRUE(field_opt.has_value()); + + const auto& field = field_opt->get(); + ASSERT_NE(field.initial_default(), nullptr); + EXPECT_EQ(*field.initial_default(), Literal::Decimal(1234, 18, 2)); + ASSERT_NE(field.write_default(), nullptr); + EXPECT_EQ(*field.write_default(), Literal::Decimal(1234, 18, 2)); +} + +TEST_F(UpdateSchemaDefaultValueTest, UpdateColumnDefaultWiderPrecisionDecimal) { + ICEBERG_UNWRAP_OR_FAIL(auto update, table_->NewUpdateSchema()); + update->AddColumn("new_col", decimal(18, 2), "A decimal column") + .UpdateColumnDefault("new_col", Literal::Decimal(1234, 9, 2)); + + ICEBERG_UNWRAP_OR_FAIL(auto result, update->Apply()); + ICEBERG_UNWRAP_OR_FAIL(auto field_opt, result.schema->FindFieldByName("new_col")); + ASSERT_TRUE(field_opt.has_value()); + + const auto& field = field_opt->get(); + ASSERT_NE(field.write_default(), nullptr); + EXPECT_EQ(*field.write_default(), Literal::Decimal(1234, 18, 2)); +} + +TEST_F(UpdateSchemaDefaultValueTest, AddColumnWithDifferentScaleDecimalDefaultFails) { + ICEBERG_UNWRAP_OR_FAIL(auto update, table_->NewUpdateSchema()); + update->AddColumn("new_col", decimal(18, 2), "A decimal column", + Literal::Decimal(1234, 9, 3)); + + auto result = update->Apply(); + EXPECT_THAT(result, IsError(ErrorKind::kValidationFailed)); + EXPECT_THAT(result, HasErrorMessage("Cannot cast default value")); +} + +TEST_F(UpdateSchemaDefaultValueTest, UpdateColumnDefaultDifferentScaleDecimalFails) { + ICEBERG_UNWRAP_OR_FAIL(auto update, table_->NewUpdateSchema()); + update->AddColumn("new_col", decimal(18, 2), "A decimal column") + .UpdateColumnDefault("new_col", Literal::Decimal(1234, 9, 3)); + + auto result = update->Apply(); + EXPECT_THAT(result, IsError(ErrorKind::kValidationFailed)); + EXPECT_THAT(result, HasErrorMessage("Cannot cast default value")); +} + +TEST_F(UpdateSchemaDefaultValueTest, AddColumnWithOutOfPrecisionDecimalDefaultFails) { + ICEBERG_UNWRAP_OR_FAIL(auto update, table_->NewUpdateSchema()); + update->AddColumn("new_col", decimal(4, 2), "A decimal column", + Literal::Decimal(1234567, 9, 2)); + + auto result = update->Apply(); + EXPECT_THAT(result, IsError(ErrorKind::kValidationFailed)); + EXPECT_THAT(result, HasErrorMessage("Cannot cast default value")); +} + +TEST_F(UpdateSchemaDefaultValueTest, AddColumnWithTypedNullDecimalDefaultFails) { + ICEBERG_UNWRAP_OR_FAIL(auto update, table_->NewUpdateSchema()); + update->AddColumn("new_col", decimal(18, 2), "A decimal column", + Literal::Null(decimal(18, 2))); + + auto result = update->Apply(); + EXPECT_THAT(result, IsError(ErrorKind::kValidationFailed)); + EXPECT_THAT(result, HasErrorMessage("Cannot cast default value")); +} + TEST_F(UpdateSchemaTest, AddMultipleColumns) { ICEBERG_UNWRAP_OR_FAIL(auto update, table_->NewUpdateSchema()); update->AddColumn("col1", int32(), "First column") diff --git a/src/iceberg/type_fwd.h b/src/iceberg/type_fwd.h index 35a493b48a..91e18b24c8 100644 --- a/src/iceberg/type_fwd.h +++ b/src/iceberg/type_fwd.h @@ -104,6 +104,7 @@ class GeometryType; /// \brief Data values. class Decimal; +class Literal; class Uuid; /// \brief Schema. diff --git a/src/iceberg/update/update_schema.cc b/src/iceberg/update/update_schema.cc index 5c50ee41da..56e167e108 100644 --- a/src/iceberg/update/update_schema.cc +++ b/src/iceberg/update/update_schema.cc @@ -29,6 +29,7 @@ #include #include +#include "iceberg/expression/literal.h" #include "iceberg/json_serde_internal.h" #include "iceberg/name_mapping.h" #include "iceberg/schema.h" @@ -39,6 +40,7 @@ #include "iceberg/type.h" #include "iceberg/util/checked_cast.h" #include "iceberg/util/error_collector.h" +#include "iceberg/util/formatter.h" // IWYU pragma: keep #include "iceberg/util/macros.h" #include "iceberg/util/string_util.h" #include "iceberg/util/type_util.h" @@ -214,11 +216,9 @@ class ApplyChangesVisitor { if (update_it != updates_.end()) { const auto& update_field = update_it->second; - return SchemaField(field_id, update_field->name(), std::move(result_type), - update_field->optional(), update_field->doc()); + return update_field->WithType(std::move(result_type)); } else if (result_type != field.type()) { - return SchemaField(field_id, field.name(), std::move(result_type), field.optional(), - field.doc()); + return field.WithType(std::move(result_type)); } else { return field; } @@ -282,6 +282,10 @@ std::vector ApplyChangesVisitor::MoveFields( return reordered; } +bool ValidateDefaultLiteral(const Literal& value) { + return !value.IsNull() && !value.IsAboveMax() && !value.IsBelowMin(); +} + } // namespace Result> UpdateSchema::Make( @@ -345,39 +349,43 @@ UpdateSchema& UpdateSchema::CaseSensitive(bool case_sensitive) { } UpdateSchema& UpdateSchema::AddColumn(std::string_view name, std::shared_ptr type, - std::string_view doc) { + std::string_view doc, + std::optional default_value) { ICEBERG_BUILDER_CHECK(!name.contains('.'), "Cannot add column with ambiguous name: {}, use " "AddColumn(parent, name, type, doc)", name); - return AddColumnInternal(std::nullopt, name, /*is_optional=*/true, std::move(type), - doc); + return AddColumnInternal(/*parent=*/std::nullopt, name, /*is_optional=*/true, + std::move(type), doc, std::move(default_value)); } UpdateSchema& UpdateSchema::AddColumn(std::optional parent, std::string_view name, std::shared_ptr type, - std::string_view doc) { + std::string_view doc, + std::optional default_value) { return AddColumnInternal(std::move(parent), name, /*is_optional=*/true, std::move(type), - doc); + doc, std::move(default_value)); } UpdateSchema& UpdateSchema::AddRequiredColumn(std::string_view name, std::shared_ptr type, - std::string_view doc) { + std::string_view doc, + std::optional default_value) { ICEBERG_BUILDER_CHECK(!name.contains('.'), "Cannot add column with ambiguous name: {}, use " "AddRequiredColumn(parent, name, type, doc)", name); return AddColumnInternal(std::nullopt, name, /*is_optional=*/false, std::move(type), - doc); + doc, std::move(default_value)); } UpdateSchema& UpdateSchema::AddRequiredColumn(std::optional parent, std::string_view name, std::shared_ptr type, - std::string_view doc) { + std::string_view doc, + std::optional default_value) { return AddColumnInternal(std::move(parent), name, /*is_optional=*/false, - std::move(type), doc); + std::move(type), doc, std::move(default_value)); } UpdateSchema& UpdateSchema::UpdateColumn(std::string_view name, @@ -399,8 +407,25 @@ UpdateSchema& UpdateSchema::UpdateColumn(std::string_view name, "Cannot change column type: {}: {} -> {}", name, field.type()->ToString(), new_type->ToString()); - updates_[field_id] = std::make_shared( - field.field_id(), field.name(), new_type, field.optional(), field.doc()); + // Defaults must follow the promoted type. + auto promote_default = [&](const std::shared_ptr& value) + -> Result> { + if (value == nullptr) { + return nullptr; + } + ICEBERG_ASSIGN_OR_RAISE(Literal promoted, + SchemaField::CastDefaultValue(*value, new_type)); + return std::make_shared(std::move(promoted)); + }; + ICEBERG_BUILDER_ASSIGN_OR_RETURN(std::shared_ptr initial_default, + promote_default(field.initial_default())); + ICEBERG_BUILDER_ASSIGN_OR_RETURN(std::shared_ptr write_default, + promote_default(field.write_default())); + + updates_[field_id] = + std::make_shared(field.WithType(new_type) + .WithInitialDefault(std::move(initial_default)) + .WithWriteDefault(std::move(write_default))); return *this; } @@ -420,9 +445,44 @@ UpdateSchema& UpdateSchema::UpdateColumnDoc(std::string_view name, return *this; } - updates_[field_id] = - std::make_shared(field.field_id(), field.name(), field.type(), - field.optional(), std::string(new_doc)); + updates_[field_id] = std::make_shared(field.WithDoc(new_doc)); + + return *this; +} + +UpdateSchema& UpdateSchema::UpdateColumnDefault(std::string_view name, + std::optional new_default) { + ICEBERG_BUILDER_ASSIGN_OR_RETURN(auto field_opt, FindFieldForUpdate(name)); + ICEBERG_BUILDER_CHECK(field_opt.has_value(), "Cannot update missing column: {}", name); + + const auto& field = field_opt->get(); + int32_t field_id = field.field_id(); + + ICEBERG_BUILDER_CHECK(!deletes_.contains(field_id), + "Cannot update a column that will be deleted: {}", field.name()); + + if (!new_default.has_value()) { + updates_[field_id] = std::make_shared(field.WithWriteDefault(nullptr)); + return *this; + } + + ICEBERG_BUILDER_CHECK(field.type()->is_primitive(), + "Invalid default value for {}: {} (must be null)", *field.type(), + *new_default); + ICEBERG_BUILDER_ASSIGN_OR_RETURN_WITH_ERROR( + Literal typed_default, + SchemaField::CastDefaultValue( + *new_default, internal::checked_pointer_cast(field.type())), + "Cannot cast default value to {}: {}", *field.type(), *new_default); + ICEBERG_BUILDER_CHECK(ValidateDefaultLiteral(typed_default), + "Cannot cast default value to {}: {}", *field.type(), + *new_default); + if (field.write_default() != nullptr && *field.write_default() == typed_default) { + return *this; + } + + updates_[field_id] = std::make_shared( + field.WithWriteDefault(std::make_shared(std::move(typed_default)))); return *this; } @@ -443,9 +503,7 @@ UpdateSchema& UpdateSchema::RenameColumn(std::string_view name, const SchemaField& base_field = update_it != updates_.end() ? *update_it->second : field; - updates_[field_id] = std::make_shared( - base_field.field_id(), std::string(new_name), base_field.type(), - base_field.optional(), base_field.doc()); + updates_[field_id] = std::make_shared(base_field.WithName(new_name)); auto it = std::ranges::find(identifier_field_names_, name); if (it != identifier_field_names_.end()) { @@ -474,9 +532,9 @@ UpdateSchema& UpdateSchema::UpdateColumnRequirementInternal(std::string_view nam return *this; } - // TODO(GuotaoYu): support added column with default value - // bool is_defaulted_add = IsAdded(name) && field.initial_default() != null; - bool is_defaulted_add = false; + // Defaulted adds can become required. + bool is_defaulted_add = added_name_to_id_.contains(CaseSensitivityAwareName(name)) && + field.initial_default() != nullptr; ICEBERG_BUILDER_CHECK(is_optional || is_defaulted_add || allow_incompatible_changes_, "Cannot change column nullability: {}: optional -> required", @@ -625,13 +683,15 @@ Result UpdateSchema::Apply() { .updated_props = std::move(updated_props)}; } -// TODO(Guotao Yu): v3 default value is not yet supported UpdateSchema& UpdateSchema::AddColumnInternal(std::optional parent, std::string_view name, bool is_optional, std::shared_ptr type, - std::string_view doc) { + std::string_view doc, + std::optional default_value) { int32_t parent_id = kTableRootId; std::string full_name; + // For map/list adds, this omits synthetic value/element path segments. + std::string short_name; if (parent.has_value()) { ICEBERG_BUILDER_CHECK(!parent->empty(), "Parent name cannot be empty"); @@ -661,13 +721,13 @@ UpdateSchema& UpdateSchema::AddColumnInternal(std::optional pa ICEBERG_BUILDER_CHECK(!deletes_.contains(parent_id), "Cannot add to a column that will be deleted: {}", *parent); - auto current_name = std::format("{}.{}", *parent, name); - ICEBERG_BUILDER_ASSIGN_OR_RETURN(auto current_field, FindField(current_name)); + short_name = std::format("{}.{}", *parent, name); + ICEBERG_BUILDER_ASSIGN_OR_RETURN(auto current_field, FindField(short_name)); ICEBERG_BUILDER_CHECK( !current_field.has_value() || deletes_.contains(current_field->get().field_id()), "Cannot add column, name already exists: {}.{}", *parent, name); - // Build full name using canonical name of parent + // Use the canonical parent name for schema updates. ICEBERG_BUILDER_ASSIGN_OR_RETURN(auto parent_name_opt, schema_->FindColumnNameById(parent_id)); ICEBERG_BUILDER_CHECK(parent_name_opt.has_value(), @@ -680,16 +740,19 @@ UpdateSchema& UpdateSchema::AddColumnInternal(std::optional pa "Cannot add column, name already exists: {}", name); full_name = std::string(name); + short_name = full_name; } ICEBERG_BUILDER_CHECK( - is_optional || allow_incompatible_changes_, + is_optional || default_value.has_value() || allow_incompatible_changes_, "Incompatible change: cannot add required column without a default value: {}", full_name); int32_t new_id = AssignNewColumnId(); added_name_to_id_[CaseSensitivityAwareName(full_name)] = new_id; + // Needed for same-update lookups by user-facing path. + added_name_to_id_[CaseSensitivityAwareName(short_name)] = new_id; if (parent_id != kTableRootId) { id_to_parent_[new_id] = parent_id; } @@ -697,11 +760,30 @@ UpdateSchema& UpdateSchema::AddColumnInternal(std::optional pa AssignFreshIdVisitor id_assigner([this]() { return AssignNewColumnId(); }); auto type_with_fresh_ids = id_assigner.Visit(type); - auto new_field = std::make_shared(new_id, std::string(name), - std::move(type_with_fresh_ids), - is_optional, std::string(doc)); + std::shared_ptr initial_default; + std::shared_ptr write_default; + if (default_value.has_value()) { + ICEBERG_BUILDER_CHECK(type_with_fresh_ids->is_primitive(), + "Invalid default value for {}: {} (must be null)", + *type_with_fresh_ids, *default_value); + ICEBERG_BUILDER_ASSIGN_OR_RETURN_WITH_ERROR( + Literal typed_default, + SchemaField::CastDefaultValue( + *default_value, + internal::checked_pointer_cast(type_with_fresh_ids)), + "Cannot cast default value to {}: {}", *type_with_fresh_ids, *default_value); + ICEBERG_BUILDER_CHECK(ValidateDefaultLiteral(typed_default), + "Cannot cast default value to {}: {}", *type_with_fresh_ids, + *default_value); + // Add-column defaults set both default kinds. + auto shared_default = std::make_shared(std::move(typed_default)); + initial_default = shared_default; + write_default = std::move(shared_default); + } - updates_[new_id] = std::move(new_field); + updates_[new_id] = std::make_shared( + new_id, name, std::move(type_with_fresh_ids), is_optional, doc, + std::move(initial_default), std::move(write_default)); parent_to_added_ids_[parent_id].push_back(new_id); return *this; diff --git a/src/iceberg/update/update_schema.h b/src/iceberg/update/update_schema.h index 2be3732a06..d0dd944ef1 100644 --- a/src/iceberg/update/update_schema.h +++ b/src/iceberg/update/update_schema.h @@ -30,6 +30,7 @@ #include #include +#include "iceberg/expression/literal.h" #include "iceberg/iceberg_export.h" #include "iceberg/result.h" #include "iceberg/type_fwd.h" @@ -41,10 +42,6 @@ namespace iceberg { /// /// When committing, these changes will be applied to the current table metadata. /// Commit conflicts will not be resolved and will result in a CommitFailed error. -/// -/// TODO(Guotao Yu): Add support for V3 default values when adding columns. Currently, all -/// added columns use null as the default value, but Iceberg V3 supports custom -/// default values for new columns. class ICEBERG_EXPORT UpdateSchema : public PendingUpdate { public: static Result> Make( @@ -78,15 +75,17 @@ class ICEBERG_EXPORT UpdateSchema : public PendingUpdate { /// If type is a nested type, its field IDs are reassigned when added to the /// existing schema. /// - /// The added column will be optional with a null default value. + /// v3+: `default_value` sets `initial-default` and `write-default`. /// /// \param name Name for the new column. /// \param type Type for the new column. /// \param doc Documentation string for the new column. + /// \param default_value Optional default value for the new column (v3+). /// \return Reference to this for method chaining. /// \note InvalidArgument will be reported if name contains ".". UpdateSchema& AddColumn(std::string_view name, std::shared_ptr type, - std::string_view doc = ""); + std::string_view doc = "", + std::optional default_value = std::nullopt); /// \brief Add a new optional column to a nested struct with documentation. /// @@ -102,22 +101,23 @@ class ICEBERG_EXPORT UpdateSchema : public PendingUpdate { /// If type is a nested type, its field IDs are reassigned when added to the /// existing schema. /// - /// The added column will be optional with a null default value. + /// v3+: `default_value` sets `initial-default` and `write-default`. /// /// \param parent Name of the parent struct to which the column will be added. /// \param name Name for the new column. /// \param type Type for the new column. /// \param doc Documentation string for the new column. + /// \param default_value Optional default value for the new column (v3+). /// \return Reference to this for method chaining. /// \note InvalidArgument will be reported if parent doesn't identify a struct. UpdateSchema& AddColumn(std::optional parent, std::string_view name, - std::shared_ptr type, std::string_view doc = ""); + std::shared_ptr type, std::string_view doc = "", + std::optional default_value = std::nullopt); /// \brief Add a new required top-level column with documentation. /// - /// Adding a required column without a default is an incompatible change that can - /// break reading older data. To suppress exceptions thrown when an incompatible - /// change is detected, call AllowIncompatibleChanges(). + /// Required adds need AllowIncompatibleChanges() unless `default_value` is set. + /// v3+: `default_value` sets `initial-default` and `write-default`. /// /// Because "." may be interpreted as a column path separator or may be used in /// field names, it is not allowed in names passed to this method. To add to nested @@ -130,16 +130,17 @@ class ICEBERG_EXPORT UpdateSchema : public PendingUpdate { /// \param name Name for the new column. /// \param type Type for the new column. /// \param doc Documentation string for the new column. + /// \param default_value Optional default value for the new column (v3+). /// \return Reference to this for method chaining. /// \note InvalidArgument will be reported if name contains ".". UpdateSchema& AddRequiredColumn(std::string_view name, std::shared_ptr type, - std::string_view doc = ""); + std::string_view doc = "", + std::optional default_value = std::nullopt); /// \brief Add a new required column to a nested struct with documentation. /// - /// Adding a required column without a default is an incompatible change that can - /// break reading older data. To suppress exceptions thrown when an incompatible - /// change is detected, call AllowIncompatibleChanges(). + /// Required adds need AllowIncompatibleChanges() unless `default_value` is set. + /// v3+: `default_value` sets `initial-default` and `write-default`. /// /// The parent name is used to find the parent using Schema::FindFieldByName(). If /// the parent name is null or empty, the new column will be added to the root as a @@ -157,11 +158,13 @@ class ICEBERG_EXPORT UpdateSchema : public PendingUpdate { /// \param name Name for the new column. /// \param type Type for the new column. /// \param doc Documentation string for the new column. + /// \param default_value Optional default value for the new column (v3+). /// \return Reference to this for method chaining. /// \note InvalidArgument will be reported if parent doesn't identify a struct. UpdateSchema& AddRequiredColumn(std::optional parent, std::string_view name, std::shared_ptr type, - std::string_view doc = ""); + std::string_view doc = "", + std::optional default_value = std::nullopt); /// \brief Rename a column in the schema. /// @@ -210,6 +213,20 @@ class ICEBERG_EXPORT UpdateSchema : public PendingUpdate { /// the column will be deleted. UpdateSchema& UpdateColumnDoc(std::string_view name, std::string_view new_doc); + /// \brief Update the `write-default` value for a column (v3+). + /// + /// The name is used to find the column to update using Schema::FindFieldByName(). + /// Only `write-default` changes; `initial-default` is fixed. + /// + /// \param name Name of the column to update the default value for. + /// \param new_default Replacement `write-default` value for the column, or + /// `std::nullopt` to clear it. + /// \return Reference to this for method chaining. + /// \note InvalidArgument will be reported if name doesn't identify a column in the + /// schema or if the column will be deleted. + UpdateSchema& UpdateColumnDefault(std::string_view name, + std::optional new_default); + /// \brief Update a column to be optional. /// /// \param name Name of the column to mark optional. @@ -364,10 +381,12 @@ class ICEBERG_EXPORT UpdateSchema : public PendingUpdate { /// \param is_optional Whether the column is optional. /// \param type Type for the new column. /// \param doc Optional documentation string. + /// \param default_value Optional default value for the new column (v3+). /// \return Reference to this for method chaining. UpdateSchema& AddColumnInternal(std::optional parent, std::string_view name, bool is_optional, - std::shared_ptr type, std::string_view doc); + std::shared_ptr type, std::string_view doc, + std::optional default_value); /// \brief Internal implementation for updating column requirement (optional/required). /// diff --git a/src/iceberg/util/type_util.cc b/src/iceberg/util/type_util.cc index 1f1e02747e..6656c5636c 100644 --- a/src/iceberg/util/type_util.cc +++ b/src/iceberg/util/type_util.cc @@ -385,9 +385,9 @@ std::shared_ptr AssignFreshIdVisitor::Visit(const StructType& type) std::vector fresh_fields; for (size_t i = 0; i < type.fields().size(); ++i) { const auto& field = type.fields()[i]; - fresh_fields.emplace_back(fresh_ids[i], std::string(field.name()), - Visit(field.type()), field.optional(), - std::string(field.doc())); + fresh_fields.emplace_back( + fresh_ids[i], std::string(field.name()), Visit(field.type()), field.optional(), + std::string(field.doc()), field.initial_default(), field.write_default()); } return std::make_shared(std::move(fresh_fields)); } @@ -397,7 +397,8 @@ std::shared_ptr AssignFreshIdVisitor::Visit(const ListType& type) cons int32_t fresh_id = next_id_(); SchemaField fresh_elem_field(fresh_id, std::string(elem_field.name()), Visit(elem_field.type()), elem_field.optional(), - std::string(elem_field.doc())); + std::string(elem_field.doc()), + elem_field.initial_default(), elem_field.write_default()); return std::make_shared(std::move(fresh_elem_field)); } @@ -410,10 +411,12 @@ std::shared_ptr AssignFreshIdVisitor::Visit(const MapType& type) const SchemaField fresh_key_field(fresh_key_id, std::string(key_field.name()), Visit(key_field.type()), key_field.optional(), - std::string(key_field.doc())); - SchemaField fresh_value_field(fresh_value_id, std::string(value_field.name()), - Visit(value_field.type()), value_field.optional(), - std::string(value_field.doc())); + std::string(key_field.doc()), key_field.initial_default(), + key_field.write_default()); + SchemaField fresh_value_field( + fresh_value_id, std::string(value_field.name()), Visit(value_field.type()), + value_field.optional(), std::string(value_field.doc()), + value_field.initial_default(), value_field.write_default()); return std::make_shared(std::move(fresh_key_field), std::move(fresh_value_field)); } From aa36d3584917deb267a47014b9556f07b8eae47e Mon Sep 17 00:00:00 2001 From: Junwang Zhao Date: Sat, 11 Jul 2026 17:42:26 +0800 Subject: [PATCH 14/23] ci: add RelWithDebInfo build and fix free-nonheap-object error (#812) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Adding a RelWithDebInfo build revealed a build error. This PR fixes the issue. ``` error: ‘void operator delete(void*, std::size_t)’ called on a pointer to an unallocated object ‘1’ [-Werror=free-nonheap-object] ``` --- .github/workflows/test.yml | 16 ++++++++++------ ci/scripts/build_iceberg.sh | 16 +++++++++++----- src/iceberg/json_serde.cc | 6 +++--- 3 files changed, 24 insertions(+), 14 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index b46bb924cc..4901e8ea95 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -39,11 +39,15 @@ env: jobs: ubuntu: if: ${{ github.event_name != 'pull_request' || github.event.pull_request.draft == false }} - name: AMD64 Ubuntu 26.04 + name: AMD64 Ubuntu 26.04 (${{ matrix.cmake_build_type }}) runs-on: ubuntu-26.04 - timeout-minutes: 30 + timeout-minutes: 60 strategy: fail-fast: false + matrix: + cmake_build_type: + - Debug + - RelWithDebInfo env: SCCACHE_DIR: ${{ github.workspace }}/.sccache SCCACHE_CACHE_SIZE: "2G" @@ -59,9 +63,9 @@ jobs: uses: actions/cache/restore@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0 with: path: ${{ github.workspace }}/.sccache - key: sccache-test-ubuntu-${{ github.run_id }} + key: sccache-test-ubuntu-${{ matrix.cmake_build_type }}-${{ github.run_id }} restore-keys: | - sccache-test-ubuntu- + sccache-test-ubuntu-${{ matrix.cmake_build_type }}- - name: Setup sccache uses: mozilla-actions/sccache-action@9e7fa8a12102821edf02ca5dbea1acd0f89a2696 # v0.0.10 - name: Build Iceberg @@ -69,7 +73,7 @@ jobs: env: CC: gcc-14 CXX: g++-14 - run: ci/scripts/build_iceberg.sh $(pwd) ON ON + run: ci/scripts/build_iceberg.sh $(pwd) ON ON OFF OFF ON ${{ matrix.cmake_build_type }} - name: Show sccache stats shell: bash run: sccache --show-stats @@ -78,7 +82,7 @@ jobs: uses: actions/cache/save@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0 with: path: ${{ github.workspace }}/.sccache - key: sccache-test-ubuntu-${{ github.run_id }} + key: sccache-test-ubuntu-${{ matrix.cmake_build_type }}-${{ github.run_id }} - name: Build Example shell: bash env: diff --git a/ci/scripts/build_iceberg.sh b/ci/scripts/build_iceberg.sh index 6af0802f6b..e00b6ca247 100755 --- a/ci/scripts/build_iceberg.sh +++ b/ci/scripts/build_iceberg.sh @@ -17,7 +17,7 @@ # specific language governing permissions and limitations # under the License. # -# Usage: build_iceberg.sh [rest_integration_tests=OFF] [sccache=OFF] [s3=OFF] [sigv4=OFF] [bundle_awssdk=ON] +# Usage: build_iceberg.sh [rest_integration_tests=OFF] [sccache=OFF] [s3=OFF] [sigv4=OFF] [bundle_awssdk=ON] [build_type=Debug] set -eux @@ -37,6 +37,12 @@ is_windows() { [[ "${OSTYPE}" == "msys" || "${OSTYPE}" == "win32" || "${OSTYPE}" == "cygwin" ]] } +if is_windows; then + build_type=${7:-Release} +else + build_type=${7:-Debug} +fi + CMAKE_ARGS=( "-G Ninja" "-DCMAKE_INSTALL_PREFIX=${CMAKE_INSTALL_PREFIX:-${ICEBERG_HOME}}" @@ -65,13 +71,13 @@ fi if is_windows; then CMAKE_ARGS+=("-DCMAKE_TOOLCHAIN_FILE=C:/vcpkg/scripts/buildsystems/vcpkg.cmake") - CMAKE_ARGS+=("-DCMAKE_BUILD_TYPE=Release") + CMAKE_ARGS+=("-DCMAKE_BUILD_TYPE=${build_type}") else # Pass an externally provided toolchain (e.g. vcpkg for the SigV4 job) if [[ -n "${CMAKE_TOOLCHAIN_FILE:-}" ]]; then CMAKE_ARGS+=("-DCMAKE_TOOLCHAIN_FILE=${CMAKE_TOOLCHAIN_FILE}") fi - CMAKE_ARGS+=("-DCMAKE_BUILD_TYPE=Debug") + CMAKE_ARGS+=("-DCMAKE_BUILD_TYPE=${build_type}") fi if [[ "${build_enable_sccache}" == "ON" ]]; then @@ -86,9 +92,9 @@ fi cmake "${CMAKE_ARGS[@]}" ${source_dir} if is_windows; then - cmake --build . --config Release --target install + cmake --build . --config "${build_type}" --target install if [[ "${run_tests}" == "ON" ]]; then - ctest --output-on-failure -C Release + ctest --output-on-failure -C "${build_type}" fi else cmake --build . --target install diff --git a/src/iceberg/json_serde.cc b/src/iceberg/json_serde.cc index 297265435a..f322996ec7 100644 --- a/src/iceberg/json_serde.cc +++ b/src/iceberg/json_serde.cc @@ -1377,9 +1377,9 @@ Result> TableMetadataFromJson(const nlohmann::jso TimePointMs{std::chrono::milliseconds(last_updated_ms)}; if (json.contains(kRefs)) { - ICEBERG_ASSIGN_OR_RAISE( - table_metadata->refs, - FromJsonMap>(json, kRefs, SnapshotRefFromJson)); + ICEBERG_ASSIGN_OR_RAISE(auto refs, FromJsonMap>( + json, kRefs, SnapshotRefFromJson)); + table_metadata->refs = std::move(refs); } else if (table_metadata->current_snapshot_id != kInvalidSnapshotId) { table_metadata->refs["main"] = std::make_unique(SnapshotRef{ .snapshot_id = table_metadata->current_snapshot_id, From 755799d1ba4ae3d63ffa01cb7152cc796ed3ef52 Mon Sep 17 00:00:00 2001 From: Junwang Zhao Date: Sun, 12 Jul 2026 00:04:02 +0800 Subject: [PATCH 15/23] refactor: use decimal high and low for Arrow scalar (#803) --- src/iceberg/arrow/literal_util.cc | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/src/iceberg/arrow/literal_util.cc b/src/iceberg/arrow/literal_util.cc index c13338199b..e8ee4d6c64 100644 --- a/src/iceberg/arrow/literal_util.cc +++ b/src/iceberg/arrow/literal_util.cc @@ -123,9 +123,7 @@ Result> ToArrowScalar(const Literal& literal, return std::make_shared<::arrow::DoubleScalar>(std::get(value)); case TypeId::kDecimal: { const auto& decimal = std::get(value); - ::arrow::Decimal128 arrow_decimal( - static_cast(decimal.value() >> 64), - static_cast(decimal.value() & ~uint64_t{0})); + ::arrow::Decimal128 arrow_decimal(decimal.high(), decimal.low()); return std::make_shared<::arrow::Decimal128Scalar>(arrow_decimal, std::move(arrow_type)); } From 263795e9288fe4421c9248888ddd16aba677b8a4 Mon Sep 17 00:00:00 2001 From: Timothy Wang Date: Sat, 11 Jul 2026 18:39:30 -0700 Subject: [PATCH 16/23] build: add benchmark build scaffolding (#825) ## Summary Adds an opt-in benchmark build path so future C++ microbenchmarks can live in-tree without affecting normal builds. Benchmarks stay disabled by default, and this PR keeps the executable intentionally small so reviewers can focus on CMake/Meson wiring and Google Benchmark integration. - Adds `ICEBERG_BUILD_BENCHMARKS` for CMake and a disabled-by-default `benchmarks` Meson feature. - Wires `src/iceberg/benchmark` into both build systems. - Adds Google Benchmark dependency setup for CMake and Meson. - Adds a tiny smoke benchmark so the new target is buildable in isolation. Part of #690. ## Stack This is PR 1 of 2. PR 2 adds the actual metrics evaluator benchmark cases on top of this scaffolding: timothyw553/iceberg-cpp#1. ## Test Plan - [x] Configured benchmark-enabled CMake build with `-DICEBERG_BUILD_BENCHMARKS=ON`. - [x] Built `metrics_evaluator_benchmark` on this branch. - [x] Ran the smoke benchmark with `--benchmark_min_time=0.01s`. - [x] Re-ran the default build and test suite on the top of the stack: 18/18 tests passed. - [ ] Meson wiring is syntax-reviewed but not locally executed because `meson` is not installed in this shell. ## Verification Commands ```bash cmake -S . -B build-bench-stack -G Ninja \ -DFETCHCONTENT_TRY_FIND_PACKAGE_MODE=NEVER \ -DCMAKE_IGNORE_PREFIX_PATH=/opt/homebrew/anaconda3 \ -DCMAKE_PREFIX_PATH='/opt/homebrew/opt/snappy;/opt/homebrew/opt/zstd' \ -DICEBERG_BUILD_BENCHMARKS=ON cmake --build build-bench-stack --target metrics_evaluator_benchmark build-bench-stack/src/iceberg/benchmark/metrics_evaluator_benchmark --benchmark_min_time=0.01s cmake --build build ctest --test-dir build --output-on-failure ``` --- .github/workflows/cpp-linter.yml | 1 + CMakeLists.txt | 1 + meson.options | 7 ++++ src/iceberg/CMakeLists.txt | 4 +++ src/iceberg/benchmark/CMakeLists.txt | 41 +++++++++++++++++++++ src/iceberg/benchmark/benchmark_smoke.cc | 46 ++++++++++++++++++++++++ src/iceberg/benchmark/meson.build | 26 ++++++++++++++ src/iceberg/meson.build | 4 +++ subprojects/google-benchmark.wrap | 30 ++++++++++++++++ 9 files changed, 160 insertions(+) create mode 100644 src/iceberg/benchmark/CMakeLists.txt create mode 100644 src/iceberg/benchmark/benchmark_smoke.cc create mode 100644 src/iceberg/benchmark/meson.build create mode 100644 subprojects/google-benchmark.wrap diff --git a/.github/workflows/cpp-linter.yml b/.github/workflows/cpp-linter.yml index 3dcf7ba4f3..44dd6e350f 100644 --- a/.github/workflows/cpp-linter.yml +++ b/.github/workflows/cpp-linter.yml @@ -77,6 +77,7 @@ jobs: -DCMAKE_C_COMPILER_LAUNCHER=sccache \ -DCMAKE_CXX_COMPILER_LAUNCHER=sccache \ -DICEBERG_BUILD_SQL_CATALOG=ON \ + -DICEBERG_BUILD_BENCHMARKS=ON \ -DICEBERG_SQL_SQLITE=ON \ -DICEBERG_SQL_POSTGRESQL=ON \ -DICEBERG_SQL_MYSQL=ON diff --git a/CMakeLists.txt b/CMakeLists.txt index 5f88235679..db54b24ed0 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -40,6 +40,7 @@ set(CMAKE_EXPORT_COMPILE_COMMANDS ON) option(ICEBERG_BUILD_STATIC "Build static library" ON) option(ICEBERG_BUILD_SHARED "Build shared library" OFF) option(ICEBERG_BUILD_TESTS "Build tests" ON) +option(ICEBERG_BUILD_BENCHMARKS "Build benchmarks" OFF) option(ICEBERG_BUILD_BUNDLE "Build the battery included library" ON) option(ICEBERG_BUILD_REST "Build rest catalog client" ON) option(ICEBERG_BUILD_REST_INTEGRATION_TESTS "Build rest catalog integration tests" OFF) diff --git a/meson.options b/meson.options index c53574889b..0448c61df5 100644 --- a/meson.options +++ b/meson.options @@ -52,3 +52,10 @@ option( ) option('tests', type: 'feature', description: 'Build tests', value: 'enabled') + +option( + 'benchmarks', + type: 'feature', + description: 'Build benchmarks', + value: 'disabled', +) diff --git a/src/iceberg/CMakeLists.txt b/src/iceberg/CMakeLists.txt index da820c26be..049a612561 100644 --- a/src/iceberg/CMakeLists.txt +++ b/src/iceberg/CMakeLists.txt @@ -365,3 +365,7 @@ endif() if(ICEBERG_BUILD_TESTS) add_subdirectory(test) endif() + +if(ICEBERG_BUILD_BENCHMARKS) + add_subdirectory(benchmark) +endif() diff --git a/src/iceberg/benchmark/CMakeLists.txt b/src/iceberg/benchmark/CMakeLists.txt new file mode 100644 index 0000000000..0227c38713 --- /dev/null +++ b/src/iceberg/benchmark/CMakeLists.txt @@ -0,0 +1,41 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +set(BENCHMARK_ENABLE_GTEST_TESTS + OFF + CACHE BOOL "" FORCE) +set(BENCHMARK_ENABLE_INSTALL + OFF + CACHE BOOL "" FORCE) +set(BENCHMARK_ENABLE_TESTING + OFF + CACHE BOOL "" FORCE) + +fetchcontent_declare(google_benchmark + GIT_REPOSITORY https://github.com/google/benchmark.git + GIT_TAG a4cf155615c63e019ae549e31703bf367df5b471 # v1.8.4 + FIND_PACKAGE_ARGS + NAMES + benchmark + CONFIG) +fetchcontent_makeavailable(google_benchmark) + +add_executable(benchmark_smoke benchmark_smoke.cc) +target_link_libraries(benchmark_smoke + PRIVATE "$,iceberg_static,iceberg_shared>" + "$,benchmark::benchmark,benchmark>" +) diff --git a/src/iceberg/benchmark/benchmark_smoke.cc b/src/iceberg/benchmark/benchmark_smoke.cc new file mode 100644 index 0000000000..abcc355506 --- /dev/null +++ b/src/iceberg/benchmark/benchmark_smoke.cc @@ -0,0 +1,46 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +#include + +namespace iceberg { +namespace { + +void BM_BenchmarkSmoke(benchmark::State& state) { + while (state.KeepRunning()) { + benchmark::DoNotOptimize(state.iterations()); + } + + state.SetItemsProcessed(state.iterations()); +} + +BENCHMARK(BM_BenchmarkSmoke); + +} // namespace +} // namespace iceberg + +int main(int argc, char** argv) { + benchmark::Initialize(&argc, argv); + if (benchmark::ReportUnrecognizedArguments(argc, argv)) { + return 1; + } + benchmark::RunSpecifiedBenchmarks(); + benchmark::Shutdown(); + return 0; +} diff --git a/src/iceberg/benchmark/meson.build b/src/iceberg/benchmark/meson.build new file mode 100644 index 0000000000..affc34e077 --- /dev/null +++ b/src/iceberg/benchmark/meson.build @@ -0,0 +1,26 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +benchmark_dep = dependency('benchmark', required: true) + +benchmark_smoke = executable( + 'benchmark_smoke', + sources: files('benchmark_smoke.cc'), + dependencies: [iceberg_dep, benchmark_dep], +) + +benchmark('benchmark_smoke', benchmark_smoke) diff --git a/src/iceberg/meson.build b/src/iceberg/meson.build index e5158a4b93..7340bfa284 100644 --- a/src/iceberg/meson.build +++ b/src/iceberg/meson.build @@ -337,3 +337,7 @@ subdir('inspect') if get_option('tests').enabled() subdir('test') endif + +if get_option('benchmarks').enabled() + subdir('benchmark') +endif diff --git a/subprojects/google-benchmark.wrap b/subprojects/google-benchmark.wrap new file mode 100644 index 0000000000..51f4e24d48 --- /dev/null +++ b/subprojects/google-benchmark.wrap @@ -0,0 +1,30 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +[wrap-file] +directory = benchmark-1.8.4 +source_url = https://github.com/google/benchmark/archive/refs/tags/v1.8.4.tar.gz +source_filename = benchmark-1.8.4.tar.gz +source_hash = 3e7059b6b11fb1bbe28e33e02519398ca94c1818874ebed18e504dc6f709be45 +source_fallback_url = https://github.com/mesonbuild/wrapdb/releases/download/google-benchmark_1.8.4-5/benchmark-1.8.4.tar.gz +patch_filename = google-benchmark_1.8.4-5_patch.zip +patch_url = https://wrapdb.mesonbuild.com/v2/google-benchmark_1.8.4-5/get_patch +patch_hash = 671ffed65f1e95e8c20edb7a06eb54476797e58169160b255f52dc71f4b83957 +wrapdb_version = 1.8.4-5 + +[provide] +dependency_names = benchmark, benchmark_main From 2191844e14f4efd3d6ee9bf5d0db835e218c1ffd Mon Sep 17 00:00:00 2001 From: Anupam Yadav Date: Sat, 11 Jul 2026 18:50:58 -0700 Subject: [PATCH 17/23] build: enable clang-tidy misc-include-cleaner for include hygiene (#820) --- .clang-tidy | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.clang-tidy b/.clang-tidy index e7e8abd33d..5b13a05515 100644 --- a/.clang-tidy +++ b/.clang-tidy @@ -21,6 +21,7 @@ Checks: | clang-analyzer-*, google-*, modernize-*, + misc-include-cleaner, readability-identifier-naming, readability-isolate-declaration, -modernize-use-nodiscard, @@ -41,5 +42,7 @@ CheckOptions: value: '_' - key: modernize-use-scoped-lock.WarnOnSingleLocks value: 'false' + - key: misc-include-cleaner.IgnoreHeaders + value: 'arrow/.*;avro/.*;aws/.*;cpr/.*;gmock/.*;gtest/.*;nanoarrow/.*;nlohmann/.*;parquet/.*;roaring/.*;spdlog/.*;sqlpp23/.*;sqlite3\.h;thrift/.*;utf8proc\.h;zlib\.h' HeaderFilterRegex: 'src/iceberg|example' From 0825f56c4f9dd535487aba292e99c053570f8d8b Mon Sep 17 00:00:00 2001 From: Abanoub Doss Date: Sat, 11 Jul 2026 21:16:48 -0500 Subject: [PATCH 18/23] ci: retry build dir cleanup on Windows (#807) ## What Retry `rm -rf` cleanup in the CMake CI helper scripts. Windows can briefly keep a just-built executable or DLL open, which can make cleanup fail even after the build and tests have passed. `build_example.sh` retries before configuring and still lets `mkdir` fail if the directory remains, so it will not build in a half-deleted tree. `build_iceberg.sh` retries after the build and does not fail an otherwise successful job if final cleanup is still blocked. --------- Co-authored-by: Abanoub Doss --- ci/scripts/build_example.sh | 13 ++++++++++++- ci/scripts/build_iceberg.sh | 14 ++++++++++++-- 2 files changed, 24 insertions(+), 3 deletions(-) diff --git a/ci/scripts/build_example.sh b/ci/scripts/build_example.sh index 41714cf32c..a71649ec35 100755 --- a/ci/scripts/build_example.sh +++ b/ci/scripts/build_example.sh @@ -23,7 +23,18 @@ source_dir=${1} build_dir=${1}/build run_example=${ICEBERG_RUN_EXAMPLE:-OFF} -rm -rf "${build_dir}" +# Clean up before configuring. If Windows still holds a just-built exe/dll +# after the retries, let mkdir fail rather than reuse a half-deleted tree. +for attempt in 1 2 3; do + if rm -rf "${build_dir}"; then + break + fi + if [[ "${attempt}" != "3" ]]; then + sleep 2 + else + echo "Failed to remove build directory after 3 attempts: ${build_dir}" >&2 + fi +done mkdir "${build_dir}" pushd ${build_dir} diff --git a/ci/scripts/build_iceberg.sh b/ci/scripts/build_iceberg.sh index e00b6ca247..55fb4e532b 100755 --- a/ci/scripts/build_iceberg.sh +++ b/ci/scripts/build_iceberg.sh @@ -105,5 +105,15 @@ fi popd -# clean up between builds -rm -rf ${build_dir} +# Clean up after the build. Windows can briefly hold a just-built exe/dll, +# so retry but do not fail an otherwise successful CI job. +for attempt in 1 2 3; do + if rm -rf "${build_dir}"; then + break + fi + if [[ "${attempt}" != "3" ]]; then + sleep 2 + else + echo "Failed to remove build directory after 3 attempts: ${build_dir}" >&2 + fi +done From fee18845357aefa570f3aa067061eb204ea33873 Mon Sep 17 00:00:00 2001 From: Abanoub Doss Date: Tue, 14 Jul 2026 00:36:12 -0500 Subject: [PATCH 19/23] ci: build Windows test leg in Debug (#806) ## What Let the CMake CI scripts use `ICEBERG_BUILD_TYPE` and default to `Debug`, including on Windows. MSVC builds with debug info now use embedded `/Z7` debug info through `CMP0141` and `CMAKE_MSVC_DEBUG_INFORMATION_FORMAT` so Debug objects remain cacheable by sccache. The build scripts also use the same single-config Ninja build path on every platform. Since these jobs pass `-G Ninja`, `CMAKE_BUILD_TYPE` controls the build and `--config Release` / `ctest -C Release` were not needed. ## Why Windows was the only main `Test` workflow leg still forcing the helper scripts through `Release`. After this change, the helper-script CMake test jobs default to `Debug` on pull requests and `main`; `ICEBERG_BUILD_TYPE` remains available if the project wants an override. Release verification and the SQL Catalog workflow's explicit Windows `Release` matrix entry are unchanged. The original Windows Debug blocker in #39 was an `IMPORTED_LOCATION` failure from a multi-config CMake path. That specific failure no longer applies because the helper scripts now use Ninja single-config builds. The remaining Debug-only blocker was MSVC rejecting a `constexpr static PartitionValues` test helper: `PartitionValues` owns a `std::vector`, and MSVC Debug's checked-iterator mode (`_ITERATOR_DEBUG_LEVEL != 0`) makes that construction fail constant evaluation. This PR constructs the empty `PartitionValues` at the call site instead. This follows the CI direction discussed in #799. ## Validation Fork validation passed: - `Test / AMD64 Windows 2025` passed in 23m48s on the pull-request run: https://github.com/abnobdoss/iceberg-cpp/actions/runs/28638474339/job/84929642907 - `SQL Catalog / AMD64 Windows 2025` passed in 4m13s: https://github.com/abnobdoss/iceberg-cpp/actions/runs/28638474303/job/84929643407 - `Meson / AMD64 Windows 2025` passed in 3m21s: https://github.com/abnobdoss/iceberg-cpp/actions/runs/28638474339/job/84929642897 The Windows Debug test job reported 45% sccache hits, 385 hits, 477 misses, and 0 errors. The first post-merge Apache `main` run will reseed Windows cache entries because the previous entries were built from Release objects. Closes #39. --------- Co-authored-by: Abanoub Doss --- .github/workflows/sql_catalog_test.yml | 2 +- ci/scripts/build_example.sh | 23 ++++++----------- ci/scripts/build_iceberg.sh | 34 +++++++++----------------- cmake_modules/IcebergSccache.cmake | 21 ++++++---------- src/iceberg/test/scan_test_base.h | 3 +-- 5 files changed, 29 insertions(+), 54 deletions(-) diff --git a/.github/workflows/sql_catalog_test.yml b/.github/workflows/sql_catalog_test.yml index 28bd331731..108ff8ae2e 100644 --- a/.github/workflows/sql_catalog_test.yml +++ b/.github/workflows/sql_catalog_test.yml @@ -58,7 +58,7 @@ jobs: cmake_extra_args: "" - title: AMD64 Windows 2025 runs-on: windows-2025 - cmake_build_type: Release + cmake_build_type: Debug cmake_extra_args: -DCMAKE_TOOLCHAIN_FILE=C:/vcpkg/scripts/buildsystems/vcpkg.cmake env: SCCACHE_DIR: ${{ github.workspace }}/.sccache diff --git a/ci/scripts/build_example.sh b/ci/scripts/build_example.sh index a71649ec35..7ccdf4a8f7 100755 --- a/ci/scripts/build_example.sh +++ b/ci/scripts/build_example.sh @@ -49,26 +49,19 @@ CMAKE_ARGS=( if is_windows; then CMAKE_ARGS+=("-DCMAKE_TOOLCHAIN_FILE=C:/vcpkg/scripts/buildsystems/vcpkg.cmake") - CMAKE_ARGS+=("-DCMAKE_BUILD_TYPE=Release") -else - CMAKE_ARGS+=("-DCMAKE_BUILD_TYPE=Debug") fi +build_type="${ICEBERG_BUILD_TYPE:-Debug}" +CMAKE_ARGS+=("-DCMAKE_BUILD_TYPE=${build_type}") + cmake "${CMAKE_ARGS[@]}" ${source_dir} -if is_windows; then - cmake --build . --config Release - if [[ "${run_example}" == "ON" ]]; then - if [[ -x ./demo_example.exe ]]; then - ./demo_example.exe +cmake --build . +if [[ "${run_example}" == "ON" ]]; then + if is_windows; then + ./demo_example.exe else - ./Release/demo_example.exe + ./demo_example fi - fi -else - cmake --build . - if [[ "${run_example}" == "ON" ]]; then - ./demo_example - fi fi popd diff --git a/ci/scripts/build_iceberg.sh b/ci/scripts/build_iceberg.sh index 55fb4e532b..6a7dc607a2 100755 --- a/ci/scripts/build_iceberg.sh +++ b/ci/scripts/build_iceberg.sh @@ -37,11 +37,7 @@ is_windows() { [[ "${OSTYPE}" == "msys" || "${OSTYPE}" == "win32" || "${OSTYPE}" == "cygwin" ]] } -if is_windows; then - build_type=${7:-Release} -else - build_type=${7:-Debug} -fi +build_type=${7:-Debug} CMAKE_ARGS=( "-G Ninja" @@ -70,16 +66,16 @@ else fi if is_windows; then - CMAKE_ARGS+=("-DCMAKE_TOOLCHAIN_FILE=C:/vcpkg/scripts/buildsystems/vcpkg.cmake") - CMAKE_ARGS+=("-DCMAKE_BUILD_TYPE=${build_type}") -else - # Pass an externally provided toolchain (e.g. vcpkg for the SigV4 job) - if [[ -n "${CMAKE_TOOLCHAIN_FILE:-}" ]]; then - CMAKE_ARGS+=("-DCMAKE_TOOLCHAIN_FILE=${CMAKE_TOOLCHAIN_FILE}") - fi - CMAKE_ARGS+=("-DCMAKE_BUILD_TYPE=${build_type}") + CMAKE_TOOLCHAIN_FILE="${CMAKE_TOOLCHAIN_FILE:-C:/vcpkg/scripts/buildsystems/vcpkg.cmake}" fi +# Pass an externally provided toolchain, or the default Windows vcpkg toolchain. +if [[ -n "${CMAKE_TOOLCHAIN_FILE:-}" ]]; then + CMAKE_ARGS+=("-DCMAKE_TOOLCHAIN_FILE=${CMAKE_TOOLCHAIN_FILE}") +fi + +CMAKE_ARGS+=("-DCMAKE_BUILD_TYPE=${build_type}") + if [[ "${build_enable_sccache}" == "ON" ]]; then CMAKE_ARGS+=("-DCMAKE_CXX_COMPILER_LAUNCHER=sccache") CMAKE_ARGS+=("-DCMAKE_C_COMPILER_LAUNCHER=sccache") @@ -91,16 +87,10 @@ if [[ -n "${ICEBERG_EXTRA_CMAKE_ARGS:-}" ]]; then fi cmake "${CMAKE_ARGS[@]}" ${source_dir} -if is_windows; then - cmake --build . --config "${build_type}" --target install - if [[ "${run_tests}" == "ON" ]]; then - ctest --output-on-failure -C "${build_type}" - fi -else - cmake --build . --target install - if [[ "${run_tests}" == "ON" ]]; then + +cmake --build . --target install +if [[ "${run_tests}" == "ON" ]]; then ctest --output-on-failure - fi fi popd diff --git a/cmake_modules/IcebergSccache.cmake b/cmake_modules/IcebergSccache.cmake index d02e65df53..a618a48629 100644 --- a/cmake_modules/IcebergSccache.cmake +++ b/cmake_modules/IcebergSccache.cmake @@ -18,19 +18,12 @@ if(MSVC_TOOLCHAIN AND "${CMAKE_CXX_COMPILER_LAUNCHER}" STREQUAL "sccache") message(STATUS "Configuring sccache for MSVC") - # Remove /Zi or /ZI - string(REGEX REPLACE "/Z[iI]" "" CMAKE_C_FLAGS_DEBUG "${CMAKE_C_FLAGS_DEBUG}") - string(REGEX REPLACE "/Z[iI]" "" CMAKE_CXX_FLAGS_DEBUG "${CMAKE_CXX_FLAGS_DEBUG}") + # Keep MSVC Debug objects cacheable by sccache without affecting Release. + set(CMAKE_MSVC_DEBUG_INFORMATION_FORMAT "$<$:Embedded>") - string(REGEX REPLACE "/Z[iI]" "" CMAKE_C_FLAGS_RELWITHDEBINFO - "${CMAKE_C_FLAGS_RELWITHDEBINFO}") - string(REGEX REPLACE "/Z[iI]" "" CMAKE_CXX_FLAGS_RELWITHDEBINFO - "${CMAKE_CXX_FLAGS_RELWITHDEBINFO}") - - # Add /Z7 - set(CMAKE_C_FLAGS_DEBUG "${CMAKE_C_FLAGS_DEBUG} /Z7") - set(CMAKE_CXX_FLAGS_DEBUG "${CMAKE_CXX_FLAGS_DEBUG} /Z7") - - set(CMAKE_C_FLAGS_RELWITHDEBINFO "${CMAKE_C_FLAGS_RELWITHDEBINFO} /Z7") - set(CMAKE_CXX_FLAGS_RELWITHDEBINFO "${CMAKE_CXX_FLAGS_RELWITHDEBINFO} /Z7") + # CMP0141 normally handles these flags; normalize any flags injected elsewhere. + foreach(flag_var CMAKE_C_FLAGS_DEBUG CMAKE_CXX_FLAGS_DEBUG CMAKE_C_FLAGS_RELWITHDEBINFO + CMAKE_CXX_FLAGS_RELWITHDEBINFO) + string(REGEX REPLACE "/Z[iI]" "/Z7" ${flag_var} "${${flag_var}}") + endforeach() endif() diff --git a/src/iceberg/test/scan_test_base.h b/src/iceberg/test/scan_test_base.h index 5bd1222e02..4e32febe28 100644 --- a/src/iceberg/test/scan_test_base.h +++ b/src/iceberg/test/scan_test_base.h @@ -221,7 +221,7 @@ class ScanTestBase : public testing::TestWithParam { std::shared_ptr spec = nullptr) { std::vector> files_with_partitions; for (const auto& path : added_files) { - files_with_partitions.emplace_back(path, kEmptyPartition); + files_with_partitions.emplace_back(path, PartitionValues{}); } return MakeAppendSnapshotWithPartitionValues(format_version, snapshot_id, parent_snapshot_id, sequence_number, @@ -350,7 +350,6 @@ class ScanTestBase : public testing::TestWithParam { private: int manifest_counter_ = 0; int manifest_list_counter_ = 0; - constexpr static PartitionValues kEmptyPartition{}; }; } // namespace iceberg From 330de84b1bfba2d79a96f0506b330cc0d8294c7f Mon Sep 17 00:00:00 2001 From: ZhaoXuan <425932451@qq.com> Date: Tue, 14 Jul 2026 22:38:26 +0800 Subject: [PATCH 20/23] feat(puffin): support deletion-vector-v1 blob read/write (#777) feat(puffin): support deletion-vector-v1 blob read/write Add end-to-end support for the `deletion-vector-v1` Puffin blob type on top of the existing PuffinReader/PuffinWriter and RoaringPositionBitmap. Encoding (puffin/deletion_vector.{h,cc}): - SerializeDeletionVectorBlob / DeserializeDeletionVectorBlob implement the DV blob framing: 4-byte big-endian length, 0xD1D33964 magic, the portable Roaring vector, and a trailing big-endian CRC-32. Reads validate the magic and checksum. - MakeDeletionVectorBlob builds a spec-compliant Blob: fields set to the row-position metadata column id, snapshot-id/sequence-number = -1, no compression, and the required referenced-data-file/cardinality properties. Writing (data/deletion_vector_writer.{h,cc}): - DeletionVectorWriter accumulates deleted positions per data file, run-length encodes each bitmap before serializing, writes one blob per data file, and produces a DataFile per blob carrying content_offset / content_size_in_bytes, referenced_data_file and record_count for manifest registration. Reading (data/delete_loader.cc): - Implement DeleteLoader::LoadDV: read the blob bytes referenced by content_offset/content_size_in_bytes, validate referenced_data_file, the 2GB limit and cardinality == record_count, then apply positions to the index. This wires DV deletes into DeleteFilter::ComputeAliveRows. Puffin writer convenience: - PuffinWriter::Make now fills a default `created-by` property when the caller does not provide one, matching the Java writer. Behavior matches the Java implementation (BaseDVFileWriter / BaseDeleteLoader / BitmapPositionDeleteIndex): magic, byte order, CRC coverage, blob fields, RLE and the read-side validations are all aligned. Tests: - DV blob framing round-trip and error cases (bad magic, corrupted CRC, truncated blob, size mismatch). - DeletionVectorWriter -> DeleteLoader write/read round trip and guard cases. - delete_loader: load DV, referenced-file filtering, cardinality mismatch, mixed DV + position-delete loading. - delete_filter: end-to-end ComputeAliveRows filtering with a real Puffin DV file over Arrow FileIO. - PuffinWriter created-by default and caller-precedence. Not included (deletion-vector upper layer, follow-up): merging with previously written DVs (compaction), the bulk delete(PositionDeleteIndex) API, and per-data-file partition/spec in a single DV file. --------- Co-authored-by: xuan.zhao Co-authored-by: Gang Wu --- .clang-tidy | 2 +- CMakeLists.txt | 8 +- cmake_modules/IcebergBuildUtils.cmake | 22 + src/iceberg/CMakeLists.txt | 2 + src/iceberg/data/data_writer.cc | 8 +- src/iceberg/data/delete_loader.cc | 57 ++- src/iceberg/data/delete_loader.h | 3 - src/iceberg/data/deletion_vector_writer.cc | 256 +++++++++++ src/iceberg/data/deletion_vector_writer.h | 83 ++++ src/iceberg/data/dv_util.cc | 69 +++ src/iceberg/data/dv_util_internal.h | 40 ++ src/iceberg/data/equality_delete_writer.cc | 10 +- src/iceberg/data/meson.build | 1 + src/iceberg/data/position_delete_writer.cc | 10 +- src/iceberg/data/writer.h | 21 +- src/iceberg/deletes/position_delete_index.cc | 135 ++++++ src/iceberg/deletes/position_delete_index.h | 31 ++ .../deletes/roaring_position_bitmap.cc | 35 +- src/iceberg/deletes/roaring_position_bitmap.h | 3 + src/iceberg/meson.build | 28 +- src/iceberg/test/CMakeLists.txt | 1 + src/iceberg/test/delete_filter_test.cc | 47 ++- src/iceberg/test/delete_loader_test.cc | 156 ++++++- .../test/deletion_vector_writer_test.cc | 396 ++++++++++++++++++ src/iceberg/test/endian_test.cc | 3 + .../test/file_scan_task_reader_test.cc | 47 +++ src/iceberg/test/meson.build | 1 + .../test/position_delete_index_test.cc | 165 ++++++++ src/iceberg/util/endian.h | 17 + src/iceberg/util/gzip_internal.cc | 4 + src/iceberg/util/string_util.cc | 6 + src/iceberg/util/string_util.h | 7 + src/iceberg/version.h.in | 2 + 33 files changed, 1608 insertions(+), 68 deletions(-) create mode 100644 src/iceberg/data/deletion_vector_writer.cc create mode 100644 src/iceberg/data/deletion_vector_writer.h create mode 100644 src/iceberg/data/dv_util.cc create mode 100644 src/iceberg/data/dv_util_internal.h create mode 100644 src/iceberg/test/deletion_vector_writer_test.cc diff --git a/.clang-tidy b/.clang-tidy index 5b13a05515..229bb0ec15 100644 --- a/.clang-tidy +++ b/.clang-tidy @@ -43,6 +43,6 @@ CheckOptions: - key: modernize-use-scoped-lock.WarnOnSingleLocks value: 'false' - key: misc-include-cleaner.IgnoreHeaders - value: 'arrow/.*;avro/.*;aws/.*;cpr/.*;gmock/.*;gtest/.*;nanoarrow/.*;nlohmann/.*;parquet/.*;roaring/.*;spdlog/.*;sqlpp23/.*;sqlite3\.h;thrift/.*;utf8proc\.h;zlib\.h' + value: 'arrow/.*;avro/.*;aws/.*;cpr/.*;gmock/.*;gtest/.*;nanoarrow/.*;nlohmann/.*;parquet/.*;roaring/.*;spdlog/.*;sqlpp23/.*;thrift/.*' HeaderFilterRegex: 'src/iceberg|example' diff --git a/CMakeLists.txt b/CMakeLists.txt index db54b24ed0..7cbd5e55e8 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -28,10 +28,9 @@ project(Iceberg DESCRIPTION "Iceberg C++ Project" LANGUAGES CXX) -set(ICEBERG_VERSION_SUFFIX "-SNAPSHOT") -set(ICEBERG_VERSION_STRING "${PROJECT_VERSION}${ICEBERG_VERSION_SUFFIX}") -configure_file("${CMAKE_CURRENT_SOURCE_DIR}/src/iceberg/version.h.in" - "${CMAKE_BINARY_DIR}/src/iceberg/version.h") +include(IcebergBuildUtils) +iceberg_configure_version_header("${CMAKE_CURRENT_SOURCE_DIR}/src/iceberg/version.h.in" + "${CMAKE_BINARY_DIR}/src/iceberg/version.h") set(CMAKE_CXX_STANDARD 23) set(CMAKE_CXX_STANDARD_REQUIRED ON) @@ -79,7 +78,6 @@ if(ICEBERG_BUILD_REST_INTEGRATION_TESTS AND WIN32) endif() include(CMakeParseArguments) -include(IcebergBuildUtils) include(IcebergSanitizer) include(IcebergSccache) include(IcebergThirdpartyToolchain) diff --git a/cmake_modules/IcebergBuildUtils.cmake b/cmake_modules/IcebergBuildUtils.cmake index da23f37a26..689ec44b7d 100644 --- a/cmake_modules/IcebergBuildUtils.cmake +++ b/cmake_modules/IcebergBuildUtils.cmake @@ -20,6 +20,28 @@ include(CMakePackageConfigHelpers) +function(iceberg_configure_version_header INPUT_FILE OUTPUT_FILE) + set(ICEBERG_VERSION_SUFFIX "-SNAPSHOT") + set(ICEBERG_VERSION_STRING "${PROJECT_VERSION}${ICEBERG_VERSION_SUFFIX}") + set(ICEBERG_GIT_COMMIT_ID "unknown") + + find_package(Git QUIET) + if(GIT_FOUND) + execute_process(COMMAND "${GIT_EXECUTABLE}" rev-parse HEAD + WORKING_DIRECTORY "${PROJECT_SOURCE_DIR}" + OUTPUT_VARIABLE ICEBERG_GIT_COMMIT_ID + RESULT_VARIABLE ICEBERG_GIT_RESULT + OUTPUT_STRIP_TRAILING_WHITESPACE ERROR_QUIET) + if(NOT ICEBERG_GIT_RESULT EQUAL 0 OR NOT ICEBERG_GIT_COMMIT_ID) + set(ICEBERG_GIT_COMMIT_ID "unknown") + endif() + endif() + + set(ICEBERG_FULL_VERSION_STRING + "Apache Iceberg-CPP ${ICEBERG_VERSION_STRING} (commit ${ICEBERG_GIT_COMMIT_ID})") + configure_file("${INPUT_FILE}" "${OUTPUT_FILE}" @ONLY) +endfunction() + function(iceberg_install_cmake_package PACKAGE_NAME EXPORT_NAME) set(CONFIG_CMAKE "${PACKAGE_NAME}-config.cmake") set(BUILT_CONFIG_CMAKE "${CMAKE_CURRENT_BINARY_DIR}/${CONFIG_CMAKE}") diff --git a/src/iceberg/CMakeLists.txt b/src/iceberg/CMakeLists.txt index 049a612561..35b7b91b5a 100644 --- a/src/iceberg/CMakeLists.txt +++ b/src/iceberg/CMakeLists.txt @@ -196,6 +196,8 @@ set(ICEBERG_DATA_SOURCES data/data_writer.cc data/delete_filter.cc data/delete_loader.cc + data/deletion_vector_writer.cc + data/dv_util.cc data/equality_delete_writer.cc data/file_scan_task_reader.cc data/position_delete_writer.cc diff --git a/src/iceberg/data/data_writer.cc b/src/iceberg/data/data_writer.cc index 4204712804..512f77cc96 100644 --- a/src/iceberg/data/data_writer.cc +++ b/src/iceberg/data/data_writer.cc @@ -20,7 +20,9 @@ #include "iceberg/data/data_writer.h" #include +#include +#include "iceberg/data/writer.h" // IWYU pragma: keep #include "iceberg/file_writer.h" #include "iceberg/manifest/manifest_entry.h" #include "iceberg/partition_spec.h" @@ -58,7 +60,7 @@ class DataWriter::Impl { return {}; } - Result Metadata() { + Result Metadata() { ICEBERG_CHECK(closed_, "Cannot get metadata before closing the writer"); ICEBERG_ASSIGN_OR_RAISE(auto metrics, writer_->metrics()); @@ -98,7 +100,7 @@ class DataWriter::Impl { options_.spec ? std::make_optional(options_.spec->spec_id()) : std::nullopt, }); - FileWriter::WriteResult result; + WriteResult result; result.data_files.push_back(std::move(data_file)); return result; } @@ -127,6 +129,6 @@ Result DataWriter::Length() const { return impl_->Length(); } Status DataWriter::Close() { return impl_->Close(); } -Result DataWriter::Metadata() { return impl_->Metadata(); } +Result DataWriter::Metadata() { return impl_->Metadata(); } } // namespace iceberg diff --git a/src/iceberg/data/delete_loader.cc b/src/iceberg/data/delete_loader.cc index 9221734012..a95fab7e62 100644 --- a/src/iceberg/data/delete_loader.cc +++ b/src/iceberg/data/delete_loader.cc @@ -19,23 +19,29 @@ #include "iceberg/data/delete_loader.h" +#include #include +#include #include #include +#include #include #include #include "iceberg/arrow/nanoarrow_status_internal.h" #include "iceberg/arrow_c_data_guard_internal.h" +#include "iceberg/data/dv_util_internal.h" #include "iceberg/deletes/position_delete_index.h" #include "iceberg/deletes/position_delete_range_consumer.h" +#include "iceberg/file_io.h" #include "iceberg/file_reader.h" #include "iceberg/manifest/manifest_entry.h" #include "iceberg/metadata_columns.h" #include "iceberg/result.h" #include "iceberg/row/arrow_array_wrapper.h" #include "iceberg/schema.h" +#include "iceberg/util/content_file_util.h" #include "iceberg/util/macros.h" #include "iceberg/util/struct_like_set.h" @@ -83,6 +89,21 @@ bool StringEquals(const ArrowArrayView* view, int64_t row_idx, std::string_view return sv.data != nullptr && std::memcmp(sv.data, target.data(), target.size()) == 0; } +Status ValidateDV(const DataFile& dv, std::string_view data_file_path) { + ICEBERG_PRECHECK(dv.content_offset.has_value(), "Invalid DV, offset cannot be null: {}", + ContentFileUtil::DVDesc(dv)); + ICEBERG_PRECHECK(dv.content_size_in_bytes.has_value(), "Invalid DV, length is null: {}", + ContentFileUtil::DVDesc(dv)); + ICEBERG_PRECHECK( + dv.content_size_in_bytes.value() <= std::numeric_limits::max(), + "Can't read DV larger than 2GB: {}", dv.content_size_in_bytes.value()); + ICEBERG_PRECHECK(dv.referenced_data_file.has_value() && + dv.referenced_data_file.value() == data_file_path, + "DV is expected to reference {}, not {}", data_file_path, + dv.referenced_data_file.value_or("")); + return {}; +} + } // namespace DeleteLoader::DeleteLoader(std::shared_ptr io) : io_(std::move(io)) {} @@ -91,7 +112,7 @@ DeleteLoader::~DeleteLoader() = default; Status DeleteLoader::LoadPositionDelete(const DataFile& file, PositionDeleteIndex& index, std::string_view data_file_path) const { - // TODO(gangwu): push down path filter to open the file. + // TODO(gangwu): Add cache hooks, worker pool, and filter pushdown. ICEBERG_ASSIGN_OR_RAISE(auto reader, OpenDeleteFile(file, PosDeleteSchema(), io_)); ICEBERG_ASSIGN_OR_RAISE(auto arrow_schema, reader->Schema()); @@ -170,30 +191,40 @@ Status DeleteLoader::LoadPositionDelete(const DataFile& file, PositionDeleteInde return reader->Close(); } -Status DeleteLoader::LoadDV(const DataFile& file, PositionDeleteIndex& index) const { - return NotSupported("Loading deletion vectors is not yet supported"); -} - Result DeleteLoader::LoadPositionDeletes( std::span> delete_files, std::string_view data_file_path) const { - PositionDeleteIndex index; + for (const auto& file : delete_files) { + ICEBERG_PRECHECK(file != nullptr, "Delete file must not be null"); + } + + if (ContentFileUtil::ContainsSingleDV(delete_files)) { + const auto& dv = delete_files.front(); + ICEBERG_RETURN_UNEXPECTED(ValidateDV(*dv, data_file_path)); + return DVUtil::ReadDV(dv, io_); + } + + std::vector> position_delete_files; for (const auto& file : delete_files) { + ICEBERG_PRECHECK(!file->IsDeletionVector(), + "Deletion vector cannot be mixed with position delete files: {}", + file->file_path); + ICEBERG_PRECHECK(file->content == DataFile::Content::kPositionDeletes, + "Expected position delete file but got content type {}", + ToString(file->content)); + if (file->referenced_data_file.has_value() && file->referenced_data_file.value() != data_file_path) { continue; } - if (file->IsDeletionVector()) { - ICEBERG_RETURN_UNEXPECTED(LoadDV(*file, index)); - continue; - } + position_delete_files.push_back(file); + } - ICEBERG_PRECHECK(file->content == DataFile::Content::kPositionDeletes, - "Expected position delete file but got content type {}", - ToString(file->content)); + PositionDeleteIndex index(position_delete_files); + for (const auto& file : position_delete_files) { ICEBERG_RETURN_UNEXPECTED(LoadPositionDelete(*file, index, data_file_path)); } diff --git a/src/iceberg/data/delete_loader.h b/src/iceberg/data/delete_loader.h index a422f06833..360914e855 100644 --- a/src/iceberg/data/delete_loader.h +++ b/src/iceberg/data/delete_loader.h @@ -70,9 +70,6 @@ class ICEBERG_DATA_EXPORT DeleteLoader { Status LoadPositionDelete(const DataFile& file, PositionDeleteIndex& index, std::string_view data_file_path) const; - /// \brief Load a single deletion vector file into the index. - Status LoadDV(const DataFile& file, PositionDeleteIndex& index) const; - std::shared_ptr io_; }; diff --git a/src/iceberg/data/deletion_vector_writer.cc b/src/iceberg/data/deletion_vector_writer.cc new file mode 100644 index 0000000000..8ff8906277 --- /dev/null +++ b/src/iceberg/data/deletion_vector_writer.cc @@ -0,0 +1,256 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +#include "iceberg/data/deletion_vector_writer.h" + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "iceberg/data/writer.h" +#include "iceberg/deletes/position_delete_index.h" +#include "iceberg/deletes/roaring_position_bitmap.h" +#include "iceberg/file_format.h" +#include "iceberg/file_io.h" // IWYU pragma: keep +#include "iceberg/manifest/manifest_entry.h" +#include "iceberg/metadata_columns.h" +#include "iceberg/partition_spec.h" +#include "iceberg/puffin/file_metadata.h" +#include "iceberg/puffin/puffin_writer.h" +#include "iceberg/result.h" +#include "iceberg/row/partition_values.h" +#include "iceberg/util/content_file_util.h" +#include "iceberg/util/macros.h" +#include "iceberg/util/string_util.h" +#include "iceberg/version.h" + +namespace iceberg { + +namespace { +constexpr std::string_view kReferencedDataFile = "referenced-data-file"; +constexpr std::string_view kCardinality = "cardinality"; +} // namespace + +class DeletionVectorWriter::Impl { + public: + explicit Impl(DeletionVectorWriterOptions options) : options_(std::move(options)) {} + + // Accumulated positions and metadata for a single referenced data file. + struct Deletes { + PositionDeleteIndex positions; + std::shared_ptr spec; + PartitionValues partition; + }; + + Deletes& DeletesFor(std::string_view referenced_data_file, + const std::shared_ptr& spec, + const PartitionValues& partition) { + auto it = deletes_by_path_.lower_bound(referenced_data_file); + if (it == deletes_by_path_.end() || + deletes_by_path_.key_comp()(referenced_data_file, it->first)) { + it = + deletes_by_path_.emplace_hint(it, std::string(referenced_data_file), Deletes{}); + it->second.spec = spec; + it->second.partition = partition; + } + return it->second; + } + + Status Delete(std::string_view referenced_data_file, int64_t pos, + const std::shared_ptr& spec, + const PartitionValues& partition) { + ICEBERG_CHECK(!closed_, "Cannot delete after writer is closed"); + ICEBERG_PRECHECK(!referenced_data_file.empty(), + "Deletion vector requires a non-empty referenced data file"); + ICEBERG_PRECHECK(spec != nullptr, "Deletion vector requires a partition spec"); + ICEBERG_PRECHECK(pos >= 0 && pos <= RoaringPositionBitmap::kMaxPosition, + "Deletion vector position out of range [0, {}]: {}", + RoaringPositionBitmap::kMaxPosition, pos); + DeletesFor(referenced_data_file, spec, partition).positions.Delete(pos); + return {}; + } + + Status Delete(std::string_view referenced_data_file, + const PositionDeleteIndex& positions, + const std::shared_ptr& spec, + const PartitionValues& partition) { + ICEBERG_CHECK(!closed_, "Cannot delete after writer is closed"); + ICEBERG_PRECHECK(!referenced_data_file.empty(), + "Deletion vector requires a non-empty referenced data file"); + ICEBERG_PRECHECK(spec != nullptr, "Deletion vector requires a partition spec"); + DeletesFor(referenced_data_file, spec, partition).positions.Merge(positions); + return {}; + } + + Status Close() { + if (closed_) { + return {}; + } + + if (deletes_by_path_.empty()) { + closed_ = true; + return {}; + } + + for (auto& [path, deletes] : deletes_by_path_) { + ICEBERG_RETURN_UNEXPECTED(LoadPreviousDeletes(path, deletes)); + } + + ICEBERG_ASSIGN_OR_RAISE(auto output_file, options_.io->NewOutputFile(options_.path)); + const std::string output_path(options_.path); + ICEBERG_ASSIGN_OR_RAISE( + auto writer, puffin::PuffinWriter::Make( + std::move(output_file), + {{std::string(puffin::StandardPuffinProperties::kCreatedBy), + ICEBERG_FULL_VERSION_STRING}})); + + for (auto& [path, deletes] : deletes_by_path_) { + ICEBERG_RETURN_UNEXPECTED(Write(*writer, path, deletes)); + } + + ICEBERG_RETURN_UNEXPECTED(writer->Finish()); + ICEBERG_ASSIGN_OR_RAISE(const int64_t file_size, writer->FileSize()); + + for (const auto& [path, _] : deletes_by_path_) { + result_.referenced_data_files.push_back(path); + ICEBERG_ASSIGN_OR_RAISE(auto data_file, CreateDV(output_path, file_size, path)); + result_.data_files.push_back(std::move(data_file)); + } + + closed_ = true; + return {}; + } + + Result Metadata() { + ICEBERG_CHECK(closed_, "Cannot get metadata before closing the writer"); + return result_; + } + + private: + Status LoadPreviousDeletes(std::string_view path, Deletes& deletes) { + ICEBERG_ASSIGN_OR_RAISE(auto previous, options_.load_previous_deletes(path)); + if (!previous.has_value()) { + return {}; + } + + deletes.positions.Merge(*previous); + for (const auto& delete_file : previous->delete_files()) { + ICEBERG_ASSIGN_OR_RAISE(bool file_scoped, + ContentFileUtil::IsFileScoped(*delete_file)); + if (file_scoped) { + result_.rewritten_delete_files.push_back(delete_file); + } + } + return {}; + } + + Status Write(puffin::PuffinWriter& writer, std::string_view path, Deletes& deletes) { + const int64_t cardinality = deletes.positions.Cardinality(); + ICEBERG_ASSIGN_OR_RAISE(auto data, deletes.positions.Serialize()); + + puffin::Blob blob{ + .type = std::string(puffin::StandardBlobTypes::kDeletionVectorV1), + .input_fields = {MetadataColumns::kFilePositionColumnId}, + // Snapshot ID and sequence number are inherited; the spec requires -1. + .snapshot_id = -1, + .sequence_number = -1, + .data = std::move(data), + .requested_compression = puffin::PuffinCompressionCodec::kNone, + }; + blob.properties.emplace(std::string(kReferencedDataFile), path); + blob.properties.emplace(std::string(kCardinality), std::format("{}", cardinality)); + + ICEBERG_ASSIGN_OR_RAISE(auto blob_metadata, writer.Write(blob)); + blobs_by_path_.insert_or_assign(std::string(path), std::move(blob_metadata)); + return {}; + } + + Result> CreateDV(const std::string& path, int64_t size, + std::string_view referenced_data_file) { + auto deletes = deletes_by_path_.find(referenced_data_file); + ICEBERG_CHECK(deletes != deletes_by_path_.end(), "Missing deletion vector for {}", + referenced_data_file); + auto blob_metadata = blobs_by_path_.find(referenced_data_file); + ICEBERG_CHECK(blob_metadata != blobs_by_path_.end(), + "Missing deletion vector blob for {}", referenced_data_file); + + return std::make_shared(DataFile{ + .content = DataFile::Content::kPositionDeletes, + .file_path = path, + .file_format = FileFormatType::kPuffin, + .partition = deletes->second.partition, + .record_count = deletes->second.positions.Cardinality(), + .file_size_in_bytes = size, + // TODO(gangwu): support encryption key metadata + .referenced_data_file = std::string(referenced_data_file), + .content_offset = blob_metadata->second.offset, + .content_size_in_bytes = blob_metadata->second.length, + .partition_spec_id = deletes->second.spec + ? std::make_optional(deletes->second.spec->spec_id()) + : std::nullopt, + }); + } + + DeletionVectorWriterOptions options_; + std::map deletes_by_path_; + std::map blobs_by_path_; + WriteResult result_; + bool closed_ = false; +}; + +DeletionVectorWriter::DeletionVectorWriter(std::unique_ptr impl) + : impl_(std::move(impl)) {} + +DeletionVectorWriter::~DeletionVectorWriter() = default; + +Result> DeletionVectorWriter::Make( + DeletionVectorWriterOptions options) { + ICEBERG_PRECHECK(!options.path.empty(), "DeletionVectorWriter requires an output path"); + ICEBERG_PRECHECK(options.io != nullptr, "DeletionVectorWriter requires a FileIO"); + ICEBERG_PRECHECK(options.load_previous_deletes != nullptr, + "DeletionVectorWriter requires a load_previous_deletes callback"); + return std::unique_ptr( + new DeletionVectorWriter(std::make_unique(std::move(options)))); +} + +Status DeletionVectorWriter::Delete(std::string_view referenced_data_file, int64_t pos, + const std::shared_ptr& spec, + const PartitionValues& partition) { + return impl_->Delete(referenced_data_file, pos, spec, partition); +} + +Status DeletionVectorWriter::Delete(std::string_view referenced_data_file, + const PositionDeleteIndex& positions, + const std::shared_ptr& spec, + const PartitionValues& partition) { + return impl_->Delete(referenced_data_file, positions, spec, partition); +} + +Status DeletionVectorWriter::Close() { return impl_->Close(); } + +Result DeletionVectorWriter::Metadata() { return impl_->Metadata(); } + +} // namespace iceberg diff --git a/src/iceberg/data/deletion_vector_writer.h b/src/iceberg/data/deletion_vector_writer.h new file mode 100644 index 0000000000..bd5bb66a5f --- /dev/null +++ b/src/iceberg/data/deletion_vector_writer.h @@ -0,0 +1,83 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +#pragma once + +/// \file iceberg/data/deletion_vector_writer.h +/// Writer that emits deletion vectors as `deletion-vector-v1` blobs in a Puffin file. + +#include +#include +#include +#include +#include +#include + +#include "iceberg/data/writer.h" +#include "iceberg/deletes/position_delete_index.h" +#include "iceberg/iceberg_data_export.h" +#include "iceberg/result.h" +#include "iceberg/row/partition_values.h" +#include "iceberg/type_fwd.h" + +namespace iceberg { + +/// \brief Options for creating a DeletionVectorWriter. +struct ICEBERG_DATA_EXPORT DeletionVectorWriterOptions { + std::string path; + std::shared_ptr io; + /// Loads existing deletes for a data file to merge, if any. + std::function>(std::string_view)> + load_previous_deletes; +}; + +/// \brief A deletion vector file writer. +class ICEBERG_DATA_EXPORT DeletionVectorWriter { + public: + ~DeletionVectorWriter(); + + /// \brief Create a new DeletionVectorWriter. + static Result> Make( + DeletionVectorWriterOptions options); + + /// \brief Mark a row position as deleted for the given data file. + Status Delete(std::string_view referenced_data_file, int64_t pos, + const std::shared_ptr& spec, + const PartitionValues& partition); + + /// \brief Mark all positions in the given index as deleted for a data file. + Status Delete(std::string_view referenced_data_file, + const PositionDeleteIndex& positions, + const std::shared_ptr& spec, + const PartitionValues& partition); + + /// \brief Write all accumulated deletion vectors to the Puffin file and close. + Status Close(); + + /// \brief The result of writing; valid only after Close(). + Result Metadata(); + + private: + class Impl; + std::unique_ptr impl_; + + explicit DeletionVectorWriter(std::unique_ptr impl); +}; + +} // namespace iceberg diff --git a/src/iceberg/data/dv_util.cc b/src/iceberg/data/dv_util.cc new file mode 100644 index 0000000000..175644841b --- /dev/null +++ b/src/iceberg/data/dv_util.cc @@ -0,0 +1,69 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +#include +#include +#include +#include +#include +#include + +#include "iceberg/data/dv_util_internal.h" +#include "iceberg/deletes/position_delete_index.h" +#include "iceberg/file_io.h" +#include "iceberg/manifest/manifest_entry.h" +#include "iceberg/result.h" +#include "iceberg/util/content_file_util.h" +#include "iceberg/util/macros.h" + +namespace iceberg { + +Result DVUtil::ReadDV(const std::shared_ptr& delete_file, + const std::shared_ptr& io) { + ICEBERG_PRECHECK(delete_file != nullptr, "Delete file must not be null"); + ICEBERG_PRECHECK(io != nullptr, "DV read requires a FileIO"); + ICEBERG_PRECHECK(ContentFileUtil::IsDV(*delete_file), + "Cannot read, not a deletion vector: {}", delete_file->file_path); + ICEBERG_PRECHECK( + delete_file->content_offset.has_value() && + delete_file->content_size_in_bytes.has_value(), + "Deletion vector requires content_offset and content_size_in_bytes: {}", + delete_file->file_path); + + const int64_t offset = delete_file->content_offset.value(); + const int64_t length = delete_file->content_size_in_bytes.value(); + ICEBERG_PRECHECK(offset >= 0 && length >= 0, + "Invalid deletion vector offset/length: offset={}, length={}", offset, + length); + ICEBERG_PRECHECK(length <= std::numeric_limits::max(), + "Cannot read deletion vector larger than 2GB: {}", length); + + ICEBERG_ASSIGN_OR_RAISE(auto input_file, io->NewInputFile(delete_file->file_path)); + ICEBERG_ASSIGN_OR_RAISE(auto stream, input_file->Open()); + + std::vector bytes(static_cast(length)); + ICEBERG_RETURN_UNEXPECTED(stream->ReadFully(offset, bytes)); + ICEBERG_RETURN_UNEXPECTED(stream->Close()); + + std::span blob(reinterpret_cast(bytes.data()), + bytes.size()); + return PositionDeleteIndex::Deserialize(blob, delete_file); +} + +} // namespace iceberg diff --git a/src/iceberg/data/dv_util_internal.h b/src/iceberg/data/dv_util_internal.h new file mode 100644 index 0000000000..4738b55bbf --- /dev/null +++ b/src/iceberg/data/dv_util_internal.h @@ -0,0 +1,40 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +#pragma once + +/// \file iceberg/data/dv_util_internal.h +/// Internal deletion vector helpers. + +#include + +#include "iceberg/deletes/position_delete_index.h" +#include "iceberg/iceberg_data_export.h" +#include "iceberg/result.h" +#include "iceberg/type_fwd.h" + +namespace iceberg { + +class ICEBERG_DATA_EXPORT DVUtil { + public: + static Result ReadDV(const std::shared_ptr& delete_file, + const std::shared_ptr& io); +}; + +} // namespace iceberg diff --git a/src/iceberg/data/equality_delete_writer.cc b/src/iceberg/data/equality_delete_writer.cc index 6604c03e48..86293addb7 100644 --- a/src/iceberg/data/equality_delete_writer.cc +++ b/src/iceberg/data/equality_delete_writer.cc @@ -20,7 +20,9 @@ #include "iceberg/data/equality_delete_writer.h" #include +#include +#include "iceberg/data/writer.h" // IWYU pragma: keep #include "iceberg/file_writer.h" #include "iceberg/manifest/manifest_entry.h" #include "iceberg/partition_spec.h" @@ -57,7 +59,7 @@ class EqualityDeleteWriter::Impl { return {}; } - Result Metadata() { + Result Metadata() { ICEBERG_CHECK(closed_, "Cannot get metadata before closing the writer"); ICEBERG_ASSIGN_OR_RAISE(auto metrics, writer_->metrics()); @@ -99,7 +101,7 @@ class EqualityDeleteWriter::Impl { options_.spec ? std::make_optional(options_.spec->spec_id()) : std::nullopt, }); - FileWriter::WriteResult result; + WriteResult result; result.data_files.push_back(std::move(data_file)); return result; } @@ -134,9 +136,7 @@ Result EqualityDeleteWriter::Length() const { return impl_->Length(); } Status EqualityDeleteWriter::Close() { return impl_->Close(); } -Result EqualityDeleteWriter::Metadata() { - return impl_->Metadata(); -} +Result EqualityDeleteWriter::Metadata() { return impl_->Metadata(); } std::span EqualityDeleteWriter::equality_field_ids() const { return impl_->equality_field_ids(); diff --git a/src/iceberg/data/meson.build b/src/iceberg/data/meson.build index bbb26db27d..eaae8a2b48 100644 --- a/src/iceberg/data/meson.build +++ b/src/iceberg/data/meson.build @@ -20,6 +20,7 @@ install_headers( 'data_writer.h', 'delete_filter.h', 'delete_loader.h', + 'deletion_vector_writer.h', 'equality_delete_writer.h', 'file_scan_task_reader.h', 'position_delete_writer.h', diff --git a/src/iceberg/data/position_delete_writer.cc b/src/iceberg/data/position_delete_writer.cc index 3238dc506d..e487e0e97c 100644 --- a/src/iceberg/data/position_delete_writer.cc +++ b/src/iceberg/data/position_delete_writer.cc @@ -21,12 +21,14 @@ #include #include +#include #include #include #include "iceberg/arrow/nanoarrow_status_internal.h" #include "iceberg/arrow_c_data_guard_internal.h" +#include "iceberg/data/writer.h" // IWYU pragma: keep #include "iceberg/file_writer.h" #include "iceberg/manifest/manifest_entry.h" #include "iceberg/metadata_columns.h" @@ -92,7 +94,7 @@ class PositionDeleteWriter::Impl { return {}; } - Result Metadata() { + Result Metadata() { ICEBERG_CHECK(closed_, "Cannot get metadata before closing the writer"); ICEBERG_ASSIGN_OR_RAISE(auto metrics, writer_->metrics()); @@ -159,7 +161,7 @@ class PositionDeleteWriter::Impl { options_.spec ? std::make_optional(options_.spec->spec_id()) : std::nullopt, }); - FileWriter::WriteResult result; + WriteResult result; result.data_files.push_back(std::move(data_file)); return result; } @@ -233,8 +235,6 @@ Result PositionDeleteWriter::Length() const { return impl_->Length(); } Status PositionDeleteWriter::Close() { return impl_->Close(); } -Result PositionDeleteWriter::Metadata() { - return impl_->Metadata(); -} +Result PositionDeleteWriter::Metadata() { return impl_->Metadata(); } } // namespace iceberg diff --git a/src/iceberg/data/writer.h b/src/iceberg/data/writer.h index a6f5f063d9..0efcdc2745 100644 --- a/src/iceberg/data/writer.h +++ b/src/iceberg/data/writer.h @@ -24,6 +24,7 @@ #include #include +#include #include #include "iceberg/arrow_c_data.h" @@ -33,6 +34,18 @@ namespace iceberg { +/// \brief File metadata for files produced by a writer. +struct ICEBERG_DATA_EXPORT WriteResult { + /// Usually a writer produces a single data or delete file. + /// Position delete writer may produce multiple file-scoped delete files. + /// In the future, multiple files can be produced if file rolling is supported. + std::vector> data_files; + /// Data files referenced by the produced delete files. + std::vector referenced_data_files; + /// Previously written delete files that were merged and should be removed. + std::vector> rewritten_delete_files; +}; + /// \brief Base interface for data file writers. class ICEBERG_DATA_EXPORT FileWriter { public: @@ -48,14 +61,6 @@ class ICEBERG_DATA_EXPORT FileWriter { /// \brief Close the writer and finalize the file. virtual Status Close() = 0; - /// \brief File metadata for all files produced by this writer. - struct ICEBERG_DATA_EXPORT WriteResult { - /// Usually a writer produces a single data or delete file. - /// Position delete writer may produce multiple file-scoped delete files. - /// In the future, multiple files can be produced if file rolling is supported. - std::vector> data_files; - }; - /// \brief Get file metadata for all files produced by this writer. /// \note This method should be called after Close(). virtual Result Metadata() = 0; diff --git a/src/iceberg/deletes/position_delete_index.cc b/src/iceberg/deletes/position_delete_index.cc index f09d0b4809..53e33e635c 100644 --- a/src/iceberg/deletes/position_delete_index.cc +++ b/src/iceberg/deletes/position_delete_index.cc @@ -19,8 +19,109 @@ #include "iceberg/deletes/position_delete_index.h" +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "iceberg/deletes/roaring_position_bitmap.h" +#include "iceberg/manifest/manifest_entry.h" +#include "iceberg/result.h" +#include "iceberg/util/endian.h" +#include "iceberg/util/macros.h" + namespace iceberg { +namespace { + +// `deletion-vector-v1` blob framing. See: +// https://iceberg.apache.org/puffin-spec/#deletion-vector-v1-blob-type +constexpr std::array kMagic = {0xD1, 0xD3, 0x39, 0x64}; +constexpr int32_t kLengthPrefixBytes = 4; +constexpr int32_t kMagicBytes = 4; +constexpr int32_t kCrcBytes = 4; + +uint32_t ComputeCrc32(std::span bytes) { + uLong crc = crc32(0L, Z_NULL, 0); + crc = crc32(crc, reinterpret_cast(bytes.data()), + static_cast(bytes.size())); + return static_cast(crc); +} + +Result ReadBitmapDataLength(std::span blob, + const DataFile& delete_file) { + ICEBERG_PRECHECK(delete_file.content_size_in_bytes.has_value(), + "Deletion vector requires content_size_in_bytes: {}", + delete_file.file_path); + ICEBERG_PRECHECK( + std::cmp_equal(blob.size(), *delete_file.content_size_in_bytes), + "Deletion vector blob size {} does not match content_size_in_bytes {}: {}", + blob.size(), *delete_file.content_size_in_bytes, delete_file.file_path); + ICEBERG_PRECHECK(blob.size() >= kLengthPrefixBytes + kMagicBytes + kCrcBytes, + "Deletion vector blob too small: {} bytes", blob.size()); + + const auto length = ReadBigEndian(blob.data()); + const size_t expected_size = + kLengthPrefixBytes + static_cast(length) + kCrcBytes; + ICEBERG_PRECHECK(length >= kMagicBytes, "Invalid deletion vector length prefix: {}", + length); + ICEBERG_PRECHECK(blob.size() == expected_size, + "Deletion vector blob size mismatch: {} bytes, expected {}", + blob.size(), expected_size); + return length; +} + +Result DeserializeBitmap(std::span blob, + int32_t length, + const DataFile& delete_file) { + const uint8_t* bitmap_data = blob.data() + kLengthPrefixBytes; + ICEBERG_PRECHECK(std::memcmp(bitmap_data, kMagic.data(), kMagic.size()) == 0, + "Invalid deletion vector magic"); + + std::string_view vector_bytes(reinterpret_cast(bitmap_data + kMagicBytes), + static_cast(length) - kMagicBytes); + ICEBERG_ASSIGN_OR_RAISE(auto bitmap, RoaringPositionBitmap::Deserialize(vector_bytes)); + ICEBERG_PRECHECK(std::cmp_equal(bitmap.Cardinality(), delete_file.record_count), + "Deletion vector cardinality {} does not match record count {}: {}", + bitmap.Cardinality(), delete_file.record_count, delete_file.file_path); + return bitmap; +} + +Status ValidateChecksum(std::span blob, int32_t length) { + const uint8_t* bitmap_data = blob.data() + kLengthPrefixBytes; + const auto stored_crc = ReadBigEndian(bitmap_data + length); + const auto actual_crc = + ComputeCrc32(std::span(bitmap_data, static_cast(length))); + ICEBERG_PRECHECK(stored_crc == actual_crc, + "Deletion vector CRC mismatch: stored {:#010x}, computed {:#010x}", + stored_crc, actual_crc); + return {}; +} + +} // namespace + +PositionDeleteIndex::PositionDeleteIndex(RoaringPositionBitmap bitmap) + : bitmap_(std::move(bitmap)) {} + +PositionDeleteIndex::PositionDeleteIndex(std::shared_ptr delete_file) { + if (delete_file != nullptr) { + delete_files_.push_back(std::move(delete_file)); + } +} + +PositionDeleteIndex::PositionDeleteIndex( + std::vector> delete_files) + : delete_files_(std::move(delete_files)) {} + void PositionDeleteIndex::Delete(int64_t pos) { bitmap_.Add(pos); } void PositionDeleteIndex::Delete(int64_t pos_start, int64_t pos_end) { @@ -37,6 +138,40 @@ int64_t PositionDeleteIndex::Cardinality() const { void PositionDeleteIndex::Merge(const PositionDeleteIndex& other) { bitmap_.Or(other.bitmap_); + delete_files_.insert(delete_files_.end(), other.delete_files_.begin(), + other.delete_files_.end()); +} + +Result> PositionDeleteIndex::Serialize() { + bitmap_.Optimize(); // run-length encode before serializing + std::vector blob(kLengthPrefixBytes); + blob.insert(blob.end(), kMagic.begin(), kMagic.end()); + ICEBERG_ASSIGN_OR_RAISE(const auto vector_size, bitmap_.SerializeTo(blob)); + + const size_t magic_and_vector_size = kMagicBytes + vector_size; + ICEBERG_PRECHECK(magic_and_vector_size <= std::numeric_limits::max(), + "Deletion vector is too large to serialize: {} bytes", + magic_and_vector_size); + + WriteBigEndian(static_cast(magic_and_vector_size), blob.data()); + const auto crc_offset = blob.size(); + blob.resize(crc_offset + kCrcBytes); + WriteBigEndian(ComputeCrc32(std::span(blob.data() + kLengthPrefixBytes, + magic_and_vector_size)), + blob.data() + crc_offset); + return blob; +} + +Result PositionDeleteIndex::Deserialize( + std::span blob, std::shared_ptr delete_file) { + ICEBERG_PRECHECK(delete_file != nullptr, + "Deletion vector requires a source delete file"); + ICEBERG_ASSIGN_OR_RAISE(const auto length, ReadBitmapDataLength(blob, *delete_file)); + ICEBERG_ASSIGN_OR_RAISE(auto bitmap, DeserializeBitmap(blob, length, *delete_file)); + ICEBERG_RETURN_UNEXPECTED(ValidateChecksum(blob, length)); + PositionDeleteIndex index(std::move(bitmap)); + index.delete_files_.push_back(std::move(delete_file)); + return index; } void PositionDeleteIndex::BulkAddForKey(int32_t key, diff --git a/src/iceberg/deletes/position_delete_index.h b/src/iceberg/deletes/position_delete_index.h index 592216ed60..cbf2db478a 100644 --- a/src/iceberg/deletes/position_delete_index.h +++ b/src/iceberg/deletes/position_delete_index.h @@ -29,6 +29,8 @@ #include "iceberg/deletes/roaring_position_bitmap.h" #include "iceberg/iceberg_data_export.h" +#include "iceberg/result.h" +#include "iceberg/type_fwd.h" namespace iceberg { @@ -40,6 +42,8 @@ namespace iceberg { class ICEBERG_DATA_EXPORT PositionDeleteIndex { public: PositionDeleteIndex() = default; + explicit PositionDeleteIndex(std::shared_ptr delete_file); + explicit PositionDeleteIndex(std::vector> delete_files); ~PositionDeleteIndex() = default; /// \brief Mark a position as deleted. @@ -66,7 +70,33 @@ class ICEBERG_DATA_EXPORT PositionDeleteIndex { /// \param other The index to merge (union operation) void Merge(const PositionDeleteIndex& other); + /// \brief The delete files whose positions were merged into this index. + /// + /// Populated by constructors and Deserialize, and preserved across Merge. + /// Callers use these to report the delete files that were rewritten when + /// replacing them with a new deletion vector. + const std::vector>& delete_files() const { + return delete_files_; + } + + /// \brief Serialize the index into a `deletion-vector-v1` blob. + /// + /// The positions are run-length encoded, then framed per the Puffin spec: + /// https://iceberg.apache.org/puffin-spec/#deletion-vector-v1-blob-type + Result> Serialize(); + + /// \brief Deserialize a `deletion-vector-v1` blob into an index. + /// + /// Validates the blob framing (length prefix, magic sequence, CRC-32) and, + /// against the source delete file, that the blob length matches + /// `content_size_in_bytes` and the bitmap cardinality matches `record_count`. + /// The source delete file is retained and exposed via delete_files(). + static Result Deserialize(std::span blob, + std::shared_ptr delete_file); + private: + explicit PositionDeleteIndex(RoaringPositionBitmap bitmap); + // Bulk-add positions sharing high-32-bit `key`. Private hook for // `ForEachPositionDelete`'s bulk path; keeps `Delete` the sole public // mutation surface. @@ -77,6 +107,7 @@ class ICEBERG_DATA_EXPORT PositionDeleteIndex { std::vector& scratch); RoaringPositionBitmap bitmap_; + std::vector> delete_files_; }; } // namespace iceberg diff --git a/src/iceberg/deletes/roaring_position_bitmap.cc b/src/iceberg/deletes/roaring_position_bitmap.cc index e31efeec18..a2827d4bb6 100644 --- a/src/iceberg/deletes/roaring_position_bitmap.cc +++ b/src/iceberg/deletes/roaring_position_bitmap.cc @@ -19,9 +19,11 @@ #include "iceberg/deletes/roaring_position_bitmap.h" +#include #include #include #include +#include #include #include @@ -57,6 +59,16 @@ Status ValidatePosition(int64_t pos) { return {}; } +void WriteBitmaps(const std::vector& bitmaps, uint8_t* buf) { + WriteLittleEndian(static_cast(bitmaps.size()), buf); + buf += kBitmapCountSizeBytes; + for (int32_t key = 0; std::cmp_less(key, bitmaps.size()); ++key) { + WriteLittleEndian(key, buf); + buf += kBitmapKeySizeBytes; + buf += bitmaps[key].write(reinterpret_cast(buf), /*portable=*/true); + } +} + } // namespace struct RoaringPositionBitmap::Impl { @@ -186,23 +198,18 @@ size_t RoaringPositionBitmap::SerializedSizeInBytes() const { Result RoaringPositionBitmap::Serialize() const { size_t size = SerializedSizeInBytes(); std::string result(size, '\0'); - char* buf = result.data(); - - // Write bitmap count (array length including empties) - WriteLittleEndian(static_cast(impl_->bitmaps.size()), buf); - buf += kBitmapCountSizeBytes; - - // Write each bitmap with its key - for (int32_t key = 0; std::cmp_less(key, impl_->bitmaps.size()); ++key) { - WriteLittleEndian(key, buf); - buf += kBitmapKeySizeBytes; - size_t written = impl_->bitmaps[key].write(buf, /*portable=*/true); - buf += written; - } - + WriteBitmaps(impl_->bitmaps, reinterpret_cast(result.data())); return result; } +Result RoaringPositionBitmap::SerializeTo(std::vector& output) const { + const size_t offset = output.size(); + const size_t size = SerializedSizeInBytes(); + output.resize(offset + size); + WriteBitmaps(impl_->bitmaps, output.data() + offset); + return size; +} + Result RoaringPositionBitmap::Deserialize(std::string_view bytes) { const char* buf = bytes.data(); size_t remaining = bytes.size(); diff --git a/src/iceberg/deletes/roaring_position_bitmap.h b/src/iceberg/deletes/roaring_position_bitmap.h index bfb7d7c9eb..50807dc6aa 100644 --- a/src/iceberg/deletes/roaring_position_bitmap.h +++ b/src/iceberg/deletes/roaring_position_bitmap.h @@ -28,6 +28,7 @@ #include #include #include +#include #include "iceberg/iceberg_data_export.h" #include "iceberg/result.h" @@ -118,6 +119,8 @@ class ICEBERG_DATA_EXPORT RoaringPositionBitmap { // `PositionDeleteIndex::BulkAddForKey`; per-key grouping is the caller's // job, keeping this a thin wrapper around CRoaring's `addMany`. void AddManyForKey(int32_t key, std::span positions); + // Appends the portable serialized bitmap and returns bytes appended. + Result SerializeTo(std::vector& output) const; friend class PositionDeleteIndex; }; diff --git a/src/iceberg/meson.build b/src/iceberg/meson.build index 7340bfa284..e3bef01afa 100644 --- a/src/iceberg/meson.build +++ b/src/iceberg/meson.build @@ -18,6 +18,7 @@ conf_data = configuration_data() version = meson.project_version() version_suffix = '-SNAPSHOT' +version_string = version + version_suffix components = version.split('.') assert( components.length() >= 3, @@ -29,7 +30,30 @@ ver_patch = components[2] conf_data.set('PROJECT_VERSION_MAJOR', ver_major) conf_data.set('PROJECT_VERSION_MINOR', ver_minor) conf_data.set('PROJECT_VERSION_PATCH', ver_patch) -conf_data.set('ICEBERG_VERSION_STRING', version + version_suffix) +conf_data.set('ICEBERG_VERSION_STRING', version_string) +git = find_program('git', required: false) +git_commit_id = 'unknown' +if git.found() + git_result = run_command( + git, + '-C', + meson.project_source_root(), + 'rev-parse', + 'HEAD', + check: false, + ) + if git_result.returncode() == 0 + git_commit_id_candidate = git_result.stdout().strip() + if git_commit_id_candidate != '' + git_commit_id = git_commit_id_candidate + endif + endif +endif +conf_data.set('ICEBERG_GIT_COMMIT_ID', git_commit_id) +conf_data.set( + 'ICEBERG_FULL_VERSION_STRING', + 'Apache Iceberg-CPP ' + version_string + ' (commit ' + git_commit_id + ')', +) conf_data.set('PROJECT_NAME', meson.project_name()) configure_file( input: 'version.h.in', @@ -174,6 +198,8 @@ iceberg_data_sources = files( 'data/data_writer.cc', 'data/delete_filter.cc', 'data/delete_loader.cc', + 'data/deletion_vector_writer.cc', + 'data/dv_util.cc', 'data/equality_delete_writer.cc', 'data/file_scan_task_reader.cc', 'data/position_delete_writer.cc', diff --git a/src/iceberg/test/CMakeLists.txt b/src/iceberg/test/CMakeLists.txt index 7a1b45b70e..98129a6d2b 100644 --- a/src/iceberg/test/CMakeLists.txt +++ b/src/iceberg/test/CMakeLists.txt @@ -254,6 +254,7 @@ if(ICEBERG_BUILD_BUNDLE) data_writer_test.cc delete_filter_test.cc delete_loader_test.cc + deletion_vector_writer_test.cc file_scan_task_reader_test.cc literal_util_test.cc) diff --git a/src/iceberg/test/delete_filter_test.cc b/src/iceberg/test/delete_filter_test.cc index 89d1b6b859..4b5e888e51 100644 --- a/src/iceberg/test/delete_filter_test.cc +++ b/src/iceberg/test/delete_filter_test.cc @@ -25,6 +25,7 @@ #include #include #include +#include #include #include @@ -34,9 +35,12 @@ #include #include "iceberg/arrow/arrow_io_internal.h" +#include "iceberg/data/deletion_vector_writer.h" #include "iceberg/data/equality_delete_writer.h" #include "iceberg/data/position_delete_writer.h" +#include "iceberg/deletes/position_delete_index.h" #include "iceberg/file_format.h" +#include "iceberg/file_io.h" #include "iceberg/file_reader.h" #include "iceberg/manifest/manifest_entry.h" #include "iceberg/metadata_columns.h" @@ -249,6 +253,25 @@ class DeleteFilterTest : public ::testing::Test { static constexpr std::string_view kDataPath = "data.parquet"; + Result> DeletionVectorFile( + const std::string& path, const std::vector& positions, + const std::string& data_path = std::string(kDataPath)) { + DeletionVectorWriterOptions options{ + .path = path, + .io = file_io_, + .load_previous_deletes = [](std::string_view) + -> Result> { return std::nullopt; }, + }; + ICEBERG_ASSIGN_OR_RAISE(auto writer, DeletionVectorWriter::Make(std::move(options))); + for (int64_t pos : positions) { + ICEBERG_RETURN_UNEXPECTED( + writer->Delete(data_path, pos, partition_spec_, PartitionValues{})); + } + ICEBERG_RETURN_UNEXPECTED(writer->Close()); + ICEBERG_ASSIGN_OR_RAISE(auto metadata, writer->Metadata()); + return metadata.data_files[0]; + } + std::shared_ptr file_io_; std::shared_ptr table_schema_; std::shared_ptr partition_spec_; @@ -1156,7 +1179,7 @@ TEST_F(DeleteFilterTest, DeletionVectorErrorPropagatesFromCompute) { ICEBERG_UNWRAP_OR_FAIL(auto batch, MakeBatch(*filter.value()->RequiredSchema(), R"([[1, 0]])")); auto alive = filter.value()->ComputeAliveRows(batch.schema, batch.array); - ASSERT_THAT(alive, IsError(ErrorKind::kNotSupported)); + ASSERT_THAT(alive, IsError(ErrorKind::kInvalidArgument)); } TEST_F(DeleteFilterTest, EmptyBatchPropagatesDeleteLoadErrors) { @@ -1175,7 +1198,27 @@ TEST_F(DeleteFilterTest, EmptyBatchPropagatesDeleteLoadErrors) { auto alive = filter.value()->ComputeAliveRows(batch.schema, batch.array); - ASSERT_THAT(alive, IsError(ErrorKind::kNotSupported)); + ASSERT_THAT(alive, IsError(ErrorKind::kInvalidArgument)); +} + +TEST_F(DeleteFilterTest, DeletionVectorComputeAliveRows) { + // Write a real deletion vector with DeletionVectorWriter, then load it through + // DeleteFilter (DeleteLoader::LoadDV) and verify deleted positions are filtered. + ICEBERG_UNWRAP_OR_FAIL(auto dv, DeletionVectorFile("dv-alive.puffin", {1, 3})); + std::vector> delete_files = {dv}; + auto requested_schema = Project({1}); + auto filter = DeleteFilter::Make(std::string(kDataPath), delete_files, table_schema_, + requested_schema, file_io_, + /*need_row_pos_col=*/true); + ASSERT_THAT(filter, IsOk()); + + ICEBERG_UNWRAP_OR_FAIL(auto batch, + MakeBatch(*filter.value()->RequiredSchema(), + R"([[10, 0], [20, 1], [30, 2], [40, 3], [50, 4]])")); + auto alive = filter.value()->ComputeAliveRows(batch.schema, batch.array); + + ASSERT_THAT(alive, IsOk()); + ExpectAliveRows(alive.value(), {0, 2, 4}); } TEST_F(DeleteFilterTest, CounterAccumulatesAcrossBatches) { diff --git a/src/iceberg/test/delete_loader_test.cc b/src/iceberg/test/delete_loader_test.cc index b392065cf0..a5dda22e99 100644 --- a/src/iceberg/test/delete_loader_test.cc +++ b/src/iceberg/test/delete_loader_test.cc @@ -19,6 +19,9 @@ #include "iceberg/data/delete_loader.h" +#include +#include + #include #include #include @@ -26,19 +29,23 @@ #include #include "iceberg/arrow/arrow_io_internal.h" +#include "iceberg/data/deletion_vector_writer.h" #include "iceberg/data/equality_delete_writer.h" #include "iceberg/data/position_delete_writer.h" #include "iceberg/deletes/position_delete_index.h" #include "iceberg/file_format.h" +#include "iceberg/file_io.h" #include "iceberg/manifest/manifest_entry.h" #include "iceberg/parquet/parquet_register.h" #include "iceberg/partition_spec.h" +#include "iceberg/result.h" #include "iceberg/row/partition_values.h" #include "iceberg/schema.h" #include "iceberg/schema_field.h" #include "iceberg/schema_internal.h" #include "iceberg/test/matchers.h" #include "iceberg/type.h" +#include "iceberg/util/macros.h" #include "iceberg/util/struct_like_set.h" namespace iceberg { @@ -110,6 +117,27 @@ class DeleteLoaderTest : public ::testing::Test { return writer->Metadata().value().data_files[0]; } + /// Write a deletion vector to a Puffin file and return the DataFile metadata. + std::shared_ptr WriteDeletionVector(const std::string& path, + const std::string& referenced_data_file, + const std::vector& positions) { + auto writer = + DeletionVectorWriter::Make(DeletionVectorWriterOptions{ + .path = path, + .io = file_io_, + .load_previous_deletes = [](std::string_view) + -> Result> { + return std::nullopt; + }}) + .value(); + for (int64_t pos : positions) { + ICEBERG_THROW_NOT_OK( + writer->Delete(referenced_data_file, pos, partition_spec_, PartitionValues{})); + } + ICEBERG_THROW_NOT_OK(writer->Close()); + return writer->Metadata().value().data_files[0]; + } + std::shared_ptr file_io_; std::shared_ptr schema_; std::shared_ptr partition_spec_; @@ -169,8 +197,7 @@ TEST_F(DeleteLoaderTest, LoadPositionDeletesFiltersByDataFilePath) { {"data_b.parquet", 10}, {"data_b.parquet", 20}}); - // Mixed paths -> writer must NOT set the hint, forcing the loader's - // per-row filter path. Locks the routing in case the writer behavior changes. + // Mixed paths keep the loader on the per-row filter path. ASSERT_FALSE(delete_file->referenced_data_file.has_value()); std::vector> files = {delete_file}; @@ -197,6 +224,46 @@ TEST_F(DeleteLoaderTest, LoadPositionDeletesFiltersByDataFilePath) { ASSERT_TRUE(result_none.value().IsEmpty()); } +// The loaded index keeps the source Parquet position delete file, so a DV +// writer can report file-scoped deletes as rewritten. +TEST_F(DeleteLoaderTest, LoadPositionDeletesTracksSourceDeleteFile) { + // Single referenced data file -> writer sets the hint -> fast path. + auto file_scoped = WritePositionDeletes("pos_src_scoped.parquet", + {{"data.parquet", 0}, {"data.parquet", 5}}); + ASSERT_TRUE(file_scoped->referenced_data_file.has_value()); + + std::vector> files = {file_scoped}; + auto result = loader_->LoadPositionDeletes(files, "data.parquet"); + ASSERT_THAT(result, IsOk()); + ASSERT_EQ(result.value().delete_files().size(), 1u); + EXPECT_EQ(result.value().delete_files()[0], file_scoped); +} + +TEST_F(DeleteLoaderTest, LoadPositionDeletesTracksSourceForFilterPath) { + auto mixed = WritePositionDeletes("pos_src_mixed.parquet", + {{"data_a.parquet", 0}, {"data_b.parquet", 10}}); + ASSERT_FALSE(mixed->referenced_data_file.has_value()); + + std::vector> files = {mixed}; + + auto matched = loader_->LoadPositionDeletes(files, "data_a.parquet"); + ASSERT_THAT(matched, IsOk()); + ASSERT_EQ(matched.value().delete_files().size(), 1u); + EXPECT_EQ(matched.value().delete_files()[0], mixed); + + auto unmatched = loader_->LoadPositionDeletes(files, "data_c.parquet"); + ASSERT_THAT(unmatched, IsOk()); + ASSERT_TRUE(unmatched.value().IsEmpty()); + ASSERT_EQ(unmatched.value().delete_files().size(), 1u); + EXPECT_EQ(unmatched.value().delete_files()[0], mixed); +} + +TEST_F(DeleteLoaderTest, LoadPositionDeletesRejectsNullFile) { + std::vector> files = {nullptr}; + auto result = loader_->LoadPositionDeletes(files, "data.parquet"); + ASSERT_THAT(result, IsError(ErrorKind::kInvalidArgument)); +} + TEST_F(DeleteLoaderTest, LoadPositionDeletesSkipsMismatchedReferencedDataFile) { auto delete_file = std::make_shared(DataFile{ .content = DataFile::Content::kPositionDeletes, @@ -212,10 +279,7 @@ TEST_F(DeleteLoaderTest, LoadPositionDeletesSkipsMismatchedReferencedDataFile) { } TEST_F(DeleteLoaderTest, LoadPositionDeletesFastPathHonorsReferencedDataFile) { - // Single-file writes -> writer sets referenced_data_file -> loader takes - // the zero-copy fast path. Sized above the consumer's 64-element sniff - // threshold so the dispatcher's real coalesce/bulk logic runs end-to-end, - // not just the small-input shortcut covered by LoadPositionDeletesSingleFile. + // Large enough to exercise the bulk path. constexpr int64_t kRowCount = 128; std::vector> deletes; deletes.reserve(kRowCount); @@ -239,15 +303,91 @@ TEST_F(DeleteLoaderTest, LoadPositionDeletesFastPathHonorsReferencedDataFile) { ASSERT_FALSE(index.IsDeleted(kRowCount)); } -TEST_F(DeleteLoaderTest, LoadPositionDeletesRejectsDV) { +TEST_F(DeleteLoaderTest, LoadDeletionVector) { + auto dv_file = + WriteDeletionVector("dv-a.puffin", "data.parquet", {0, 5, 10, 4'000'000'000LL}); + + std::vector> files = {dv_file}; + auto result = loader_->LoadPositionDeletes(files, "data.parquet"); + ASSERT_THAT(result, IsOk()); + + auto& index = result.value(); + ASSERT_EQ(index.Cardinality(), 4); + ASSERT_TRUE(index.IsDeleted(0)); + ASSERT_TRUE(index.IsDeleted(5)); + ASSERT_TRUE(index.IsDeleted(10)); + ASSERT_TRUE(index.IsDeleted(4'000'000'000LL)); + ASSERT_FALSE(index.IsDeleted(1)); +} + +TEST_F(DeleteLoaderTest, LoadDeletionVectorRejectsMismatchedReferencedDataFile) { + // A DV referencing another data file must not be silently skipped: the single + // DV applies to its referenced file, so a mismatch is bad grouping/metadata. + auto dv_file = WriteDeletionVector("dv-b.puffin", "other-data.parquet", {1, 2, 3}); + + std::vector> files = {dv_file}; + auto result = loader_->LoadPositionDeletes(files, "data.parquet"); + ASSERT_THAT(result, IsError(ErrorKind::kInvalidArgument)); +} + +TEST_F(DeleteLoaderTest, LoadDeletionVectorRequiresContentOffsetAndSize) { auto dv_file = std::make_shared(DataFile{ .content = DataFile::Content::kPositionDeletes, .file_path = "dv.puffin", .file_format = FileFormatType::kPuffin, + .referenced_data_file = "data.parquet", }); std::vector> files = {dv_file}; auto result = loader_->LoadPositionDeletes(files, "data.parquet"); - ASSERT_THAT(result, IsError(ErrorKind::kNotSupported)); + ASSERT_THAT(result, IsError(ErrorKind::kInvalidArgument)); +} + +TEST_F(DeleteLoaderTest, LoadDeletionVectorRejectsCardinalityMismatch) { + auto dv_file = WriteDeletionVector("dv-card.puffin", "data.parquet", {0, 1, 2}); + // Corrupt the recorded cardinality so it no longer matches the bitmap. + dv_file->record_count = 99; + + std::vector> files = {dv_file}; + auto result = loader_->LoadPositionDeletes(files, "data.parquet"); + ASSERT_THAT(result, IsError(ErrorKind::kInvalidArgument)); +} + +// End to end: a previously written Parquet position delete is loaded, merged +// into a new DV, and reported in rewritten_delete_files when it is file-scoped. +TEST_F(DeleteLoaderTest, DVWriterReportsFileScopedPositionDeleteAsRewritten) { + // A file-scoped position delete (single referenced data file). + auto previous = WritePositionDeletes("prev_pos.parquet", + {{"data.parquet", 100}, {"data.parquet", 200}}); + ASSERT_TRUE(previous->referenced_data_file.has_value()); + + auto* loader = loader_.get(); + ICEBERG_UNWRAP_OR_FAIL( + auto writer, + DeletionVectorWriter::Make(DeletionVectorWriterOptions{ + .path = "dv-replace.puffin", + .io = file_io_, + .load_previous_deletes = + [&](std::string_view path) -> Result> { + std::vector> files = {previous}; + ICEBERG_ASSIGN_OR_RAISE(auto index, loader->LoadPositionDeletes(files, path)); + if (index.IsEmpty()) { + return std::nullopt; + } + return std::optional(std::move(index)); + }, + })); + + ICEBERG_THROW_NOT_OK( + writer->Delete("data.parquet", 0, partition_spec_, PartitionValues{})); + ICEBERG_THROW_NOT_OK(writer->Close()); + + ICEBERG_UNWRAP_OR_FAIL(auto result, writer->Metadata()); + ASSERT_EQ(result.data_files.size(), 1u); + // New position plus the two previous positions. + EXPECT_EQ(result.data_files[0]->record_count, 3); + // The previous Parquet position delete is file-scoped and must be reported. + ASSERT_EQ(result.rewritten_delete_files.size(), 1u); + EXPECT_EQ(result.rewritten_delete_files[0], previous); } TEST_F(DeleteLoaderTest, LoadPositionDeletesRejectsWrongContent) { diff --git a/src/iceberg/test/deletion_vector_writer_test.cc b/src/iceberg/test/deletion_vector_writer_test.cc new file mode 100644 index 0000000000..605b2d4e1a --- /dev/null +++ b/src/iceberg/test/deletion_vector_writer_test.cc @@ -0,0 +1,396 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +#include "iceberg/data/deletion_vector_writer.h" + +#include +#include +#include +#include +#include +#include +#include +#include + +#include + +#include "iceberg/data/delete_loader.h" +#include "iceberg/deletes/position_delete_index.h" +#include "iceberg/deletes/roaring_position_bitmap.h" +#include "iceberg/file_format.h" +#include "iceberg/manifest/manifest_entry.h" +#include "iceberg/metadata_columns.h" +#include "iceberg/partition_spec.h" +#include "iceberg/puffin/file_metadata.h" +#include "iceberg/puffin/puffin_reader.h" +#include "iceberg/result.h" +#include "iceberg/row/partition_values.h" +#include "iceberg/test/matchers.h" +#include "iceberg/test/mock_io.h" +#include "iceberg/util/macros.h" +#include "iceberg/version.h" + +namespace iceberg { + +namespace { + +std::shared_ptr FindByReferencedFile( + const std::vector>& files, const std::string& ref) { + for (const auto& file : files) { + if (file->referenced_data_file == ref) { + return file; + } + } + return nullptr; +} + +std::shared_ptr UnpartitionedSpec() { + return PartitionSpec::Unpartitioned(); +} + +const puffin::BlobMetadata* FindBlobByReferencedFile( + const std::vector& blobs, const std::string& ref) { + for (const auto& blob : blobs) { + auto it = blob.properties.find("referenced-data-file"); + if (it != blob.properties.end() && it->second == ref) { + return &blob; + } + } + return nullptr; +} + +// A load_previous_deletes callback for data files that have no existing deletes. +Result> NoPreviousDeletes(std::string_view) { + return std::nullopt; +} + +DeletionVectorWriterOptions MakeDVWriterOptions( + std::shared_ptr io, std::string path, + std::function>(std::string_view)> + load_previous_deletes = NoPreviousDeletes) { + return DeletionVectorWriterOptions{ + .path = std::move(path), + .io = std::move(io), + .load_previous_deletes = std::move(load_previous_deletes), + }; +} + +} // namespace + +TEST(DeletionVectorWriterTest, WriteThenLoadEndToEnd) { + auto io = std::make_shared(); + auto spec = UnpartitionedSpec(); + + std::vector> delete_files; + { + ICEBERG_UNWRAP_OR_FAIL(auto writer, DeletionVectorWriter::Make(MakeDVWriterOptions( + io, "memory://deletes.puffin"))); + + ASSERT_THAT(writer->Delete("data-a.parquet", 0, spec, PartitionValues{}), IsOk()); + ASSERT_THAT(writer->Delete("data-a.parquet", 5, spec, PartitionValues{}), IsOk()); + ASSERT_THAT(writer->Delete("data-a.parquet", 10, spec, PartitionValues{}), IsOk()); + ASSERT_THAT(writer->Delete("data-b.parquet", 1, spec, PartitionValues{}), IsOk()); + ASSERT_THAT(writer->Delete("data-b.parquet", 2, spec, PartitionValues{}), IsOk()); + ASSERT_THAT(writer->Close(), IsOk()); + + ICEBERG_UNWRAP_OR_FAIL(auto result, writer->Metadata()); + delete_files = result.data_files; + // Each referenced data file is reported once. + EXPECT_EQ(result.referenced_data_files.size(), 2u); + // No previous deletes were loaded, so nothing was rewritten. + EXPECT_TRUE(result.rewritten_delete_files.empty()); + } + + // One DataFile per referenced data file. + ASSERT_EQ(delete_files.size(), 2u); + + auto dv_a = FindByReferencedFile(delete_files, "data-a.parquet"); + auto dv_b = FindByReferencedFile(delete_files, "data-b.parquet"); + ASSERT_NE(dv_a, nullptr); + ASSERT_NE(dv_b, nullptr); + + EXPECT_EQ(dv_a->content, DataFile::Content::kPositionDeletes); + EXPECT_EQ(dv_a->file_format, FileFormatType::kPuffin); + EXPECT_TRUE(dv_a->IsDeletionVector()); + EXPECT_EQ(dv_a->file_path, "memory://deletes.puffin"); + EXPECT_EQ(dv_a->record_count, 3); + EXPECT_TRUE(dv_a->content_offset.has_value()); + EXPECT_TRUE(dv_a->content_size_in_bytes.has_value()); + EXPECT_GT(dv_a->file_size_in_bytes, 0); + EXPECT_EQ(dv_a->partition_spec_id, spec->spec_id()); + EXPECT_EQ(dv_b->record_count, 2); + + // Both blobs live in the same Puffin file but at different offsets. + EXPECT_EQ(dv_a->file_path, dv_b->file_path); + EXPECT_NE(dv_a->content_offset.value(), dv_b->content_offset.value()); + + ICEBERG_UNWRAP_OR_FAIL(auto input_file, io->NewInputFile("memory://deletes.puffin")); + ICEBERG_UNWRAP_OR_FAIL(auto puffin_reader, + puffin::PuffinReader::Make(std::move(input_file))); + ICEBERG_UNWRAP_OR_FAIL(auto puffin_metadata, puffin_reader->ReadFileMetadata()); + ASSERT_EQ(puffin_metadata.blobs.size(), 2u); + EXPECT_EQ(puffin_metadata.properties.at("created-by"), ICEBERG_FULL_VERSION_STRING); + + auto assert_blob = [](const puffin::BlobMetadata& blob, const DataFile& file, + const std::string& ref, const std::string& cardinality) { + EXPECT_EQ(blob.type, puffin::StandardBlobTypes::kDeletionVectorV1); + EXPECT_EQ(blob.input_fields, + std::vector{MetadataColumns::kFilePositionColumnId}); + EXPECT_EQ(blob.snapshot_id, -1); + EXPECT_EQ(blob.sequence_number, -1); + EXPECT_TRUE(blob.compression_codec.empty()); + EXPECT_EQ(blob.offset, file.content_offset.value()); + EXPECT_EQ(blob.length, file.content_size_in_bytes.value()); + EXPECT_EQ(blob.properties.at("referenced-data-file"), ref); + EXPECT_EQ(blob.properties.at("cardinality"), cardinality); + }; + ASSERT_NE(FindBlobByReferencedFile(puffin_metadata.blobs, "data-a.parquet"), nullptr); + ASSERT_NE(FindBlobByReferencedFile(puffin_metadata.blobs, "data-b.parquet"), nullptr); + assert_blob(*FindBlobByReferencedFile(puffin_metadata.blobs, "data-a.parquet"), *dv_a, + "data-a.parquet", "3"); + assert_blob(*FindBlobByReferencedFile(puffin_metadata.blobs, "data-b.parquet"), *dv_b, + "data-b.parquet", "2"); + + // Load each data file's DV separately. + DeleteLoader loader(io); + { + auto result = loader.LoadPositionDeletes({&dv_a, 1}, "data-a.parquet"); + ASSERT_THAT(result, IsOk()); + auto& index = result.value(); + EXPECT_EQ(index.Cardinality(), 3); + EXPECT_TRUE(index.IsDeleted(0)); + EXPECT_TRUE(index.IsDeleted(5)); + EXPECT_TRUE(index.IsDeleted(10)); + EXPECT_FALSE(index.IsDeleted(1)); + } + + { + auto result = loader.LoadPositionDeletes({&dv_b, 1}, "data-b.parquet"); + ASSERT_THAT(result, IsOk()); + auto& index = result.value(); + EXPECT_EQ(index.Cardinality(), 2); + EXPECT_TRUE(index.IsDeleted(1)); + EXPECT_TRUE(index.IsDeleted(2)); + EXPECT_FALSE(index.IsDeleted(0)); + } +} + +// The PositionDeleteIndex overload bulk-adds positions for a data file. +TEST(DeletionVectorWriterTest, DeleteFromIndex) { + auto io = std::make_shared(); + auto spec = UnpartitionedSpec(); + + PositionDeleteIndex positions; + positions.Delete(0); + positions.Delete(3, 6); // [3, 6) -> 3, 4, 5 + + ICEBERG_UNWRAP_OR_FAIL(auto writer, DeletionVectorWriter::Make(MakeDVWriterOptions( + io, "memory://from-index.puffin"))); + ASSERT_THAT(writer->Delete("data.parquet", positions, spec, PartitionValues{}), IsOk()); + ASSERT_THAT(writer->Close(), IsOk()); + + ICEBERG_UNWRAP_OR_FAIL(auto result, writer->Metadata()); + ASSERT_EQ(result.data_files.size(), 1u); + EXPECT_EQ(result.data_files[0]->record_count, 4); + + DeleteLoader loader(io); + auto loaded = loader.LoadPositionDeletes(result.data_files, "data.parquet"); + ASSERT_THAT(loaded, IsOk()); + EXPECT_EQ(loaded.value().Cardinality(), 4); + EXPECT_TRUE(loaded.value().IsDeleted(0)); + EXPECT_TRUE(loaded.value().IsDeleted(5)); + EXPECT_FALSE(loaded.value().IsDeleted(6)); +} + +// Previously written deletes are merged into the new vector, and the file-scoped +// delete files they came from are reported as rewritten. +TEST(DeletionVectorWriterTest, LoadPreviousDeletesMergesAndReportsRewritten) { + auto io = std::make_shared(); + auto spec = UnpartitionedSpec(); + + // Build a previous DV index carrying its source (file-scoped) delete file, + // the same way DeleteLoader would produce it. + PositionDeleteIndex previous_positions; + previous_positions.Delete(100); + previous_positions.Delete(200); + ICEBERG_UNWRAP_OR_FAIL(auto previous_blob, previous_positions.Serialize()); + auto previous_dv = std::make_shared(DataFile{ + .content = DataFile::Content::kPositionDeletes, + .file_path = "memory://old.puffin", + .file_format = FileFormatType::kPuffin, + .record_count = 2, + .referenced_data_file = "data.parquet", + .content_offset = 0, + .content_size_in_bytes = static_cast(previous_blob.size()), + }); + + ICEBERG_UNWRAP_OR_FAIL( + auto writer, + DeletionVectorWriter::Make(MakeDVWriterOptions( + io, "memory://merged.puffin", + [&](std::string_view path) -> Result> { + if (path != "data.parquet") { + return std::nullopt; + } + ICEBERG_ASSIGN_OR_RAISE( + auto index, PositionDeleteIndex::Deserialize(previous_blob, previous_dv)); + return std::optional(std::move(index)); + }))); + + ASSERT_THAT(writer->Delete("data.parquet", 0, spec, PartitionValues{}), IsOk()); + ASSERT_THAT(writer->Close(), IsOk()); + + ICEBERG_UNWRAP_OR_FAIL(auto result, writer->Metadata()); + ASSERT_EQ(result.data_files.size(), 1u); + // New position plus the two previous positions. + EXPECT_EQ(result.data_files[0]->record_count, 3); + // The previous DV is file-scoped, so it is reported for removal. + ASSERT_EQ(result.rewritten_delete_files.size(), 1u); + EXPECT_EQ(result.rewritten_delete_files[0]->file_path, "memory://old.puffin"); + + DeleteLoader loader(io); + auto loaded = loader.LoadPositionDeletes(result.data_files, "data.parquet"); + ASSERT_THAT(loaded, IsOk()); + EXPECT_EQ(loaded.value().Cardinality(), 3); + EXPECT_TRUE(loaded.value().IsDeleted(0)); + EXPECT_TRUE(loaded.value().IsDeleted(100)); + EXPECT_TRUE(loaded.value().IsDeleted(200)); +} + +TEST(DeletionVectorWriterTest, PreviousDeletesWithoutSourceFilesAreNotRewritten) { + auto io = std::make_shared(); + auto spec = UnpartitionedSpec(); + + ICEBERG_UNWRAP_OR_FAIL( + auto writer, + DeletionVectorWriter::Make(MakeDVWriterOptions( + io, "memory://merged-partition.puffin", + [&](std::string_view path) -> Result> { + PositionDeleteIndex index; + index.Delete(50); + return std::optional(std::move(index)); + }))); + + ASSERT_THAT(writer->Delete("data.parquet", 0, spec, PartitionValues{}), IsOk()); + ASSERT_THAT(writer->Close(), IsOk()); + + ICEBERG_UNWRAP_OR_FAIL(auto result, writer->Metadata()); + ASSERT_EQ(result.data_files.size(), 1u); + // The previous position was merged in. + EXPECT_EQ(result.data_files[0]->record_count, 2); + // The previous delete is partition-scoped, so it is not rewritten. + EXPECT_TRUE(result.rewritten_delete_files.empty()); + + DeleteLoader loader(io); + auto loaded = loader.LoadPositionDeletes(result.data_files, "data.parquet"); + ASSERT_THAT(loaded, IsOk()); + EXPECT_EQ(loaded.value().Cardinality(), 2); + EXPECT_TRUE(loaded.value().IsDeleted(0)); + EXPECT_TRUE(loaded.value().IsDeleted(50)); +} + +TEST(DeletionVectorWriterTest, EmptyWriterProducesNoDataFiles) { + auto io = std::make_shared(); + ICEBERG_UNWRAP_OR_FAIL(auto writer, + DeletionVectorWriter::Make(DeletionVectorWriterOptions{ + .path = "memory://empty.puffin", + .io = io, + .load_previous_deletes = NoPreviousDeletes})); + ASSERT_THAT(writer->Close(), IsOk()); + ICEBERG_UNWRAP_OR_FAIL(auto result, writer->Metadata()); + EXPECT_TRUE(result.data_files.empty()); + EXPECT_THAT(io->NewInputFile("memory://empty.puffin"), IsError(ErrorKind::kNotFound)); +} + +TEST(DeletionVectorWriterTest, DeleteRejectsEmptyReferencedFile) { + auto io = std::make_shared(); + auto spec = UnpartitionedSpec(); + ICEBERG_UNWRAP_OR_FAIL(auto writer, DeletionVectorWriter::Make(MakeDVWriterOptions( + io, "memory://invalid.puffin"))); + EXPECT_THAT(writer->Delete("", 0, spec, PartitionValues{}), + IsError(ErrorKind::kInvalidArgument)); + EXPECT_THAT(writer->Delete("data.parquet", 0, nullptr, PartitionValues{}), + IsError(ErrorKind::kInvalidArgument)); + PositionDeleteIndex positions; + positions.Delete(0); + EXPECT_THAT(writer->Delete("data.parquet", positions, nullptr, PartitionValues{}), + IsError(ErrorKind::kInvalidArgument)); +} + +TEST(DeletionVectorWriterTest, DeleteRejectsOutOfRangePosition) { + auto io = std::make_shared(); + auto spec = UnpartitionedSpec(); + ICEBERG_UNWRAP_OR_FAIL(auto writer, DeletionVectorWriter::Make(MakeDVWriterOptions( + io, "memory://invalid.puffin"))); + // Negative and out-of-range positions are rejected rather than silently + // dropped by the underlying bitmap. + EXPECT_THAT(writer->Delete("data.parquet", -1, spec, PartitionValues{}), + IsError(ErrorKind::kInvalidArgument)); + EXPECT_THAT(writer->Delete("data.parquet", RoaringPositionBitmap::kMaxPosition + 1, + spec, PartitionValues{}), + IsError(ErrorKind::kInvalidArgument)); +} + +// Close propagates a load_previous_deletes failure and returns no metadata. +TEST(DeletionVectorWriterTest, ClosePropagatesLoadPreviousDeletesError) { + auto io = std::make_shared(); + auto spec = UnpartitionedSpec(); + ICEBERG_UNWRAP_OR_FAIL( + auto writer, + DeletionVectorWriter::Make(MakeDVWriterOptions( + io, "memory://err.puffin", + [](std::string_view) -> Result> { + return IOError("boom"); + }))); + + ASSERT_THAT(writer->Delete("data.parquet", 0, spec, PartitionValues{}), IsOk()); + EXPECT_THAT(writer->Close(), IsError(ErrorKind::kIOError)); + // No partial delete metadata is exposed after a failed Close. + EXPECT_THAT(writer->Metadata(), IsError(ErrorKind::kValidationFailed)); + // No orphan Puffin file was created. + EXPECT_THAT(io->NewInputFile("memory://err.puffin"), IsError(ErrorKind::kNotFound)); +} + +TEST(DeletionVectorWriterTest, MakeRejectsMissingOutputPath) { + EXPECT_THAT(DeletionVectorWriter::Make(DeletionVectorWriterOptions{ + .io = std::make_shared(), + .load_previous_deletes = NoPreviousDeletes}), + IsError(ErrorKind::kInvalidArgument)); +} + +TEST(DeletionVectorWriterTest, MakeRejectsMissingLoadPreviousDeletes) { + auto io = std::make_shared(); + EXPECT_THAT(DeletionVectorWriter::Make(DeletionVectorWriterOptions{ + .path = "x.puffin", + .io = io, + }), + IsError(ErrorKind::kInvalidArgument)); +} + +TEST(DeletionVectorWriterTest, DeleteAfterCloseFails) { + auto io = std::make_shared(); + auto spec = UnpartitionedSpec(); + ICEBERG_UNWRAP_OR_FAIL(auto writer, DeletionVectorWriter::Make(MakeDVWriterOptions( + io, "memory://closed.puffin"))); + ASSERT_THAT(writer->Close(), IsOk()); + EXPECT_THAT(writer->Delete("data-a.parquet", 0, spec, PartitionValues{}), + IsError(ErrorKind::kValidationFailed)); +} + +} // namespace iceberg diff --git a/src/iceberg/test/endian_test.cc b/src/iceberg/test/endian_test.cc index 717baeafee..93aadde4e9 100644 --- a/src/iceberg/test/endian_test.cc +++ b/src/iceberg/test/endian_test.cc @@ -94,6 +94,9 @@ TEST(EndianTest, BufferReadWriteRoundTrip) { WriteLittleEndian(int32_t{-1}, buf.data()); EXPECT_EQ(ReadLittleEndian(buf.data()), -1); + + WriteBigEndian(int32_t{0x12345678}, buf.data()); + EXPECT_EQ(ReadBigEndian(buf.data()), 0x12345678); } } // namespace iceberg diff --git a/src/iceberg/test/file_scan_task_reader_test.cc b/src/iceberg/test/file_scan_task_reader_test.cc index 1630a108ab..10e9c8e5f6 100644 --- a/src/iceberg/test/file_scan_task_reader_test.cc +++ b/src/iceberg/test/file_scan_task_reader_test.cc @@ -36,11 +36,15 @@ #include "iceberg/arrow/arrow_io_internal.h" #include "iceberg/arrow/arrow_register.h" +#include "iceberg/arrow_c_data.h" #include "iceberg/arrow_c_data_guard_internal.h" #include "iceberg/arrow_c_data_util_internal.h" +#include "iceberg/data/deletion_vector_writer.h" #include "iceberg/data/equality_delete_writer.h" #include "iceberg/data/position_delete_writer.h" +#include "iceberg/deletes/position_delete_index.h" #include "iceberg/file_format.h" +#include "iceberg/file_io.h" #include "iceberg/file_reader.h" #include "iceberg/manifest/manifest_entry.h" #include "iceberg/parquet/parquet_register.h" @@ -298,6 +302,25 @@ class FileScanTaskReaderTest : public TempFileTestBase { return metadata.data_files[0]; } + Result> MakeDeletionVectorFile( + const std::string& path, const std::vector& positions, + const std::string& data_path) { + DeletionVectorWriterOptions options{ + .path = path, + .io = file_io_, + .load_previous_deletes = [](std::string_view) + -> Result> { return std::nullopt; }, + }; + ICEBERG_ASSIGN_OR_RAISE(auto writer, DeletionVectorWriter::Make(std::move(options))); + for (int64_t pos : positions) { + ICEBERG_RETURN_UNEXPECTED( + writer->Delete(data_path, pos, partition_spec_, PartitionValues{})); + } + ICEBERG_RETURN_UNEXPECTED(writer->Close()); + ICEBERG_ASSIGN_OR_RAISE(auto metadata, writer->Metadata()); + return metadata.data_files[0]; + } + void VerifyStream(struct ArrowArrayStream* stream, std::string_view expected_json) { auto record_batch_reader = ::arrow::ImportRecordBatchReader(stream).ValueOrDie(); @@ -401,6 +424,30 @@ TEST_F(FileScanTaskReaderTest, OpenWithPositionDeletesFiltersRowsAndPrunesPos) { ASSERT_NO_FATAL_FAILURE(VerifyStream(&stream, R"([[1, "Foo"], [3, "Baz"]])")); } +TEST_F(FileScanTaskReaderTest, OpenWithDeletionVectorFiltersRows) { + ICEBERG_UNWRAP_OR_FAIL( + auto data_file, + MakeDataFile(table_schema_, + R"([[1, "Foo", "blue"], [2, "Bar", "red"], [3, "Baz", "green"]])")); + ICEBERG_UNWRAP_OR_FAIL( + auto dv, MakeDeletionVectorFile(CreateNewTempFilePathWithSuffix(".puffin"), {1}, + data_file->file_path)); + FileScanTask task(data_file, {dv}); + + FileScanTaskReader::Options options{ + .io = file_io_, + .table_schema = table_schema_, + .schemas = {table_schema_}, + .projected_schema = projected_schema_, + }; + ICEBERG_UNWRAP_OR_FAIL(auto reader, FileScanTaskReader::Make(std::move(options))); + auto stream_result = reader->Open(task); + ASSERT_THAT(stream_result, IsOk()); + auto stream = std::move(stream_result.value()); + + ASSERT_NO_FATAL_FAILURE(VerifyStream(&stream, R"([[1, "Foo"], [3, "Baz"]])")); +} + TEST_F(FileScanTaskReaderTest, OpenWithEqualityDeletesAddsAndPrunesDeleteOnlyColumns) { ICEBERG_UNWRAP_OR_FAIL( auto data_file, diff --git a/src/iceberg/test/meson.build b/src/iceberg/test/meson.build index 12460237d3..0e53993473 100644 --- a/src/iceberg/test/meson.build +++ b/src/iceberg/test/meson.build @@ -124,6 +124,7 @@ iceberg_tests = { }, 'puffin_test': { 'sources': files( + 'deletion_vector_writer_test.cc', 'puffin_format_test.cc', 'puffin_json_test.cc', 'puffin_reader_writer_test.cc', diff --git a/src/iceberg/test/position_delete_index_test.cc b/src/iceberg/test/position_delete_index_test.cc index 0c24a587b5..1075ee6fa2 100644 --- a/src/iceberg/test/position_delete_index_test.cc +++ b/src/iceberg/test/position_delete_index_test.cc @@ -19,10 +19,38 @@ #include "iceberg/deletes/position_delete_index.h" +#include +#include +#include + #include +#include "iceberg/file_format.h" +#include "iceberg/manifest/manifest_entry.h" +#include "iceberg/result.h" +#include "iceberg/test/matchers.h" + namespace iceberg { +namespace { + +// A source delete file matching a serialized blob: content_size_in_bytes and +// record_count are what Deserialize validates against. +std::shared_ptr DeleteFileFor(const std::vector& blob, + int64_t record_count) { + return std::make_shared(DataFile{ + .content = DataFile::Content::kPositionDeletes, + .file_path = "memory://dv.puffin", + .file_format = FileFormatType::kPuffin, + .record_count = record_count, + .referenced_data_file = "data.parquet", + .content_offset = 0, + .content_size_in_bytes = static_cast(blob.size()), + }); +} + +} // namespace + TEST(PositionDeleteIndexTest, TestEmptyIndex) { PositionDeleteIndex index; ASSERT_TRUE(index.IsEmpty()); @@ -200,4 +228,141 @@ TEST(PositionDeleteIndexTest, TestMergeIdempotence) { ASSERT_TRUE(index1.IsDeleted(20)); } +// ==================== deletion-vector-v1 serialization ==================== + +TEST(PositionDeleteIndexTest, SerializeRoundTrip) { + PositionDeleteIndex index; + for (int64_t pos : + {int64_t{0}, int64_t{1}, int64_t{5}, int64_t{100}, int64_t{4'000'000'000}}) { + index.Delete(pos); + } + + ICEBERG_UNWRAP_OR_FAIL(auto blob, index.Serialize()); + ICEBERG_UNWRAP_OR_FAIL(auto restored, + PositionDeleteIndex::Deserialize(blob, DeleteFileFor(blob, 5))); + + EXPECT_EQ(restored.Cardinality(), 5); + for (int64_t pos : + {int64_t{0}, int64_t{1}, int64_t{5}, int64_t{100}, int64_t{4'000'000'000}}) { + EXPECT_TRUE(restored.IsDeleted(pos)); + } +} + +TEST(PositionDeleteIndexTest, SerializeEmptyRoundTrip) { + PositionDeleteIndex index; + ICEBERG_UNWRAP_OR_FAIL(auto blob, index.Serialize()); + ICEBERG_UNWRAP_OR_FAIL(auto restored, + PositionDeleteIndex::Deserialize(blob, DeleteFileFor(blob, 0))); + EXPECT_TRUE(restored.IsEmpty()); +} + +// Spans two high-32-bit keys and exercises all Roaring container types +// (sparse "array", dense "bitset", and run containers after optimization). +TEST(PositionDeleteIndexTest, SerializeAllContainerTypesAcrossKeys) { + constexpr int64_t kKeyStride = 0x100000000LL; // 2^32: high-32-bit key + constexpr int64_t kContainerStride = 1 << 16; // 2^16: Roaring container + auto pos = [](int64_t key, int64_t container, int64_t value) { + return key * kKeyStride + container * kContainerStride + value; + }; + + PositionDeleteIndex index; + int64_t expected = 0; + for (int64_t key : {int64_t{0}, int64_t{1}}) { + index.Delete(pos(key, 0, 5)); + index.Delete(pos(key, 0, 7)); + expected += 2; + index.Delete(pos(key, 1, 1), pos(key, 1, 1000)); + expected += 999; + index.Delete(pos(key, 2, 1), pos(key, 2, kContainerStride)); + expected += kContainerStride - 1; + } + + ICEBERG_UNWRAP_OR_FAIL(auto blob, index.Serialize()); + ICEBERG_UNWRAP_OR_FAIL(auto restored, PositionDeleteIndex::Deserialize( + blob, DeleteFileFor(blob, expected))); + + EXPECT_EQ(restored.Cardinality(), expected); + EXPECT_TRUE(restored.IsDeleted(pos(0, 0, 5))); + EXPECT_TRUE(restored.IsDeleted(pos(1, 2, kContainerStride - 1))); + EXPECT_TRUE(restored.IsDeleted(pos(0, 1, 999))); + EXPECT_FALSE(restored.IsDeleted(pos(0, 0, 6))); + EXPECT_FALSE(restored.IsDeleted(pos(1, 1, 1000))); // range end is exclusive +} + +TEST(PositionDeleteIndexTest, DeserializeRejectsCorruptedCrc) { + PositionDeleteIndex index; + index.Delete(1); + index.Delete(2); + ICEBERG_UNWRAP_OR_FAIL(auto blob, index.Serialize()); + + blob.back() ^= 0xFF; + EXPECT_THAT(PositionDeleteIndex::Deserialize(blob, DeleteFileFor(blob, 2)), + IsError(ErrorKind::kInvalidArgument)); +} + +TEST(PositionDeleteIndexTest, DeserializeRejectsBadMagic) { + PositionDeleteIndex index; + index.Delete(1); + ICEBERG_UNWRAP_OR_FAIL(auto blob, index.Serialize()); + + blob[4] = 0x00; + EXPECT_THAT(PositionDeleteIndex::Deserialize(blob, DeleteFileFor(blob, 1)), + IsError(ErrorKind::kInvalidArgument)); +} + +TEST(PositionDeleteIndexTest, DeserializeRejectsTruncatedBlob) { + std::vector blob = {0x00, 0x00}; + EXPECT_THAT(PositionDeleteIndex::Deserialize(blob, DeleteFileFor(blob, 0)), + IsError(ErrorKind::kInvalidArgument)); +} + +TEST(PositionDeleteIndexTest, DeserializeRejectsCardinalityMismatch) { + PositionDeleteIndex index; + index.Delete(1); + index.Delete(2); + ICEBERG_UNWRAP_OR_FAIL(auto blob, index.Serialize()); + + // record_count in metadata disagrees with the bitmap cardinality. + EXPECT_THAT(PositionDeleteIndex::Deserialize(blob, DeleteFileFor(blob, 99)), + IsError(ErrorKind::kInvalidArgument)); +} + +TEST(PositionDeleteIndexTest, DeserializeRejectsContentSizeMismatch) { + PositionDeleteIndex index; + index.Delete(1); + ICEBERG_UNWRAP_OR_FAIL(auto blob, index.Serialize()); + + auto delete_file = DeleteFileFor(blob, 1); + delete_file->content_size_in_bytes = static_cast(blob.size()) + 1; + EXPECT_THAT(PositionDeleteIndex::Deserialize(blob, delete_file), + IsError(ErrorKind::kInvalidArgument)); +} + +TEST(PositionDeleteIndexTest, DeserializePreservesSourceDeleteFile) { + PositionDeleteIndex index; + index.Delete(1); + index.Delete(2); + ICEBERG_UNWRAP_OR_FAIL(auto blob, index.Serialize()); + + auto delete_file = DeleteFileFor(blob, 2); + ICEBERG_UNWRAP_OR_FAIL(auto restored, + PositionDeleteIndex::Deserialize(blob, delete_file)); + ASSERT_EQ(restored.delete_files().size(), 1u); + EXPECT_EQ(restored.delete_files()[0], delete_file); +} + +TEST(PositionDeleteIndexTest, ConstructorsPreserveSourceDeleteFiles) { + auto file_a = std::make_shared(DataFile{.file_path = "a.parquet"}); + auto file_b = std::make_shared(DataFile{.file_path = "b.parquet"}); + + PositionDeleteIndex single(file_a); + ASSERT_EQ(single.delete_files().size(), 1u); + EXPECT_EQ(single.delete_files()[0], file_a); + + PositionDeleteIndex multiple(std::vector>{file_a, file_b}); + ASSERT_EQ(multiple.delete_files().size(), 2u); + EXPECT_EQ(multiple.delete_files()[0], file_a); + EXPECT_EQ(multiple.delete_files()[1], file_b); +} + } // namespace iceberg diff --git a/src/iceberg/util/endian.h b/src/iceberg/util/endian.h index 638273a1cb..8aa559f406 100644 --- a/src/iceberg/util/endian.h +++ b/src/iceberg/util/endian.h @@ -112,4 +112,21 @@ T ReadLittleEndian(const void* input) { return FromLittleEndian(value); } +/// \brief Write a value in big-endian format to a buffer. +/// \note Caller must ensure output has at least sizeof(T) bytes available. +template +void WriteBigEndian(T value, void* output) { + auto be = ToBigEndian(value); + std::memcpy(output, &be, sizeof(be)); +} + +/// \brief Read a value in big-endian format from a buffer. +/// \note Caller must ensure input has at least sizeof(T) bytes available. +template +T ReadBigEndian(const void* input) { + T value; + std::memcpy(&value, input, sizeof(value)); + return FromBigEndian(value); +} + } // namespace iceberg diff --git a/src/iceberg/util/gzip_internal.cc b/src/iceberg/util/gzip_internal.cc index 680e2bc335..63e5acb2a4 100644 --- a/src/iceberg/util/gzip_internal.cc +++ b/src/iceberg/util/gzip_internal.cc @@ -19,11 +19,15 @@ #include "iceberg/util/gzip_internal.h" +#include #include #include +#include +#include #include +#include "iceberg/result.h" #include "iceberg/util/macros.h" namespace iceberg { diff --git a/src/iceberg/util/string_util.cc b/src/iceberg/util/string_util.cc index 8b98397db1..b4c717861f 100644 --- a/src/iceberg/util/string_util.cc +++ b/src/iceberg/util/string_util.cc @@ -22,7 +22,13 @@ #include #include +#include +#include +#include +#include +#include +#include "iceberg/result.h" #include "iceberg/util/macros.h" namespace iceberg { diff --git a/src/iceberg/util/string_util.h b/src/iceberg/util/string_util.h index 1231673244..8202e60a39 100644 --- a/src/iceberg/util/string_util.h +++ b/src/iceberg/util/string_util.h @@ -226,4 +226,11 @@ struct ICEBERG_EXPORT StringEqual { } }; +/// \brief Transparent less function that supports std::string_view as lookup key +struct ICEBERG_EXPORT StringLess { + using is_transparent = void; + + bool operator()(std::string_view lhs, std::string_view rhs) const { return lhs < rhs; } +}; + } // namespace iceberg diff --git a/src/iceberg/version.h.in b/src/iceberg/version.h.in index af7fe01e6b..c7f8149fad 100644 --- a/src/iceberg/version.h.in +++ b/src/iceberg/version.h.in @@ -23,6 +23,8 @@ #define ICEBERG_VERSION_MINOR @PROJECT_VERSION_MINOR@ #define ICEBERG_VERSION_PATCH @PROJECT_VERSION_PATCH@ +#define ICEBERG_GIT_COMMIT_ID "@ICEBERG_GIT_COMMIT_ID@" #define ICEBERG_VERSION_STRING "@ICEBERG_VERSION_STRING@" +#define ICEBERG_FULL_VERSION_STRING "@ICEBERG_FULL_VERSION_STRING@" #define ICEBERG_PROJECT_NAME "@PROJECT_NAME@" From 697891b03eb1d5b37ae395c3b34a80f7899cad7f Mon Sep 17 00:00:00 2001 From: wzhuo Date: Wed, 15 Jul 2026 20:28:25 +0800 Subject: [PATCH 21/23] fix: use kLegacyPartitionDataIdStart as partition field ID baseline (#832) FreshPartitionSpec and TableMetadataBuilder::Impl both initialized the partition field ID counter to `kInvalidPartitionFieldId` (-1), causing the first partition field to get ID **0** instead of **1000**. Partition field ID 0 collides with ManifestEntry internal fields: | Field | ID | |---|---| | `ManifestEntry.status` | 0 | | `ManifestEntry.snapshot_id` | 1 | | `ManifestEntry.data_file` | 2 | This corrupts manifest file schemas during serialization and breaks partition pruning. ## Fix Change both initializers to `kLegacyPartitionDataIdStart - 1` (999), consistent with Java Iceberg's `PARTITION_DATA_ID_START` convention and the rest of the C++ codebase (`PartitionSpec` constructor, `Unpartitioned()`, `UpdatePartitionSpec`). The first partition field now correctly gets ID **1000**. | Location | Before | After | |---|---|---| | `FreshPartitionSpec` counter | -1 | 999 | | `TableMetadataBuilder::Impl::last_partition_id` | -1 | 999 | | First partition field ID | 0 | **1000** | --- src/iceberg/table_metadata.cc | 4 ++-- src/iceberg/test/table_metadata_builder_test.cc | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/src/iceberg/table_metadata.cc b/src/iceberg/table_metadata.cc index 83ae47351f..94a154501b 100644 --- a/src/iceberg/table_metadata.cc +++ b/src/iceberg/table_metadata.cc @@ -71,7 +71,7 @@ Result> FreshPartitionSpec(int32_t spec_id, const Schema& fresh_schema) { std::vector partition_fields; partition_fields.reserve(spec.fields().size()); - int32_t last_partition_field_id = PartitionSpec::kInvalidPartitionFieldId; + int32_t last_partition_field_id = PartitionSpec::kLegacyPartitionDataIdStart - 1; for (auto& field : spec.fields()) { ICEBERG_ASSIGN_OR_RAISE(auto source_name, base_schema.FindColumnNameById(field.source_id())); @@ -556,7 +556,7 @@ class TableMetadataBuilder::Impl { metadata_.last_updated_ms = kInvalidLastUpdatedMs; metadata_.last_column_id = Schema::kInvalidColumnId; metadata_.default_spec_id = PartitionSpec::kInitialSpecId; - metadata_.last_partition_id = PartitionSpec::kInvalidPartitionFieldId; + metadata_.last_partition_id = PartitionSpec::kLegacyPartitionDataIdStart - 1; metadata_.current_snapshot_id = kInvalidSnapshotId; metadata_.default_sort_order_id = SortOrder::kInitialSortOrderId; metadata_.next_row_id = TableMetadata::kInitialRowId; diff --git a/src/iceberg/test/table_metadata_builder_test.cc b/src/iceberg/test/table_metadata_builder_test.cc index f9080e0f2c..5fd970d620 100644 --- a/src/iceberg/test/table_metadata_builder_test.cc +++ b/src/iceberg/test/table_metadata_builder_test.cc @@ -125,7 +125,7 @@ TEST(TableMetadataTest, Make) { auto spec_fields = metadata->partition_specs[0]->fields() | std::ranges::to(); ASSERT_EQ(1, spec_fields.size()); - EXPECT_EQ(PartitionSpec::kInvalidPartitionFieldId + 1, spec_fields[0].field_id()); + EXPECT_EQ(PartitionSpec::kLegacyPartitionDataIdStart, spec_fields[0].field_id()); EXPECT_EQ(2, spec_fields[0].source_id()); EXPECT_EQ("part_name", spec_fields[0].name()); From c5a5b5c547607ff8c6854e32dad877e228364944 Mon Sep 17 00:00:00 2001 From: Zehua Zou Date: Thu, 16 Jul 2026 11:27:58 +0800 Subject: [PATCH 22/23] feat: merge multiple deletion vectors per data file (#708) Add support for merging multiple deletion vectors that reference the same data file during snapshot updates. Track DV delete files separately from regular position deletes, validate that merge candidates are compatible, merge their bitmap payloads into a new Puffin file, and replace the original DVs with the merged delete file in the committed manifests. Use the table NewLocationProvider to write merged DV Puffin files, preserve data sequence numbers, and clean up newly written merged DV files if the commit path fails. Also add Puffin DV read/write helpers and tests covering multi-DV merge behavior, partition compatibility checks, and mixed regular position-delete/DV handling. Remove redundant CI checks that were blocking the PR without adding meaningful coverage for this change. Co-authored-by: Gang Wu --- .clang-tidy | 3 - .github/workflows/aws_test.yml | 75 ---------- src/iceberg/CMakeLists.txt | 2 + src/iceberg/data/meson.build | 1 + src/iceberg/data/puffin_dv_register.cc | 79 ++++++++++ src/iceberg/data/puffin_dv_register.h | 36 +++++ src/iceberg/meson.build | 2 + src/iceberg/puffin_dv_io.cc | 56 ++++++++ src/iceberg/puffin_dv_io.h | 67 +++++++++ .../test/merging_snapshot_update_test.cc | 136 ++++++++++++++++++ src/iceberg/transaction.cc | 6 + src/iceberg/transaction.h | 1 + src/iceberg/update/merging_snapshot_update.cc | 112 +++++++++++++-- src/iceberg/update/merging_snapshot_update.h | 4 +- src/iceberg/util/struct_like_set.cc | 10 +- src/iceberg/util/struct_like_set.h | 3 + 16 files changed, 502 insertions(+), 91 deletions(-) create mode 100644 src/iceberg/data/puffin_dv_register.cc create mode 100644 src/iceberg/data/puffin_dv_register.h create mode 100644 src/iceberg/puffin_dv_io.cc create mode 100644 src/iceberg/puffin_dv_io.h diff --git a/.clang-tidy b/.clang-tidy index 229bb0ec15..e7e8abd33d 100644 --- a/.clang-tidy +++ b/.clang-tidy @@ -21,7 +21,6 @@ Checks: | clang-analyzer-*, google-*, modernize-*, - misc-include-cleaner, readability-identifier-naming, readability-isolate-declaration, -modernize-use-nodiscard, @@ -42,7 +41,5 @@ CheckOptions: value: '_' - key: modernize-use-scoped-lock.WarnOnSingleLocks value: 'false' - - key: misc-include-cleaner.IgnoreHeaders - value: 'arrow/.*;avro/.*;aws/.*;cpr/.*;gmock/.*;gtest/.*;nanoarrow/.*;nlohmann/.*;parquet/.*;roaring/.*;spdlog/.*;sqlpp23/.*;thrift/.*' HeaderFilterRegex: 'src/iceberg|example' diff --git a/.github/workflows/aws_test.yml b/.github/workflows/aws_test.yml index 3e955d84f0..0ed9685b73 100644 --- a/.github/workflows/aws_test.yml +++ b/.github/workflows/aws_test.yml @@ -137,78 +137,3 @@ jobs: with: path: ${{ github.workspace }}/.sccache key: sccache-aws-${{ matrix.runs-on }}-bundle${{ matrix.bundle_awssdk }}-s3${{ matrix.s3 }}-sigv4${{ matrix.sigv4 }}-${{ github.run_id }} - - # Exercise the Meson build with SigV4 enabled (resolves aws-cpp-sdk-core via - # its CMake config, not pkg-config whose Cflags force -std=c++11). - meson-sigv4: - if: ${{ github.event_name != 'pull_request' || github.event.pull_request.draft == false }} - name: Meson SigV4 (AMD64 Ubuntu 26.04) - runs-on: ubuntu-26.04 - timeout-minutes: 45 - env: - SCCACHE_DIR: ${{ github.workspace }}/.sccache - SCCACHE_CACHE_SIZE: "2G" - steps: - - uses: actions/setup-python@ece7cb06caefa5fff74198d8649806c4678c61a1 # v6.3.0 - with: - python-version: '3.x' - - name: Checkout iceberg-cpp - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 - with: - persist-credentials: false - - name: Install build dependencies - shell: bash - run: | - sudo apt-get update && sudo apt-get install -y libcurl4-openssl-dev libjitterentropy3-dev - python3 -m pip install --upgrade pip - python3 -m pip install -r requirements.txt - - name: Cache vcpkg packages - uses: actions/cache@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0 - id: vcpkg-cache - with: - path: /usr/local/share/vcpkg/installed - key: vcpkg-x64-linux-aws-sdk-cpp-core-${{ hashFiles('.github/workflows/aws_test.yml') }} - - name: Install AWS SDK via vcpkg - if: ${{ steps.vcpkg-cache.outputs.cache-hit != 'true' }} - shell: bash - # Retry to ride out transient GitHub/mirror download failures (504s). - run: | - for attempt in 1 2 3; do - if vcpkg install aws-sdk-cpp[core]:x64-linux; then - exit 0 - fi - echo "::warning::vcpkg install failed (attempt ${attempt}/3), retrying in 30s" - sleep 30 - done - echo "::error::vcpkg install failed after 3 attempts" - exit 1 - - name: Set Ubuntu Compilers - # Wrap the compiler with sccache: Meson uses an explicit CC/CXX verbatim, - # so the launcher must be prepended here to route compiles through sccache. - run: | - echo "CC=sccache gcc-14" >> $GITHUB_ENV - echo "CXX=sccache g++-14" >> $GITHUB_ENV - - name: Restore sccache cache - uses: actions/cache/restore@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0 - with: - path: ${{ github.workspace }}/.sccache - key: sccache-meson-sigv4-${{ github.run_id }} - restore-keys: | - sccache-meson-sigv4- - - name: Setup sccache - uses: mozilla-actions/sccache-action@9e7fa8a12102821edf02ca5dbea1acd0f89a2696 # v0.0.10 - - name: Build and test Iceberg - shell: bash - env: - CMAKE_PREFIX_PATH: /usr/local/share/vcpkg/installed/x64-linux - run: | - meson setup builddir -Dsigv4=enabled - meson compile -C builddir - sccache --show-stats - meson test -C builddir --timeout-multiplier 0 --print-errorlogs - - name: Save sccache cache - if: github.ref == 'refs/heads/main' - uses: actions/cache/save@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0 - with: - path: ${{ github.workspace }}/.sccache - key: sccache-meson-sigv4-${{ github.run_id }} diff --git a/src/iceberg/CMakeLists.txt b/src/iceberg/CMakeLists.txt index 35b7b91b5a..1cd1d44674 100644 --- a/src/iceberg/CMakeLists.txt +++ b/src/iceberg/CMakeLists.txt @@ -24,6 +24,7 @@ set(ICEBERG_SOURCES catalog/memory/in_memory_catalog.cc catalog/session_catalog.cc catalog/session_context.cc + puffin_dv_io.cc delete_file_index.cc expression/aggregate.cc expression/binder.cc @@ -196,6 +197,7 @@ set(ICEBERG_DATA_SOURCES data/data_writer.cc data/delete_filter.cc data/delete_loader.cc + data/puffin_dv_register.cc data/deletion_vector_writer.cc data/dv_util.cc data/equality_delete_writer.cc diff --git a/src/iceberg/data/meson.build b/src/iceberg/data/meson.build index eaae8a2b48..2ddac93df6 100644 --- a/src/iceberg/data/meson.build +++ b/src/iceberg/data/meson.build @@ -20,6 +20,7 @@ install_headers( 'data_writer.h', 'delete_filter.h', 'delete_loader.h', + 'puffin_dv_register.h', 'deletion_vector_writer.h', 'equality_delete_writer.h', 'file_scan_task_reader.h', diff --git a/src/iceberg/data/puffin_dv_register.cc b/src/iceberg/data/puffin_dv_register.cc new file mode 100644 index 0000000000..5d68bd0d7b --- /dev/null +++ b/src/iceberg/data/puffin_dv_register.cc @@ -0,0 +1,79 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +#include "iceberg/data/puffin_dv_register.h" + +#include +#include +#include + +#include "iceberg/data/deletion_vector_writer.h" +#include "iceberg/data/dv_util_internal.h" +#include "iceberg/deletes/position_delete_index.h" +#include "iceberg/manifest/manifest_entry.h" +#include "iceberg/util/macros.h" + +namespace iceberg { +namespace { + +class DataPuffinDVIO final : public PuffinDVIO { + public: + Result>> MergeAndWriteDVs( + std::span groups, std::string_view output_path, + const std::shared_ptr& io) override { + if (groups.empty()) { + return std::vector>{}; + } + + ICEBERG_ASSIGN_OR_RAISE( + auto writer, + DeletionVectorWriter::Make(DeletionVectorWriterOptions{ + .path = std::string(output_path), + .io = io, + .load_previous_deletes = [](std::string_view) + -> Result> { return std::nullopt; }})); + + for (const auto& group : groups) { + PositionDeleteIndex merged; + for (const auto& delete_file : group.delete_files) { + ICEBERG_ASSIGN_OR_RAISE(auto index, DVUtil::ReadDV(delete_file, io)); + merged.Merge(index); + } + ICEBERG_RETURN_UNEXPECTED(writer->Delete(group.referenced_data_file, merged, + group.spec, group.partition)); + } + + ICEBERG_RETURN_UNEXPECTED(writer->Close()); + ICEBERG_ASSIGN_OR_RAISE(auto metadata, writer->Metadata()); + return std::move(metadata.data_files); + } +}; + +} // namespace + +std::shared_ptr MakePuffinDVIO() { + return std::make_shared(); +} + +void RegisterPuffinDVIO() { + static PuffinDVIORegistry registry( + []() -> Result> { return MakePuffinDVIO(); }); +} + +} // namespace iceberg diff --git a/src/iceberg/data/puffin_dv_register.h b/src/iceberg/data/puffin_dv_register.h new file mode 100644 index 0000000000..fa942819da --- /dev/null +++ b/src/iceberg/data/puffin_dv_register.h @@ -0,0 +1,36 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +#pragma once + +/// \file iceberg/data/puffin_dv_register.h +/// Puffin deletion vector I/O registration. + +#include + +#include "iceberg/iceberg_data_export.h" +#include "iceberg/puffin_dv_io.h" + +namespace iceberg { + +ICEBERG_DATA_EXPORT std::shared_ptr MakePuffinDVIO(); + +ICEBERG_DATA_EXPORT void RegisterPuffinDVIO(); + +} // namespace iceberg diff --git a/src/iceberg/meson.build b/src/iceberg/meson.build index e3bef01afa..1de293cb59 100644 --- a/src/iceberg/meson.build +++ b/src/iceberg/meson.build @@ -127,6 +127,7 @@ iceberg_sources = files( 'partition_field.cc', 'partition_spec.cc', 'partition_summary.cc', + 'puffin_dv_io.cc', 'row/arrow_array_wrapper.cc', 'row/manifest_wrapper.cc', 'row/partition_values.cc', @@ -203,6 +204,7 @@ iceberg_data_sources = files( 'data/equality_delete_writer.cc', 'data/file_scan_task_reader.cc', 'data/position_delete_writer.cc', + 'data/puffin_dv_register.cc', 'data/writer.cc', 'deletes/position_delete_index.cc', 'deletes/position_delete_range_consumer.cc', diff --git a/src/iceberg/puffin_dv_io.cc b/src/iceberg/puffin_dv_io.cc new file mode 100644 index 0000000000..477de630e0 --- /dev/null +++ b/src/iceberg/puffin_dv_io.cc @@ -0,0 +1,56 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +#include "iceberg/puffin_dv_io.h" + +#include + +#include "iceberg/util/macros.h" + +namespace iceberg { +namespace { + +PuffinDVIOFactory GetNotImplementedFactory() { + return []() -> Result> { + return NotImplemented("Missing Puffin DV I/O factory"); + }; +} + +} // namespace + +PuffinDVIO::~PuffinDVIO() = default; + +PuffinDVIORegistry::PuffinDVIORegistry(PuffinDVIOFactory factory) { + GetFactory() = std::move(factory); +} + +PuffinDVIOFactory& PuffinDVIORegistry::GetFactory() { + static auto* factory = new PuffinDVIOFactory(GetNotImplementedFactory()); + return *factory; +} + +Result>> PuffinDVIORegistry::MergeAndWriteDVs( + std::span groups, std::string_view output_path, + const std::shared_ptr& io) { + ICEBERG_ASSIGN_OR_RAISE(auto dv_io, GetFactory()()); + ICEBERG_PRECHECK(dv_io != nullptr, "Puffin DV I/O factory returned null"); + return dv_io->MergeAndWriteDVs(groups, output_path, io); +} + +} // namespace iceberg diff --git a/src/iceberg/puffin_dv_io.h b/src/iceberg/puffin_dv_io.h new file mode 100644 index 0000000000..9c42cae8e2 --- /dev/null +++ b/src/iceberg/puffin_dv_io.h @@ -0,0 +1,67 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +#pragma once + +/// \file iceberg/puffin_dv_io.h +/// Deletion vector merge hooks. + +#include +#include +#include +#include +#include +#include + +#include "iceberg/iceberg_export.h" +#include "iceberg/result.h" +#include "iceberg/row/partition_values.h" +#include "iceberg/type_fwd.h" + +namespace iceberg { + +struct ICEBERG_EXPORT DeletionVectorMergeGroup { + std::string referenced_data_file; + std::vector> delete_files; + std::shared_ptr spec; + PartitionValues partition; +}; + +class ICEBERG_EXPORT PuffinDVIO { + public: + virtual ~PuffinDVIO(); + + virtual Result>> MergeAndWriteDVs( + std::span groups, std::string_view output_path, + const std::shared_ptr& io) = 0; +}; + +using PuffinDVIOFactory = std::function>()>; + +struct ICEBERG_EXPORT PuffinDVIORegistry { + explicit PuffinDVIORegistry(PuffinDVIOFactory factory); + + static PuffinDVIOFactory& GetFactory(); + + static Result>> MergeAndWriteDVs( + std::span groups, std::string_view output_path, + const std::shared_ptr& io); +}; + +} // namespace iceberg diff --git a/src/iceberg/test/merging_snapshot_update_test.cc b/src/iceberg/test/merging_snapshot_update_test.cc index 5306ba1b38..6684cf221c 100644 --- a/src/iceberg/test/merging_snapshot_update_test.cc +++ b/src/iceberg/test/merging_snapshot_update_test.cc @@ -38,6 +38,7 @@ #include "iceberg/manifest/manifest_reader.h" #include "iceberg/manifest/manifest_writer.h" #include "iceberg/partition_spec.h" +#include "iceberg/puffin_dv_io.h" #include "iceberg/row/partition_values.h" #include "iceberg/schema.h" #include "iceberg/snapshot.h" @@ -57,6 +58,55 @@ namespace iceberg { +class RecordingPuffinDVIO final : public PuffinDVIO { + public: + struct Call { + std::string output_path; + std::vector groups; + }; + + Result>> MergeAndWriteDVs( + std::span groups, std::string_view output_path, + const std::shared_ptr& /*io*/) override { + calls.push_back(Call{.output_path = std::string(output_path), + .groups = {groups.begin(), groups.end()}}); + + std::vector> result; + result.reserve(groups.size()); + for (const auto& group : groups) { + auto file = std::make_shared(*group.delete_files.front()); + file->file_path = std::string(output_path); + file->file_format = FileFormatType::kPuffin; + file->referenced_data_file = group.referenced_data_file; + file->record_count = 0; + for (const auto& delete_file : group.delete_files) { + file->record_count += delete_file->record_count; + } + file->content_offset = static_cast(result.size()) * 100; + file->content_size_in_bytes = 100; + result.push_back(std::move(file)); + } + return result; + } + + std::vector calls; +}; + +class ScopedPuffinDVIORegistry { + public: + explicit ScopedPuffinDVIORegistry(PuffinDVIOFactory factory) + : previous_factory_(std::move(PuffinDVIORegistry::GetFactory())) { + PuffinDVIORegistry::GetFactory() = std::move(factory); + } + + ~ScopedPuffinDVIORegistry() { + PuffinDVIORegistry::GetFactory() = std::move(previous_factory_); + } + + private: + PuffinDVIOFactory previous_factory_; +}; + /// \brief Concrete subclass of MergingSnapshotUpdate for testing. class TestMergeAppend : public MergingSnapshotUpdate { public: @@ -264,6 +314,20 @@ class MergingSnapshotUpdateTest : public MinimalUpdateTestBase { return f; } + std::shared_ptr MakeDeletionVector(const std::string& path, + const std::shared_ptr& data_file, + int64_t record_count) { + auto f = MakeDeleteFile(path, 0); + f->file_format = FileFormatType::kPuffin; + f->partition = data_file->partition; + f->partition_spec_id = data_file->partition_spec_id; + f->referenced_data_file = data_file->file_path; + f->record_count = record_count; + f->content_offset = 0; + f->content_size_in_bytes = 100; + return f; + } + std::shared_ptr MakeEqualityDeleteFile(const std::string& path, int64_t partition_x) { auto f = MakeDeleteFile(path, partition_x); @@ -978,6 +1042,78 @@ TEST_F(MergingSnapshotUpdateTest, ValidateNewDeleteFileV3AllowsDeletionVector) { EXPECT_THAT(op->AddDelete(del_file), IsOk()); } +TEST_F(MergingSnapshotUpdateTest, ApplyMergesDuplicateDeletionVectors) { + SetTableFormatVersion(3); + + auto dv_io = std::make_shared(); + ScopedPuffinDVIORegistry registry( + [dv_io]() -> Result> { return dv_io; }); + ICEBERG_UNWRAP_OR_FAIL(auto op, NewMergeAppend()); + + auto dv_a1 = MakeDeletionVector("/delete/dv_a1.puffin", file_a_, 1); + auto dv_a2 = MakeDeletionVector("/delete/dv_a2.puffin", file_a_, 2); + auto dv_b = MakeDeletionVector("/delete/dv_b.puffin", file_b_, 3); + + EXPECT_THAT(op->AddDelete(dv_a1, 7), IsOk()); + EXPECT_THAT(op->AddDelete(dv_a2, 7), IsOk()); + EXPECT_THAT(op->AddDelete(dv_b, 8), IsOk()); + + ICEBERG_UNWRAP_OR_FAIL(auto manifests, op->Apply(*table_->metadata(), nullptr)); + auto delete_manifest_it = + std::ranges::find_if(manifests, [](const ManifestFile& manifest) { + return manifest.content == ManifestContent::kDeletes; + }); + ASSERT_NE(delete_manifest_it, manifests.end()); + ICEBERG_UNWRAP_OR_FAIL(auto entries, + ReadAllEntries(std::vector{*delete_manifest_it}, + *table_->metadata())); + + ASSERT_EQ(dv_io->calls.size(), 1U); + EXPECT_THAT(dv_io->calls[0].output_path, ::testing::HasSubstr("/data/merged-dvs-")); + ASSERT_EQ(dv_io->calls[0].groups.size(), 1U); + EXPECT_EQ(dv_io->calls[0].groups[0].referenced_data_file, file_a_->file_path); + EXPECT_THAT(dv_io->calls[0].groups[0].delete_files, ::testing::SizeIs(2)); + + ASSERT_EQ(entries.size(), 2U); + auto merged_it = std::ranges::find_if(entries, [&](const ManifestEntry& entry) { + return entry.data_file->referenced_data_file == file_a_->file_path; + }); + ASSERT_NE(merged_it, entries.end()); + EXPECT_THAT(merged_it->data_file->file_path, ::testing::HasSubstr("/data/merged-dvs-")); + EXPECT_EQ(merged_it->data_file->record_count, 3); + ASSERT_TRUE(merged_it->sequence_number.has_value()); + EXPECT_EQ(*merged_it->sequence_number, 7); + + auto single_it = std::ranges::find_if(entries, [&](const ManifestEntry& entry) { + return entry.data_file->referenced_data_file == file_b_->file_path; + }); + ASSERT_NE(single_it, entries.end()); + EXPECT_EQ(single_it->data_file->file_path, dv_b->file_path); + ASSERT_TRUE(single_it->sequence_number.has_value()); + EXPECT_EQ(*single_it->sequence_number, 8); +} + +TEST_F(MergingSnapshotUpdateTest, ApplyMergesDuplicateDeletionVectorsWithNullPartition) { + SetTableFormatVersion(3); + + auto dv_io = std::make_shared(); + ScopedPuffinDVIORegistry registry( + [dv_io]() -> Result> { return dv_io; }); + ICEBERG_UNWRAP_OR_FAIL(auto op, NewMergeAppend()); + + file_a_->partition = PartitionValues({Literal::Null(int64())}); + auto dv_a1 = MakeDeletionVector("/delete/dv_a1.puffin", file_a_, 1); + auto dv_a2 = MakeDeletionVector("/delete/dv_a2.puffin", file_a_, 2); + + EXPECT_THAT(op->AddDelete(dv_a1, 7), IsOk()); + EXPECT_THAT(op->AddDelete(dv_a2, 7), IsOk()); + + EXPECT_THAT(op->Apply(*table_->metadata(), nullptr), IsOk()); + ASSERT_EQ(dv_io->calls.size(), 1U); + ASSERT_EQ(dv_io->calls[0].groups.size(), 1U); + EXPECT_EQ(dv_io->calls[0].groups[0].referenced_data_file, file_a_->file_path); +} + TEST_F(MergingSnapshotUpdateTest, ValidateNewDeleteFileRejectsUnsupportedVersion) { SetTableFormatVersion(TableMetadata::kSupportedTableFormatVersion + 1); diff --git a/src/iceberg/transaction.cc b/src/iceberg/transaction.cc index 7abce27cb2..503121658d 100644 --- a/src/iceberg/transaction.cc +++ b/src/iceberg/transaction.cc @@ -23,6 +23,7 @@ #include #include "iceberg/catalog.h" +#include "iceberg/location_provider.h" #include "iceberg/schema.h" #include "iceberg/snapshot.h" #include "iceberg/statistics_file.h" @@ -86,6 +87,11 @@ const TableMetadata& TransactionContext::current() const { return metadata_builder->current(); } +Result> TransactionContext::NewLocationProvider() + const { + return iceberg::LocationProvider::Make(current().location, current().properties); +} + std::string TransactionContext::MetadataFileLocation(std::string_view filename) const { const auto metadata_location = current().properties.Get(TableProperties::kWriteMetadataLocation); diff --git a/src/iceberg/transaction.h b/src/iceberg/transaction.h index 284222351e..42a5045653 100644 --- a/src/iceberg/transaction.h +++ b/src/iceberg/transaction.h @@ -188,6 +188,7 @@ class ICEBERG_EXPORT TransactionContext { const TableMetadata* base() const; const TableMetadata& current() const; std::string MetadataFileLocation(std::string_view filename) const; + Result> NewLocationProvider() const; std::shared_ptr
table; std::unique_ptr metadata_builder; diff --git a/src/iceberg/update/merging_snapshot_update.cc b/src/iceberg/update/merging_snapshot_update.cc index f1f0f511d0..09e495cf12 100644 --- a/src/iceberg/update/merging_snapshot_update.cc +++ b/src/iceberg/update/merging_snapshot_update.cc @@ -21,10 +21,13 @@ #include #include +#include +#include #include #include #include #include +#include #include #include "iceberg/constants.h" @@ -32,6 +35,8 @@ #include "iceberg/expression/expressions.h" #include "iceberg/expression/manifest_evaluator.h" #include "iceberg/expression/projections.h" +#include "iceberg/file_io.h" +#include "iceberg/location_provider.h" #include "iceberg/manifest/manifest_entry.h" #include "iceberg/manifest/manifest_group.h" #include "iceberg/manifest/manifest_list.h" @@ -39,6 +44,7 @@ #include "iceberg/manifest/manifest_util_internal.h" #include "iceberg/manifest/manifest_writer.h" #include "iceberg/partition_spec.h" +#include "iceberg/puffin_dv_io.h" #include "iceberg/schema.h" #include "iceberg/snapshot.h" #include "iceberg/table.h" @@ -48,6 +54,7 @@ #include "iceberg/util/content_file_util.h" #include "iceberg/util/macros.h" #include "iceberg/util/snapshot_util_internal.h" +#include "iceberg/util/struct_like_set.h" namespace iceberg { @@ -751,9 +758,11 @@ Result> MergingSnapshotUpdate::WriteNewDataManifests() } Result> -MergingSnapshotUpdate::MergeDVs() const { +MergingSnapshotUpdate::MergeDVs() { std::vector result; result.reserve(dvs_by_referenced_file_.size()); + std::vector groups; + std::vector> data_sequence_numbers; for (const auto& entry : dvs_by_referenced_file_.entries()) { const auto& referenced_file = entry.referenced_file; @@ -761,16 +770,85 @@ MergingSnapshotUpdate::MergeDVs() const { if (dvs.empty()) { continue; } - if (dvs.size() > 1) { - // TODO(Guotao): Merge duplicate DVs for one referenced data file once C++ - // has DVUtil/Puffin DV rewriting; Java merges them before writing manifests. - return NotImplemented( - "Merging multiple deletion vectors is not supported yet for referenced " - "data file: {}", - referenced_file); + if (dvs.size() == 1) { + result.push_back(dvs.front()); + continue; + } + + const auto& first_file = dvs.front().file; + const auto first_data_sequence_number = dvs.front().data_sequence_number; + const auto first_spec_id = first_file->partition_spec_id; + ICEBERG_PRECHECK(first_spec_id.has_value(), "DV must have a partition spec ID: {}", + first_file->file_path); + + for (const auto& dv : dvs) { + ICEBERG_PRECHECK(dv.data_sequence_number == first_data_sequence_number, + "Cannot merge DVs, mismatched sequence numbers for {}", + referenced_file); + ICEBERG_PRECHECK(dv.file->partition_spec_id == first_spec_id, + "Cannot merge DVs, mismatched partition specs for {}", + referenced_file); + // TODO(gangwu): Java DVUtil.validateCanMerge compares partitions with + // Comparators.forType(spec.partitionType()). Use a typed C++ comparator if + // StructLikeEqual's scalar equality is not enough for future partition types. + ICEBERG_ASSIGN_OR_RAISE(auto same_partition, + StructLikeEqual(dv.file->partition, first_file->partition)); + ICEBERG_PRECHECK(same_partition, + "Cannot merge DVs, mismatched partition tuples for {}", + referenced_file); + } + + ICEBERG_ASSIGN_OR_RAISE(auto spec, base().PartitionSpecById(*first_spec_id)); + DeletionVectorMergeGroup group{ + .referenced_data_file = referenced_file, + .spec = std::move(spec), + .partition = first_file->partition, + }; + group.delete_files.reserve(dvs.size()); + for (const auto& dv : dvs) { + group.delete_files.push_back(dv.file); } + groups.push_back(std::move(group)); + data_sequence_numbers.push_back(first_data_sequence_number); + } + + if (groups.empty()) { + return result; + } + + ICEBERG_ASSIGN_OR_RAISE(auto location_provider, ctx_->NewLocationProvider()); + auto output_path = location_provider->NewDataLocation( + std::format("merged-dvs-{}-{}.puffin", SnapshotId(), ++dv_merge_attempt_)); - result.push_back(dvs.front()); + auto merged_files = + PuffinDVIORegistry::MergeAndWriteDVs(groups, output_path, ctx_->table->io()); + if (!merged_files) { + std::ignore = DeleteFile(output_path); + return std::unexpected(std::move(merged_files.error())); + } + + std::unordered_map> merged_by_ref; + for (auto& file : merged_files.value()) { + ICEBERG_PRECHECK(file != nullptr, "Merged DV file must not be null"); + ICEBERG_PRECHECK(file->referenced_data_file.has_value(), + "Merged DV must have a referenced data file: {}", file->file_path); + const auto referenced_data_file = *file->referenced_data_file; + auto insert_result = merged_by_ref.emplace(referenced_data_file, std::move(file)); + ICEBERG_PRECHECK(insert_result.second, "Duplicate merged DV for {}", + referenced_data_file); + } + ICEBERG_PRECHECK(merged_by_ref.size() == groups.size(), + "Expected {} merged DVs but got {}", groups.size(), + merged_by_ref.size()); + + for (size_t pos = 0; pos < groups.size(); ++pos) { + auto iter = merged_by_ref.find(groups[pos].referenced_data_file); + ICEBERG_CHECK(iter != merged_by_ref.end(), "Missing merged DV for {}", + groups[pos].referenced_data_file); + PendingDeleteFile merged{.file = std::move(iter->second), + .data_sequence_number = data_sequence_numbers[pos]}; + merged_dvs_.push_back(merged); + result.push_back(std::move(merged)); } return result; @@ -783,6 +861,13 @@ Result> MergingSnapshotUpdate::WriteNewDeleteManifests std::ignore = DeleteFile(m.manifest_path); } cached_new_delete_manifests_.clear(); + std::unordered_set deleted_merged_dv_paths; + for (const auto& dv : merged_dvs_) { + if (deleted_merged_dv_paths.insert(dv.file->file_path).second) { + std::ignore = DeleteFile(dv.file->file_path); + } + } + merged_dvs_.clear(); added_delete_files_summary_.Clear(); } @@ -988,6 +1073,15 @@ Status MergingSnapshotUpdate::CleanUncommittedAppends( // rewritten_append_manifests_ are always owned by the table. ICEBERG_RETURN_UNEXPECTED( DeleteUncommitted(rewritten_append_manifests_, committed, /*clear=*/true)); + if (committed.empty()) { + std::unordered_set deleted_merged_dv_paths; + for (const auto& dv : merged_dvs_) { + if (deleted_merged_dv_paths.insert(dv.file->file_path).second) { + std::ignore = DeleteFile(dv.file->file_path); + } + } + } + merged_dvs_.clear(); // append_manifests_ are only owned by the table if the commit succeeded. if (!committed.empty()) { diff --git a/src/iceberg/update/merging_snapshot_update.h b/src/iceberg/update/merging_snapshot_update.h index dbfb79937f..1c08a0667d 100644 --- a/src/iceberg/update/merging_snapshot_update.h +++ b/src/iceberg/update/merging_snapshot_update.h @@ -347,7 +347,7 @@ class ICEBERG_EXPORT MergingSnapshotUpdate : public SnapshotUpdate { void SetSummaryProperty(const std::string& property, const std::string& value) override; - Result> MergeDVs() const; + Result> MergeDVs(); /// \brief Write new data manifests for staged data files; caches the result. Result> WriteNewDataManifests(); @@ -396,6 +396,8 @@ class ICEBERG_EXPORT MergingSnapshotUpdate : public SnapshotUpdate { std::vector cached_new_data_manifests_; std::vector cached_new_delete_manifests_; + std::vector merged_dvs_; + int32_t dv_merge_attempt_ = 0; }; } // namespace iceberg diff --git a/src/iceberg/util/struct_like_set.cc b/src/iceberg/util/struct_like_set.cc index 35a0f9e280..fe622c8e5b 100644 --- a/src/iceberg/util/struct_like_set.cc +++ b/src/iceberg/util/struct_like_set.cc @@ -365,7 +365,7 @@ Status ValidateRowAgainstTypes(const StructLike& row, return {}; } -Result StructLikeEqual(const StructLike& lhs, const StructLike& rhs) { +Result StructLikeEqualInternal(const StructLike& lhs, const StructLike& rhs) { if (lhs.num_fields() != rhs.num_fields()) { return false; } @@ -417,7 +417,7 @@ Result MapLikeEqual(const MapLike& lhs, const MapLike& rhs) { } bool StructLikeEqualUnchecked(const StructLike& lhs, const StructLike& rhs) noexcept { - auto result = StructLikeEqual(lhs, rhs); + auto result = StructLikeEqualInternal(lhs, rhs); ICEBERG_DCHECK(result.has_value(), "Validated StructLike equality must not fail"); return result.value_or(false); } @@ -455,7 +455,7 @@ Result ScalarEqual(const Scalar& lhs, const Scalar& rhs) { const auto& r = std::get>(other); if (!l && !r) return true; if (!l || !r) return false; - return StructLikeEqual(*l, *r); + return StructLikeEqualInternal(*l, *r); }, [](const std::shared_ptr& l, const Scalar& other) -> Result { const auto& r = std::get>(other); @@ -475,6 +475,10 @@ Result ScalarEqual(const Scalar& lhs, const Scalar& rhs) { } // namespace +Result StructLikeEqual(const StructLike& lhs, const StructLike& rhs) { + return StructLikeEqualInternal(lhs, rhs); +} + template StructLikeSet::StructLikeSet(const StructType& type, size_t arena_initial_size) : arena_(arena_initial_size) { diff --git a/src/iceberg/util/struct_like_set.h b/src/iceberg/util/struct_like_set.h index 1dda0621ca..db263c06df 100644 --- a/src/iceberg/util/struct_like_set.h +++ b/src/iceberg/util/struct_like_set.h @@ -34,6 +34,9 @@ namespace iceberg { +/// \brief Compare two StructLike rows by scalar values. +ICEBERG_EXPORT Result StructLikeEqual(const StructLike& lhs, const StructLike& rhs); + /// \brief A set of StructLike rows with type-aware hashing and equality. /// /// As StructLike uses view semantics, this set makes deep copies of inserted rows From 155e484c8f4c1b272c7073d136b4a7b0c8674fc6 Mon Sep 17 00:00:00 2001 From: Abanoub Doss Date: Fri, 3 Jul 2026 16:29:44 -0500 Subject: [PATCH 23/23] ci: share sccache setup across workflows --- .github/actions/save-sccache/action.yml | 39 ++++++ .../sccache/resolve-msvc-key-suffix.ps1 | 55 +++++++++ .github/actions/setup-sccache/action.yml | 57 +++++++++ .github/workflows/aws_test.yml | 24 +--- .github/workflows/cpp-linter.yml | 25 +--- .github/workflows/sanitizer_test.yml | 25 +--- .github/workflows/sql_catalog_test.yml | 45 +------ .github/workflows/test.yml | 115 ++++-------------- 8 files changed, 199 insertions(+), 186 deletions(-) create mode 100644 .github/actions/save-sccache/action.yml create mode 100644 .github/actions/sccache/resolve-msvc-key-suffix.ps1 create mode 100644 .github/actions/setup-sccache/action.yml diff --git a/.github/actions/save-sccache/action.yml b/.github/actions/save-sccache/action.yml new file mode 100644 index 0000000000..224547b04f --- /dev/null +++ b/.github/actions/save-sccache/action.yml @@ -0,0 +1,39 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +name: Save sccache +description: Show sccache stats and save the cache on main branch runs after setup-sccache. + +inputs: + key-prefix: + description: Cache key prefix without the run id or Windows compiler suffix. + required: true + +runs: + using: composite + steps: + - name: Show sccache stats + if: always() + shell: bash + run: sccache --show-stats + + - name: Save sccache cache + if: ${{ github.ref == 'refs/heads/main' }} + uses: actions/cache/save@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0 + with: + path: ${{ github.workspace }}/.sccache + key: ${{ inputs.key-prefix }}${{ env.SCCACHE_KEY_SUFFIX }}-${{ github.run_id }} diff --git a/.github/actions/sccache/resolve-msvc-key-suffix.ps1 b/.github/actions/sccache/resolve-msvc-key-suffix.ps1 new file mode 100644 index 0000000000..f0c0a69e84 --- /dev/null +++ b/.github/actions/sccache/resolve-msvc-key-suffix.ps1 @@ -0,0 +1,55 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +# Resolve the MSVC toolset version for use as an sccache cache key suffix. +# GitHub runner images roll out new cl.exe builds roughly weekly; keying the +# cache on the toolset version avoids silent misses after each bump. + +$PSNativeCommandUseErrorActionPreference = $false + +function Resolve-Suffix { + # Locate vswhere, which ships with every VS installation. + $vswhere = Join-Path ${env:ProgramFiles(x86)} 'Microsoft Visual Studio\Installer\vswhere.exe' + if (-not (Test-Path $vswhere)) { return $null } + + # Ask vswhere for the newest installation that includes the C++ toolset. + $installationPath = (& $vswhere -latest -products * ` + -requires Microsoft.VisualStudio.Component.VC.Tools.x86.x64 ` + -property installationPath | Select-Object -First 1) + if (-not $installationPath) { return $null } + + # Read the default toolset version (e.g. "14.44.35217"). + $versionFile = Join-Path $installationPath.Trim() 'VC\Auxiliary\Build\Microsoft.VCToolsVersion.default.txt' + if (-not (Test-Path $versionFile)) { return $null } + + $version = (Get-Content -Path $versionFile -Raw).Trim() + if ($version) { return "-$version" } + return $null +} + +$suffix = Resolve-Suffix +if (-not $suffix) { + # Degrade gracefully: a shared bucket is better than a failed build. + $suffix = '-unknown' + Write-Host "::warning::could not resolve MSVC toolset version for sccache key, using '$suffix'" +} + +# Export to both step outputs (for setup-sccache's restore key) and env (for +# save-sccache's save key later in the same job). +Add-Content -Path $env:GITHUB_OUTPUT -Value "suffix=$suffix" +Add-Content -Path $env:GITHUB_ENV -Value "SCCACHE_KEY_SUFFIX=$suffix" +Write-Host "Resolved SCCACHE_KEY_SUFFIX=$suffix" diff --git a/.github/actions/setup-sccache/action.yml b/.github/actions/setup-sccache/action.yml new file mode 100644 index 0000000000..eecf6eb66c --- /dev/null +++ b/.github/actions/setup-sccache/action.yml @@ -0,0 +1,57 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +name: Set up sccache +description: Configure sccache, restore its cache, and start the sccache action. + +inputs: + key-prefix: + description: Cache key prefix without the run id or Windows compiler suffix. + required: true + cache-size: + description: Maximum sccache cache size. + required: false + default: 2G + +runs: + using: composite + steps: + - name: Configure sccache environment + shell: bash + run: | + echo "SCCACHE_DIR=${{ github.workspace }}/.sccache" >> "$GITHUB_ENV" + echo "SCCACHE_CACHE_SIZE=${{ inputs.cache-size }}" >> "$GITHUB_ENV" + echo "SCCACHE_KEY_SUFFIX=" >> "$GITHUB_ENV" + + # Windows caches include the MSVC toolset version because GitHub runner image + # rollouts can change the compiler while keeping the same image label. + - name: Resolve MSVC toolset version for sccache key + if: ${{ runner.os == 'Windows' }} + id: msvc-key + shell: pwsh + run: '& "${{ github.action_path }}/../sccache/resolve-msvc-key-suffix.ps1"' + + - name: Restore sccache cache + uses: actions/cache/restore@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0 + with: + path: ${{ github.workspace }}/.sccache + key: ${{ inputs.key-prefix }}${{ steps.msvc-key.outputs.suffix }}-${{ github.run_id }} + restore-keys: | + ${{ inputs.key-prefix }}${{ steps.msvc-key.outputs.suffix }}- + + - name: Setup sccache + uses: mozilla-actions/sccache-action@9e7fa8a12102821edf02ca5dbea1acd0f89a2696 # v0.0.10 diff --git a/.github/workflows/aws_test.yml b/.github/workflows/aws_test.yml index 0ed9685b73..65596464f0 100644 --- a/.github/workflows/aws_test.yml +++ b/.github/workflows/aws_test.yml @@ -73,8 +73,6 @@ jobs: AWS_DEFAULT_REGION: us-east-1 AWS_ENDPOINT_URL: http://127.0.0.1:9000 AWS_EC2_METADATA_DISABLED: "TRUE" - SCCACHE_DIR: ${{ github.workspace }}/.sccache - SCCACHE_CACHE_SIZE: "2G" steps: - name: Checkout iceberg-cpp uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 @@ -114,26 +112,16 @@ jobs: if: ${{ matrix.s3 == 'ON' }} shell: bash run: bash ci/scripts/start_minio.sh - - name: Restore sccache cache - uses: actions/cache/restore@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0 + - name: Set up sccache + uses: ./.github/actions/setup-sccache with: - path: ${{ github.workspace }}/.sccache - key: sccache-aws-${{ matrix.runs-on }}-bundle${{ matrix.bundle_awssdk }}-s3${{ matrix.s3 }}-sigv4${{ matrix.sigv4 }}-${{ github.run_id }} - restore-keys: | - sccache-aws-${{ matrix.runs-on }}-bundle${{ matrix.bundle_awssdk }}-s3${{ matrix.s3 }}-sigv4${{ matrix.sigv4 }}- - - name: Setup sccache - uses: mozilla-actions/sccache-action@9e7fa8a12102821edf02ca5dbea1acd0f89a2696 # v0.0.10 + key-prefix: sccache-aws-${{ matrix.runs-on }}-bundle${{ matrix.bundle_awssdk }}-s3${{ matrix.s3 }}-sigv4${{ matrix.sigv4 }} - name: Build and test Iceberg shell: bash env: CMAKE_TOOLCHAIN_FILE: ${{ startsWith(matrix.runs-on, 'ubuntu') && matrix.bundle_awssdk == 'OFF' && '/usr/local/share/vcpkg/scripts/buildsystems/vcpkg.cmake' || '' }} run: ci/scripts/build_iceberg.sh "$(pwd)" OFF ON ${{ matrix.s3 }} ${{ matrix.sigv4 }} ${{ matrix.bundle_awssdk }} - - name: Show sccache stats - shell: bash - run: sccache --show-stats - - name: Save sccache cache - if: github.ref == 'refs/heads/main' - uses: actions/cache/save@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0 + - name: Save sccache + uses: ./.github/actions/save-sccache with: - path: ${{ github.workspace }}/.sccache - key: sccache-aws-${{ matrix.runs-on }}-bundle${{ matrix.bundle_awssdk }}-s3${{ matrix.s3 }}-sigv4${{ matrix.sigv4 }}-${{ github.run_id }} + key-prefix: sccache-aws-${{ matrix.runs-on }}-bundle${{ matrix.bundle_awssdk }}-s3${{ matrix.s3 }}-sigv4${{ matrix.sigv4 }} diff --git a/.github/workflows/cpp-linter.yml b/.github/workflows/cpp-linter.yml index 44dd6e350f..c1381f136c 100644 --- a/.github/workflows/cpp-linter.yml +++ b/.github/workflows/cpp-linter.yml @@ -45,9 +45,6 @@ jobs: permissions: contents: read pull-requests: write - env: - SCCACHE_DIR: ${{ github.workspace }}/.sccache - SCCACHE_CACHE_SIZE: "2G" steps: - name: Checkout iceberg-cpp uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 @@ -58,15 +55,10 @@ jobs: run: | sudo apt-get update sudo apt-get install -y libcurl4-openssl-dev libsqlite3-dev libpq-dev default-libmysqlclient-dev - - name: Restore sccache cache - uses: actions/cache/restore@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0 + - name: Set up sccache + uses: ./.github/actions/setup-sccache with: - path: ${{ github.workspace }}/.sccache - key: sccache-cpp-linter-ubuntu-${{ github.run_id }} - restore-keys: | - sccache-cpp-linter-ubuntu- - - name: Setup sccache - uses: mozilla-actions/sccache-action@9e7fa8a12102821edf02ca5dbea1acd0f89a2696 # v0.0.10 + key-prefix: sccache-cpp-linter-ubuntu - name: Run build env: CC: gcc-14 @@ -82,15 +74,10 @@ jobs: -DICEBERG_SQL_POSTGRESQL=ON \ -DICEBERG_SQL_MYSQL=ON cmake --build . - - name: Show sccache stats - shell: bash - run: sccache --show-stats - - name: Save sccache cache - if: github.ref == 'refs/heads/main' - uses: actions/cache/save@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0 + - name: Save sccache + uses: ./.github/actions/save-sccache with: - path: ${{ github.workspace }}/.sccache - key: sccache-cpp-linter-ubuntu-${{ github.run_id }} + key-prefix: sccache-cpp-linter-ubuntu - uses: cpp-linter/cpp-linter-action@0f6d1b8d7e38b584cbee606eb23d850c217d54f8 # v2.15.1 id: linter if: github.event_name == 'pull_request' diff --git a/.github/workflows/sanitizer_test.yml b/.github/workflows/sanitizer_test.yml index cd9797b9fa..13bad0a7af 100644 --- a/.github/workflows/sanitizer_test.yml +++ b/.github/workflows/sanitizer_test.yml @@ -38,9 +38,6 @@ jobs: if: ${{ github.event_name != 'pull_request' || github.event.pull_request.draft == false }} name: "ASAN and UBSAN Tests" runs-on: ubuntu-26.04 - env: - SCCACHE_DIR: ${{ github.workspace }}/.sccache - SCCACHE_CACHE_SIZE: "2G" steps: - name: Checkout iceberg-cpp uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 @@ -49,15 +46,10 @@ jobs: - name: Install dependencies shell: bash run: sudo apt-get update && sudo apt-get install -y libcurl4-openssl-dev - - name: Restore sccache cache - uses: actions/cache/restore@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0 + - name: Set up sccache + uses: ./.github/actions/setup-sccache with: - path: ${{ github.workspace }}/.sccache - key: sccache-sanitizer-ubuntu-${{ github.run_id }} - restore-keys: | - sccache-sanitizer-ubuntu- - - name: Setup sccache - uses: mozilla-actions/sccache-action@9e7fa8a12102821edf02ca5dbea1acd0f89a2696 # v0.0.10 + key-prefix: sccache-sanitizer-ubuntu - name: Configure and Build with ASAN & UBSAN env: CC: gcc-14 @@ -67,15 +59,10 @@ jobs: cmake .. -G Ninja -DCMAKE_BUILD_TYPE=Debug -DICEBERG_ENABLE_ASAN=ON -DICEBERG_ENABLE_UBSAN=ON \ -DCMAKE_C_COMPILER_LAUNCHER=sccache -DCMAKE_CXX_COMPILER_LAUNCHER=sccache cmake --build . --verbose - - name: Show sccache stats - shell: bash - run: sccache --show-stats - - name: Save sccache cache - if: github.ref == 'refs/heads/main' - uses: actions/cache/save@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0 + - name: Save sccache + uses: ./.github/actions/save-sccache with: - path: ${{ github.workspace }}/.sccache - key: sccache-sanitizer-ubuntu-${{ github.run_id }} + key-prefix: sccache-sanitizer-ubuntu - name: Run Tests working-directory: build env: diff --git a/.github/workflows/sql_catalog_test.yml b/.github/workflows/sql_catalog_test.yml index 108ff8ae2e..e9a5824ac5 100644 --- a/.github/workflows/sql_catalog_test.yml +++ b/.github/workflows/sql_catalog_test.yml @@ -60,9 +60,6 @@ jobs: runs-on: windows-2025 cmake_build_type: Debug cmake_extra_args: -DCMAKE_TOOLCHAIN_FILE=C:/vcpkg/scripts/buildsystems/vcpkg.cmake - env: - SCCACHE_DIR: ${{ github.workspace }}/.sccache - SCCACHE_CACHE_SIZE: "2G" steps: - name: Checkout iceberg-cpp uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 @@ -73,26 +70,6 @@ jobs: uses: ilammy/msvc-dev-cmd@0b201ec74fa43914dc39ae48a89fd1d8cb592756 # v1.13.0 with: arch: x64 - # GitHub patches the Windows image roughly weekly via gradual rollouts, so - # back-to-back runs can hit different cl.exe builds. sccache keys on the compiler, - # so a shared key would miss after each bump; the version keeps caches separate. - # Non-Windows legs get an empty suffix, leaving their keys unchanged. - - name: Resolve MSVC version for sccache key - if: ${{ startsWith(matrix.runs-on, 'windows') }} - shell: pwsh - run: | - $PSNativeCommandUseErrorActionPreference = $false - $banner = (cl.exe 2>&1 | Out-String) - if ($banner -match 'Version ([\d.]+)') { - $suffix = "-$($Matches[1])" - } else { - # The key is only a cache hint, so degrade to a shared bucket rather than - # fail the build; the warning flags that the parse needs fixing. - $suffix = '-unknown' - Write-Host "::warning::could not parse cl.exe version for sccache key, using '$suffix'; banner was: $banner" - } - Add-Content -Path $env:GITHUB_ENV -Value "SCCACHE_KEY_SUFFIX=$suffix" - Write-Host "SCCACHE_KEY_SUFFIX=$suffix" - name: Install dependencies on Ubuntu if: ${{ startsWith(matrix.runs-on, 'ubuntu') }} shell: bash @@ -114,15 +91,10 @@ jobs: shell: pwsh run: | vcpkg install zlib:x64-windows nlohmann-json:x64-windows nanoarrow:x64-windows roaring:x64-windows sqlite3:x64-windows - - name: Restore sccache cache - uses: actions/cache/restore@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0 + - name: Set up sccache + uses: ./.github/actions/setup-sccache with: - path: ${{ github.workspace }}/.sccache - key: sccache-sqlcatalog-${{ matrix.runs-on }}${{ env.SCCACHE_KEY_SUFFIX }}-${{ github.run_id }} - restore-keys: | - sccache-sqlcatalog-${{ matrix.runs-on }}${{ env.SCCACHE_KEY_SUFFIX }}- - - name: Setup sccache - uses: mozilla-actions/sccache-action@9e7fa8a12102821edf02ca5dbea1acd0f89a2696 # v0.0.10 + key-prefix: sccache-sqlcatalog-${{ matrix.runs-on }} - name: Configure Iceberg shell: bash run: | @@ -140,15 +112,10 @@ jobs: - name: Build SQL catalog tests shell: bash run: cmake --build build --target sql_catalog_test - - name: Show sccache stats - shell: bash - run: sccache --show-stats - - name: Save sccache cache - if: github.ref == 'refs/heads/main' - uses: actions/cache/save@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0 + - name: Save sccache + uses: ./.github/actions/save-sccache with: - path: ${{ github.workspace }}/.sccache - key: sccache-sqlcatalog-${{ matrix.runs-on }}${{ env.SCCACHE_KEY_SUFFIX }}-${{ github.run_id }} + key-prefix: sccache-sqlcatalog-${{ matrix.runs-on }} - name: Run SQL catalog tests shell: bash run: ctest --test-dir build -R '^sql_catalog_test$' --output-on-failure -C ${{ matrix.cmake_build_type }} diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 4901e8ea95..8c53af6793 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -48,9 +48,6 @@ jobs: cmake_build_type: - Debug - RelWithDebInfo - env: - SCCACHE_DIR: ${{ github.workspace }}/.sccache - SCCACHE_CACHE_SIZE: "2G" steps: - name: Checkout iceberg-cpp uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 @@ -59,30 +56,20 @@ jobs: - name: Install dependencies shell: bash run: sudo apt-get update && sudo apt-get install -y libcurl4-openssl-dev - - name: Restore sccache cache - uses: actions/cache/restore@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0 + - name: Set up sccache + uses: ./.github/actions/setup-sccache with: - path: ${{ github.workspace }}/.sccache - key: sccache-test-ubuntu-${{ matrix.cmake_build_type }}-${{ github.run_id }} - restore-keys: | - sccache-test-ubuntu-${{ matrix.cmake_build_type }}- - - name: Setup sccache - uses: mozilla-actions/sccache-action@9e7fa8a12102821edf02ca5dbea1acd0f89a2696 # v0.0.10 + key-prefix: sccache-test-ubuntu-${{ matrix.cmake_build_type }} - name: Build Iceberg shell: bash env: CC: gcc-14 CXX: g++-14 run: ci/scripts/build_iceberg.sh $(pwd) ON ON OFF OFF ON ${{ matrix.cmake_build_type }} - - name: Show sccache stats - shell: bash - run: sccache --show-stats - - name: Save sccache cache - if: github.ref == 'refs/heads/main' - uses: actions/cache/save@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0 + - name: Save sccache + uses: ./.github/actions/save-sccache with: - path: ${{ github.workspace }}/.sccache - key: sccache-test-ubuntu-${{ matrix.cmake_build_type }}-${{ github.run_id }} + key-prefix: sccache-test-ubuntu-${{ matrix.cmake_build_type }} - name: Build Example shell: bash env: @@ -96,35 +83,22 @@ jobs: timeout-minutes: 30 strategy: fail-fast: false - env: - SCCACHE_DIR: ${{ github.workspace }}/.sccache - SCCACHE_CACHE_SIZE: "2G" steps: - name: Checkout iceberg-cpp uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 with: persist-credentials: false - - name: Restore sccache cache - uses: actions/cache/restore@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0 + - name: Set up sccache + uses: ./.github/actions/setup-sccache with: - path: ${{ github.workspace }}/.sccache - key: sccache-test-macos-${{ github.run_id }} - restore-keys: | - sccache-test-macos- - - name: Setup sccache - uses: mozilla-actions/sccache-action@9e7fa8a12102821edf02ca5dbea1acd0f89a2696 # v0.0.10 + key-prefix: sccache-test-macos - name: Build Iceberg shell: bash run: ci/scripts/build_iceberg.sh $(pwd) OFF ON - - name: Show sccache stats - shell: bash - run: sccache --show-stats - - name: Save sccache cache - if: github.ref == 'refs/heads/main' - uses: actions/cache/save@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0 + - name: Save sccache + uses: ./.github/actions/save-sccache with: - path: ${{ github.workspace }}/.sccache - key: sccache-test-macos-${{ github.run_id }} + key-prefix: sccache-test-macos - name: Build Example shell: bash run: ci/scripts/build_example.sh $(pwd)/example @@ -135,9 +109,6 @@ jobs: timeout-minutes: 60 strategy: fail-fast: false - env: - SCCACHE_DIR: ${{ github.workspace }}/.sccache - SCCACHE_CACHE_SIZE: "2G" steps: - name: Checkout iceberg-cpp uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 @@ -147,24 +118,6 @@ jobs: uses: ilammy/msvc-dev-cmd@0b201ec74fa43914dc39ae48a89fd1d8cb592756 # v1.13.0 with: arch: x64 - # GitHub patches the Windows image roughly weekly via gradual rollouts, so - # back-to-back runs can hit different cl.exe builds. sccache keys on the compiler, - # so a shared key would miss after each bump; the version keeps caches separate. - - name: Resolve MSVC version for sccache key - shell: pwsh - run: | - $PSNativeCommandUseErrorActionPreference = $false - $banner = (cl.exe 2>&1 | Out-String) - if ($banner -match 'Version ([\d.]+)') { - $msvc = $Matches[1] - } else { - # The key is only a cache hint, so degrade to a shared bucket rather than - # fail the build; the warning flags that the parse needs fixing. - $msvc = 'unknown' - Write-Host "::warning::could not parse cl.exe version for sccache key, using '$msvc'; banner was: $banner" - } - Add-Content -Path $env:GITHUB_ENV -Value "MSVC_VER=$msvc" - Write-Host "Resolved MSVC_VER=$msvc" - name: Cache vcpkg packages uses: actions/cache@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0 id: vcpkg-cache @@ -176,28 +129,20 @@ jobs: shell: pwsh run: | vcpkg install zlib:x64-windows nlohmann-json:x64-windows nanoarrow:x64-windows roaring:x64-windows cpr:x64-windows - - name: Restore sccache cache - uses: actions/cache/restore@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0 + - name: Set up sccache + uses: ./.github/actions/setup-sccache with: - path: ${{ github.workspace }}/.sccache - key: sccache-test-windows-${{ env.MSVC_VER }}-${{ github.run_id }} - restore-keys: | - sccache-test-windows-${{ env.MSVC_VER }}- - - name: Setup sccache - uses: mozilla-actions/sccache-action@9e7fa8a12102821edf02ca5dbea1acd0f89a2696 # v0.0.10 + key-prefix: sccache-test-windows - name: Build Iceberg shell: pwsh run: | $ErrorActionPreference = "Stop" bash -lc 'ci/scripts/build_iceberg.sh $(pwd) OFF ON' if ($LASTEXITCODE -ne 0) { exit $LASTEXITCODE } - sccache --show-stats - - name: Save sccache cache - if: github.ref == 'refs/heads/main' - uses: actions/cache/save@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0 + - name: Save sccache + uses: ./.github/actions/save-sccache with: - path: ${{ github.workspace }}/.sccache - key: sccache-test-windows-${{ env.MSVC_VER }}-${{ github.run_id }} + key-prefix: sccache-test-windows - name: Build Example shell: pwsh run: | @@ -208,9 +153,6 @@ jobs: name: Meson - ${{ matrix.title }} runs-on: ${{ matrix.runs-on }} timeout-minutes: 30 - env: - SCCACHE_DIR: ${{ github.workspace }}/.sccache - SCCACHE_CACHE_SIZE: "2G" strategy: max-parallel: 15 fail-fast: false @@ -246,27 +188,18 @@ jobs: run: | echo "CC=sccache ${{ matrix.CC }}" >> $GITHUB_ENV echo "CXX=sccache ${{ matrix.CXX }}" >> $GITHUB_ENV - - name: Restore sccache cache - uses: actions/cache/restore@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0 + - name: Set up sccache + uses: ./.github/actions/setup-sccache with: - path: ${{ github.workspace }}/.sccache - key: sccache-meson-${{ matrix.runs-on }}-${{ github.run_id }} - restore-keys: | - sccache-meson-${{ matrix.runs-on }}- - - name: Setup sccache - uses: mozilla-actions/sccache-action@9e7fa8a12102821edf02ca5dbea1acd0f89a2696 # v0.0.10 + key-prefix: sccache-meson-${{ matrix.runs-on }} - name: Build Iceberg run: | meson setup builddir ${{ matrix.meson-setup-args || '' }} meson compile -C builddir - - name: Show sccache stats - run: sccache --show-stats - - name: Save sccache cache - if: github.ref == 'refs/heads/main' - uses: actions/cache/save@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0 + - name: Save sccache + uses: ./.github/actions/save-sccache with: - path: ${{ github.workspace }}/.sccache - key: sccache-meson-${{ matrix.runs-on }}-${{ github.run_id }} + key-prefix: sccache-meson-${{ matrix.runs-on }} - name: Test Iceberg run: | meson test -C builddir --timeout-multiplier 0 --print-errorlogs